def validate(model, dataset, epoch): """Perform an evaluation of `model` on the examples from `dataset`.""" batch_loss, avg_loss = 0., 0. start = time.time() for batch, (images, loc_trues, cls_trues) in enumerate(tfe.Iterator(dataset)): loc_preds, cls_preds = model(images) loc_loss, cls_loss = loss_fn(loc_preds, loc_trues, cls_preds, cls_trues, num_classes=conf.num_class) batch_loss += loc_loss + cls_loss avg_loss = batch_loss / (batch + 1) if batch % conf.log_interval == 0: fmt = [ i.numpy() for i in [loc_loss, cls_loss, loc_loss + cls_loss, avg_loss] ] print( "[EVALUATION] Batch: {}({:.0f}/{})\t".format( batch, epoch, conf.num_epochs), "loc_loss: {:.6f} | cls_loss: {:.6f} | total_loss: {:.6f} | avg_loss: {:.2f}" .format(*fmt)) time_in_ms = (time.time() - start) / 60 print('[EVALUATION] Batch: {} Average time: {:.2f}min\n'.format( batch, time_in_ms)) with tf.contrib.summary.always_record_summaries(): tf.contrib.summary.scalar('avg_loss', avg_loss) return avg_loss
def train(self, mode=True): train_loader = DataLoader(dataset=self.train_dataset, batch_size=self.batch_size, drop_last=True, shuffle=False) if len(self.train_dataset) == 0: print("No training data was provided!") return generation = 0 for epoch in range(self.iteration): print('\n\nTraining epoch: %d' % epoch) self.scheduler.step() print("Learning_rate: ", self.scheduler.get_lr()[0]) output3 = 0 label3 = 0 gse3 = 0 loss = 0 for items in train_loader: generation += 1 img, label3, label2, label1, gse3, gse2 = self.to_cuda(*items) self.optimizer.zero_grad() output1, output2, output3 = self.model(img) loss = loss_fn(output1, output2, output3, label1.long(), label2.long(), label3.long(), gse2, gse3, self.weight) loss.backward() self.optimizer.step() output3 = np.argmax(output3.data.cpu().numpy(), 1) label3 = np.squeeze(label3.data.cpu().numpy()) gse3 = gse3.data.cpu().numpy() miou = self.IOU(output3, label3, self.num_class) acc = np.equal(output3, label3) gse_acc = np.sum(acc * gse3) / np.sum(gse3) acc = np.mean(acc) print('----------Loss = {:5.5f} ; Accuracy = {:2.2f}% ---------'. format(np.float64(loss), acc * 100)) print('----------GSE_acc = {:2.2f}%; mIOU = {:2.2f}% ---------'. format(gse_acc, miou)) cv2.imwrite('../pic3/{}.png'.format(epoch), output3[0].reshape(512, 512) * 20) if epoch % 10 == 0: self.save(epoch)
def model_loss(images, loc_trues, cls_trues): loc_preds, cls_preds = model(images, training=True) loc_loss, cls_loss = loss_fn(loc_preds, loc_trues, cls_preds, cls_trues, num_classes=conf.num_class) total_loss = loc_loss + cls_loss tf.contrib.summary.scalar('loc_loss', loc_loss) tf.contrib.summary.scalar('cls_loss', cls_loss) tf.contrib.summary.scalar('total_loss', total_loss) return total_loss
def validation(net: Model, loader: Loader): val_accuracy = [] loss = [] queue_seg = Queue(maxsize=1) queue_weightmap = Queue(maxsize=1) p = Process(target=extract_weight_map, args=(queue_seg, queue_weightmap)) p.start() for i, (image_batch, seg_batch) in enumerate(loader.get_minibatch(train=False)): queue_seg.put(seg_batch) pred = net(image_batch) weight_map = queue_weightmap.get() loss.append( loss_fn(seg=seg_batch, predictions=pred, weight_map=weight_map)) val_accuracy.append(Jaccard_Index(predictions=pred, seg=seg_batch)) if i == config.validation_steps: return np.mean(val_accuracy), np.mean(loss) p.terminate()
def train(net: Model, optimizer: tf.compat.v2.optimizers, loader: Loader): if not os.path.exists(config.log_dir): os.mkdir(config.log_dir) queue_seg = Queue(maxsize=1) queue_weightmap = Queue(maxsize=1) p = Process(target=extract_weight_map, args=(queue_seg, queue_weightmap)) p.start() Tensorcallback = callbacks.TensorBoard(config.log_dir, write_graph=False, write_images=False) Tensorcallback.set_model(net) for epoch in range(config.epochs): train_accuracy = [] train_loss = [] for iteration, (image_batch, seg_batch) in enumerate(loader.get_minibatch()): queue_seg.put(seg_batch) with tf.GradientTape() as tape: pred = net(image_batch) weight_map = queue_weightmap.get() train_loss.append( loss_fn(seg_batch, pred, weight_map=weight_map)) train_accuracy.append(Jaccard_Index(pred, seg_batch)) print("Epoch {}/{} iteration {} : loss {} accuracy {}".format( epoch, config.epochs, iteration, train_loss[-1], train_accuracy[-1])) grads = tape.gradient(train_loss[-1], net.trainable_variables) optimizer.apply_gradients(zip(grads, net.trainable_variables)) if iteration == config.steps_per_epoch: break train_loss_mean = np.mean(train_loss) train_accuracy_mean = np.mean(train_accuracy) if epoch % config.val_freq == 0: # output validation accuracy val_accuracy, val_loss = validation(net, loader) logs = { 'acc': train_accuracy_mean, 'loss': train_loss_mean, 'val_acc': val_accuracy, 'val_loss': val_loss } Tensorcallback.on_epoch_end(epoch=epoch, logs=logs) Tensorcallback.on_train_end('_') p.terminate()
def train_fn(data_loader, model, optimizer, device, scheduler): model.train() for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)): ids = d["ids"] token_type_ids = d["token_type_ids"] mask = d["mask"] targets = d["targets"] ids = ids.to(device, dtype=torch.long) token_type_ids = token_type_ids.to(device, dtype=torch.long) mask = mask.to(device, dtype=torch.long) targets = targets.to(device, dtype=torch.float) optimizer.zero_grad() outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids) loss = loss.loss_fn(outputs, targets) loss.backward() optimizer.step() scheduler.step()
def __init__(self, model, device, config): self.model = model self.device = device self.config = config self.logger = logging.getLogger('training') self.best_dice = 0 self.best_thres = None self.epoch = 0 self.best_loss = np.inf self.oof = None self.monitored_metric = None if not os.path.exists(self.config.SAVE_PATH): os.makedirs(self.config.SAVE_PATH) if not os.path.exists(self.config.LOG_PATH): os.makedirs(self.config.LOG_PATH) self.loss = loss_fn(config).to(self.device) self.optimizer = getattr(torch.optim, config.optimizer)(self.model.parameters(), **config.optimizer_params[config.optimizer]) self.scheduler = getattr(torch.optim.lr_scheduler, config.scheduler)(optimizer=self.optimizer, **config.scheduler_params[config.scheduler])
n_batches // n_batch_verif), np.zeros( n_batches // n_batch_verif), np.zeros(n_batches // n_batch_verif) vs, vf, tg = [], [], [] idx = -1 for batch_idx, (data, target, img_file, class_id) in enumerate(train_loader): data_set = data[np.arange(0, batch_size, n_samples)].to(device) data_query = data[np.arange(1, batch_size, n_samples)].to(device) v_set = model(data_set, m=m_set) # single vector per set v_f = model(data_query, m=1) # single vector per query Sim = torch.mm(F.normalize(v_set, p=2, dim=1), F.normalize(v_f, p=2, dim=1).t()) output = logisticReg(Sim.unsqueeze(-1)).squeeze() loss_outputs, accuracy = loss_fn(output, len(v_f), m_set) tot_acc += accuracy tot_loss += loss_outputs vs.append(v_set) vf.append(v_f) tg.append(target) if (batch_idx + 1) % n_batch_verif == 0: idx += 1 vs = torch.sign( torch.stack(vs).flatten(start_dim=0, end_dim=1) + 1e-16) vf = torch.sign( torch.stack(vf).flatten(start_dim=0, end_dim=1) + 1e-16) tg = torch.stack(tg).flatten(start_dim=0, end_dim=1)
def main(): # Training settings parser = argparse.ArgumentParser(description='MultiView Training') parser.add_argument('--root', type=str, default="kitti", help='dataset root to load') parser.add_argument('--num-inputs', type=int, default=4, help='number of aux views') parser.add_argument('--checkpoint', type=str, default=None, help='checkpoint to start from') parser.add_argument('--batch-size', type=int, default=8, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=8, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=1000000, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr-p', type=float, default=5e-5, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--lr-d', type=float, default=1e-4, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--lr-f', type=float, default=1e-4, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.9, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--l2', type=float, default=5.0E-4) parser.add_argument('--decay-frequency', type=int, default=500) parser.add_argument('--deterministic', action='store_true', help='fixed seed PRNG') parser.add_argument('--seed', type=int, default=13, metavar='S', help='random seed (default: 1)') parser.add_argument('--multi', action='store_true', default=False, help='use all available GPUs') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') args = parser.parse_args() device, dev_count = setup_devices(args.no_cuda, args.deterministic, args.seed) if not args.multi: dev_count = 1 args.batch_size *= dev_count train_dataset, valid_dataset = create_default_splits(args.num_inputs, args.root, cache=args.multi) viz = SummaryWriter() train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=1, pin_memory=True, drop_last=False) valid_loader = DataLoader(valid_dataset, batch_size=args.test_batch_size, shuffle=False, num_workers=1, pin_memory=True) model = Multi2Novel().to(device) flow_params = [] pixel_params = [] pixel_gan_params = [] discr_params = [] for k, v in model.named_parameters(): if k.startswith('flow_predictor'): flow_params.append(v) elif k.startswith('pixel_generator'): pixel_params.append(v) pixel_gan_params.append(v) elif k.startswith('discriminator'): discr_params.append(v) pixel_gan_params.append(v) if args.multi: model = nn.DataParallel(model) opt_pixel = optim.Adam(pixel_params, lr=args.lr_p, weight_decay=args.l2) opt_pixel_gan = optim.Adam(pixel_gan_params, lr=args.lr_p, weight_decay=args.l2) opt_flow = optim.Adam(flow_params, lr=args.lr_f, weight_decay=args.l2) opt_discr = optim.Adam(discr_params, lr=args.lr_d, weight_decay=args.l2) it = 0 epoch_start = 1 best_loss = inf if args.checkpoint is not None: checkpoint_status = load_checkpoint(model, [opt_pixel, opt_pixel_gan, opt_flow, opt_discr], filename=args.checkpoint.split(".")[0]) if checkpoint_status is not None: it, epoch_start, best_loss = checkpoint_status batch_count = len(train_loader) with tqdm.trange(epoch_start, args.epochs + 1, desc="epochs") as epoch_bar, tqdm.tqdm( total=batch_count, leave=False, desc="train" ) as train_bar: for epoch in epoch_bar: avg_train_loss = 0.0 batch_it = 0.0 for batch in train_loader: model.train() opt_pixel.zero_grad() opt_pixel_gan.zero_grad() opt_flow.zero_grad() opt_discr.zero_grad() (inp_img, inp_pose), (target_img, target_pose) = batch inputs = (inp_img.to(device), inp_pose.to(device)) targets = (target_img.to(device), target_pose.to(device)) outputs = model(inputs, targets) px_loss, fl_loss, px_gan_loss, d_loss = loss_fn(*outputs, targets[0]) train_gan = it > 300000 train_loss = 0.0 opts = [] if train_gan: if it % 2 > 0: # train generator train_loss = px_gan_loss + fl_loss opts += [opt_pixel_gan, opt_flow] else: # train discriminator train_loss = d_loss opts += [opt_discr,] else: # train pixel predictor + flow estimator train_loss = px_loss + fl_loss opts += [opt_pixel, opt_flow] if viz: viz.add_scalar('loss/training', train_loss, it) train_loss.backward() for opt in opts: opt.step() it += 1 train_bar.update() train_bar.set_postfix(dict(total_it=it)) epoch_bar.refresh() if (it % batch_count) == 0: train_bar.close() # validation model.eval() eval_dict = {} val_loss = 0.0 for i, data in tqdm.tqdm( enumerate(valid_loader, 0), total=len(valid_loader), leave=False, desc="val" ): loss = None acc = None with torch.no_grad(): (inp_img, inp_pose), (target_img, target_pose) = data inputs = (inp_img.to(device), inp_pose.to(device)) targets = (target_img.to(device), target_pose.to(device)) outputs = model(inputs, targets) px_loss, fl_loss, _, _ = loss_fn(*outputs, targets[0]) per_batch_loss = px_loss + fl_loss val_loss += (per_batch_loss - val_loss) / (i+1) if viz: viz.add_scalar('loss/validation', val_loss, it) is_best = val_loss < best_loss best_loss = min(best_loss, val_loss) save_checkpoint( checkpoint_state( model, [opt_pixel, opt_pixel_gan, opt_flow, opt_discr], val_loss, epoch, it, False ), is_best, filename="checkpoints/step_ckpt", bestname="checkpoints/best_ckpt", ) train_bar = tqdm.tqdm( total=batch_count, leave=False, desc="train" ) train_bar.set_postfix(dict(total_it=it)) if viz: viz.flush() # if viz is not None: viz.close()
def build(self, reuse=False): self.reuse = reuse with tf.variable_scope(self.name, reuse=self.reuse): # train set self.mix = self.loader.mix_queue # mixture batch self.source = self.loader.source_queue # source batch self.other = self.mix - self.source # test samples (placeholder) self.mix_test = tf.placeholder(tf.float32, shape=(1,16384), name="mixture_sample") self.source_test = tf.placeholder(tf.float32, shape=(1,16384), name="source_sample") # define network self.FE = FrontEnd() if config.network == "unet": self.SEP = Unet(model=config.model) elif config.network == "lstm": self.SEP = LSTM(model=config.model) elif config.network == "bilstm": self.SEP = BiLSTM(model=config.model) elif config.network == "dilated": self.SEP = Dilated(model=config.model) else: print("No model chosen") raise ValueError self.BE = BackEnd() functions = [self.FE, self.SEP, self.BE] if config.network2 == "unet": self.SEP2 = Unet(model=config.model2) self.CON = Confidence(model=config.confidence) functions = functions + [self.SEP2, self.CON] elif config.network2 == "lstm": self.SEP2 = LSTM(model=config.model2) self.CON = Confidence(model=config.confidence) functions = functions + [self.SEP2, self.CON] elif config.network2 == "bilstm": self.SEP2 = BiLSTM(model=config.model2) self.CON = Confidence(model=config.confidence) functions = functions + [self.SEP2, self.CON] elif config.network2 == "dilated": self.SEP2 = Dilated(model=config.model2) self.CON = Confidence(model=config.confidence) functions = functions + [self.SEP2, self.CON] else: print("No model chosen") input = {"mix": self.mix, "source": self.source} input_test = {"mix": self.mix_test, "source": self.source_test} # draw graph self.graph = build_graph() self.graph_test = build_graph(train=False) self.graph(input, functions) self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) self.graph_test(input_test, functions) # variable count self.variable_size = np.sum( np.array([np.prod(np.array(v.get_shape().as_list())) for v in tf.trainable_variables()])) print("\n total varialbes : {}".format(self.variable_size), "\n") for v in tf.trainable_variables(): print(v, "{}".format(np.prod(np.array(v.get_shape().as_list())))) # Define loss and summarize # self.loss_pre = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec, # self.source, self.other, loss_type=config.loss) + \ # loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec, # self.source, self.other, loss_type=config.loss))/2 if config.hybrid: if config.loss_seq == 'mid': self.loss = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + \ loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) elif config.loss_seq == 'end': self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + \ loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec, self.source, self.other, loss_type=config.loss) elif config.loss_seq == 'both': self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss))/2 elif config.loss_seq == 'two': self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + \ loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss)) / 2 elif config.loss_seq == 'first': self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + \ loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.graph.estimated = self.graph.estimated1 self.graph_test.estimated = self.graph_test.estimated1 elif config.loss_seq == 'second': self.loss_pre = loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.loss = loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec, self.source, self.other, loss_type=config.loss) + \ loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.graph.estimated = self.graph.estimated2 self.graph_test.estimated = self.graph_test.estimated2 else: raise AssertionError("wrong config.loss_seq !!") else: self.loss = loss_fn(self.graph.masked_spec, self.graph.estimated, self.graph.source_spec, self.source, self.other, loss_type=config.loss) self.loss_summary = tf.summary.scalar("loss", self.loss) self.alpha_summary = tf.summary.scalar("alpha", tf.reduce_mean(self.graph.alpha)) self.summary = tf.summary.merge_all() print("\n loss type : %s \n" % config.loss)
epoch_time_start = time.time() model.train() logisticReg.train() for batch_idx, value in enumerate(train_loader): # data: (batch_size,3,224,224) data = value[0] target = value[1] data_set = data[np.arange(0, batch_size, n_samples)].to(device) data_query = data[np.arange(1, batch_size, n_samples)].to(device) v_set, code_set = model(data_set, m=m_set) # single vector per set v_f, code_f = model(data_query, m=1) # single vector per query # Sim = torch.mm(code_set, code_f.t()) Sim = torch.mm(F.normalize(code_set, p=2, dim=1), F.normalize(code_f, p=2, dim=1).t()) output = logisticReg(Sim.unsqueeze(-1)).squeeze() loss1, accuracy = loss_fn(output, len(code_f), m_set) h = torch.cat([v_set, v_f], dim=0) loss2 = torch.mean( torch.abs( torch.pow( torch.abs(h) - Variable(torch.ones(h.size()).cuda()), 3))) loss_outputs = loss1 + alpha * loss2 tot_acc += accuracy tot_loss += loss_outputs optimizer_model.zero_grad() # print('device:', loss_outputs.device, 'type:', loss_outputs.dtype, 'value:', loss_outputs) loss_outputs.backward() optimizer_model.step()
def train_epoch(self): self.models.train() self.epoch += 1 # record training statistics avg_meters = { 'loss_rec': AverageMeter('Loss Rec', ':.4e'), 'loss_adv': AverageMeter('Loss Adv', ':.4e'), 'loss_disc': AverageMeter('Loss Disc', ':.4e'), 'time': AverageMeter('Time', ':6.3f') } progress_meter = ProgressMeter(len(self.train_loaders[0]), avg_meters.values(), prefix="Epoch: [{}]".format(self.epoch)) # begin training from minibatches for ix, (data_0, data_1) in enumerate(zip(*self.train_loaders)): start_time = time.time() # load text and labels src_0, src_len_0, labels_0 = data_0 src_0, labels_0 = src_0.to(args.device), labels_0.to(args.device) src_1, src_len_1, labels_1 = data_1 src_1, labels_1 = src_1.to(args.device), labels_1.to(args.device) # encode encoder = self.models['encoder'] z_0 = encoder(labels_0, src_0, src_len_0) # (batch_size, dim_z) z_1 = encoder(labels_1, src_1, src_len_1) # recon & transfer generator = self.models['generator'] inputs_0 = (z_0, labels_0, src_0) h_ori_seq_0, pred_ori_0 = generator(*inputs_0, src_len_0, False) h_trans_seq_0_to_1, _ = generator(*inputs_0, src_len_1, True) inputs_1 = (z_1, labels_1, src_1) h_ori_seq_1, pred_ori_1 = generator(*inputs_1, src_len_1, False) h_trans_seq_1_to_0, _ = generator(*inputs_1, src_len_0, True) # discriminate real and transfer disc_0, disc_1 = self.models['disc_0'], self.models['disc_1'] d_0_real = disc_0(h_ori_seq_0.detach()) # detached d_0_fake = disc_0(h_trans_seq_1_to_0.detach()) d_1_real = disc_1(h_ori_seq_1.detach()) d_1_fake = disc_1(h_trans_seq_0_to_1.detach()) # discriminator loss loss_disc = (loss_fn(args.gan_type)(d_0_real, self.ones) + loss_fn(args.gan_type)(d_0_fake, self.zeros) + loss_fn(args.gan_type)(d_1_real, self.ones) + loss_fn(args.gan_type)(d_1_fake, self.zeros)) # gradient penalty if args.gan_type == 'wgan-gp': loss_disc += args.gp_weight * gradient_penalty( h_ori_seq_0, # real data for 0 h_trans_seq_1_to_0, # fake data for 0 disc_0) loss_disc += args.gp_weight * gradient_penalty( h_ori_seq_1, # real data for 1 h_trans_seq_0_to_1, # fake data for 1 disc_1) avg_meters['loss_disc'].update(loss_disc.item(), src_0.size(0)) self.disc_optimizer.zero_grad() loss_disc.backward() self.disc_optimizer.step() # reconstruction loss loss_rec = ( F.cross_entropy( # Recon 0 -> 0 pred_ori_0.view(-1, pred_ori_0.size(-1)), src_0[1:].view(-1), ignore_index=bert_tokenizer.pad_token_id, reduction='sum') + F.cross_entropy( # Recon 1 -> 1 pred_ori_1.view(-1, pred_ori_1.size(-1)), src_1[1:].view(-1), ignore_index=bert_tokenizer.pad_token_id, reduction='sum')) / ( 2.0 * args.batch_size ) # match scale with the orginal paper avg_meters['loss_rec'].update(loss_rec.item(), src_0.size(0)) # generator loss d_0_fake = disc_0(h_trans_seq_1_to_0) # not detached d_1_fake = disc_1(h_trans_seq_0_to_1) loss_adv = (loss_fn(args.gan_type, disc=False) (d_0_fake, self.ones) + loss_fn(args.gan_type, disc=False)(d_1_fake, self.ones) ) / 2.0 # match scale with the original paper avg_meters['loss_adv'].update(loss_adv.item(), src_0.size(0)) # XXX: threshold for training stability if (not args.two_stage): if (args.threshold is not None and loss_disc < args.threshold): loss = loss_rec + args.rho * loss_adv else: loss = loss_rec else: # two_stage training if (args.second_stage_num > args.epochs - self.epoch): # last second_stage; flow loss_adv gradients loss = loss_rec + args.rho * loss_adv else: loss = loss_rec self.optimizer.zero_grad() loss.backward() self.optimizer.step() avg_meters['time'].update(time.time() - start_time) # log progress if (ix + 1) % args.log_interval == 0: progress_meter.display(ix + 1) progress_meter.display(len(self.train_loaders[0]))
n_batches = len(validation_loader) Ptp01, Ptp05, Ptp1, AUC = np.zeros(n_batches // n_batch_verif), np.zeros( n_batches // n_batch_verif), np.zeros( n_batches // n_batch_verif), np.zeros(n_batches // n_batch_verif) vs, vf, tg = [], [], [] idx = -1 with torch.no_grad(): for batch_idx, (data, target) in enumerate(validation_loader): data_set = data[np.arange(0, batch_size, n_samples)].to(device) data_query = data[np.arange(1, batch_size, n_samples)].to(device) v_set, code_set = model(data_set, m=m_set) # single vector per set v_f, code_f = model(data_query, m=1) # single vector per query Sim = torch.mm(F.normalize(code_set, p=2, dim=1), F.normalize(code_f, p=2, dim=1).t()) # output = logisticReg(Sim.unsqueeze(-1)).squeeze() _, accuracy = loss_fn(Sim, len(code_f), m_set) tot_acc += accuracy vs.append(code_set) vf.append(code_f) tg.append(target) if (batch_idx + 1) % n_batch_verif == 0: idx += 1 vs = torch.stack(vs).flatten(start_dim=0, end_dim=1) vf = torch.stack(vf).flatten(start_dim=0, end_dim=1) tg = torch.stack(tg).flatten(start_dim=0, end_dim=1) Ptp01[idx], Ptp05[idx], Ptp1[idx], AUC[ idx] = acc_authentication(model, H0_id_v, H0_data_v, tg, vf.size(0), vs, vf, m_set, n_batch_verif) vs, vf, tg = [], [], []
def __init__(self, n_channel=3, num_class=conf.num_class, image_size=conf.input_size): self.n_channel = n_channel self.num_class = num_class self.image_size = image_size self.images = tf.placeholder(dtype=tf.float32, shape=[None, self.image_size[0], self.image_size[1], self.n_channel],name='images') num_bbox = 4 if conf.use_secondbig_loss_constrain: num_bbox = 8 self.loc_trues = tf.placeholder(dtype=tf.float32, shape=[None, None, num_bbox], name='loc_target') ############## self.cls_trues = tf.placeholder(dtype=tf.float32, shape=[None, None], name='cls_target') ############## self.is_training = tf.placeholder(dtype=tf.bool, name='traing_mode') self.global_step = tf.Variable(0, dtype=tf.int64, name='global_step') #self.logits = self.model(self.images, is_training=self.is_training) model = RetinaNet(conf.net) #self.loc_preds, self.cls_preds = model(self.images, training=self.is_training) #retinanet1 self.loc_preds, self.cls_preds = model(self.images, is_training=self.is_training) #retinanet2 self.anchor_boxes = BoxEncoder().get_anchor_boxes(self.image_size) self.d_bboxes, self.d_cls_pred, self.d_score = BoxEncoder().decode(self.loc_preds[0], self.cls_preds[0], self.image_size, tf_box_order=conf.tf_box_order) self.loc_loss, self.cls_loss, self.iou_loss = loss_fn(self.loc_preds, self.loc_trues, self.cls_preds, self.cls_trues, self.anchor_boxes, num_classes=self.num_class) #self.loc_loss, self.cls_loss = loss_fn(self.loc_preds, self.loc_trues, self.cls_preds, self.cls_trues, num_classes=self.num_class) self.regularization_loss = tf.losses.get_regularization_loss() tf.add_to_collection('losses', self.loc_loss) tf.add_to_collection('losses', self.cls_loss) tf.add_to_collection('losses', self.iou_loss) tf.add_to_collection('losses', self.regularization_loss) self.total_loss = tf.add_n(tf.get_collection('losses')) tf.summary.scalar('loc_loss', self.loc_loss) tf.summary.scalar('cls_loss', self.cls_loss) tf.summary.scalar('total_loss', self.total_loss) # lr = tf.cond(tf.less(self.global_step, 5000), # lambda: tf.constant(0.0001), # lambda: tf.cond(tf.less(self.global_step, 8000), # lambda: tf.constant(0.00005), # lambda: tf.cond(tf.less(self.global_step, 12000), # lambda: tf.constant(0.000025), # lambda: tf.constant(0.00001)))) #lr = tf.train.exponential_decay(0.0001, self.global_step, 1000, 0.96, staircase=True) #self.optimizer = tf.train.AdamOptimizer(learning_rate=lr)#.minimize(self.avg_loss, global_step=self.global_step) #self.optimizer=tf.train.GradientDescentOptimizer(learning_rate=lr)#.minimize(self.avg_loss, global_step=self.global_step) #self.optimizer = tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9, use_nesterov=True) self.lr = tf.Variable(float(1e-5), trainable=False, dtype=tf.float32) self.learning_rate_decay_op = self.lr.assign(self.lr * 0.1) self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr) tf.summary.scalar('learning_rate', self.lr) # update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # with tf.control_dependencies(update_ops): # #self.train_op = self.optimizer.minimize(self.total_loss, global_step=self.global_step) # # grads = self.optimizer.compute_gradients(self.total_loss) # for i, (g, v) in enumerate(grads): # if g is not None: # grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients # self.train_op = self.optimizer.apply_gradients(grads, global_step=self.global_step) vars1 = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)[180:] update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): if args.grad_clip_norm is not None: grads_and_vars = self.optimizer.compute_gradients(self.total_loss) grads = [x[0] for x in grads_and_vars] vars = [x[1] for x in grads_and_vars] grads, _ = tf.clip_by_global_norm(grads, args.grad_clip_norm) self.train_op = self.optimizer.apply_gradients(zip(grads, vars), global_step=self.global_step) else: self.train_op = self.optimizer.minimize(self.total_loss, global_step=self.global_step) #self.train_op = self.optimizer.minimize(self.total_loss, var_list=vars1, global_step=self.global_step) tf.add_to_collection('input_tensor',self.images) tf.add_to_collection('input_tensor',self.loc_trues) tf.add_to_collection('input_tensor',self.cls_trues) tf.add_to_collection('input_tensor',self.is_training) tf.add_to_collection('output_tensor', self.loc_preds) tf.add_to_collection('output_tensor', self.cls_preds) tf.add_to_collection('decode_tensor', self.d_bboxes) tf.add_to_collection('decode_tensor', self.d_cls_pred) tf.add_to_collection('decode_tensor', self.d_score)
shuffle=params["train_shuffle"]) validation_loader = DataLoader(validation_dataset, batch_size=params["train_batch_size"], shuffle=params["train_shuffle"]) test_loader = DataLoader(test_dataset, batch_size=params["test_batch_size"], shuffle=params["test_shuffle"]) if __TIME__: log_info_time("generate dataloader time \t: ", datetime.timedelta(seconds=time.time() - start_time)) ''' Setting Loss Function ''' if __TIME__: start_time = time.time() criterion = loss_fn(hyper_params["loss_fn"]) inner_criterion = loss_fn(meta_params["inner_loss"]) outer_criterion = loss_fn(meta_params["outer_loss"]) # if torch.cuda.is_available(): # TODO: implement parallel training # if options["parallel_criterion"] : # print(options["parallel_criterion_comment"]) # criterion = DataParallelCriterion(criterion,device_ids=[0, 1, 2]) if __TIME__: log_info_time("setting loss func time \t: ", datetime.timedelta(seconds=time.time() - start_time)) ''' Setting Optimizer ''' if __TIME__: