def train(model, train_data_class0, train_data_class1, test_data, config, num_fold): train_class0_dataloader = DataLoader(dataset(train_data_class0, config), batch_size=config.few_shot_num, shuffle=True) train_class1_dataloader = DataLoader(dataset(train_data_class1, config), batch_size=config.few_shot_num, shuffle=True) test_dataloader = DataLoader(dataset(test_data, config), batch_size=config.batch_size, shuffle=False) optimizer = optim.Adam([ {'params': model.parameters()}, {'params': compute_class_score.parameters(), 'lr': config.tau_lr} ], lr=config.lr, weight_decay = 1e-4) BCE_criterion = nn.BCELoss() cos_criterion = nn.CosineEmbeddingLoss(margin=config.cos_loss_margin) best_model_state_dict = model.state_dict() best_acc = -1 best_sen = -1 ret_dict = {'accs': -1, 'sens':-1, 'specs':-1, 'precisions':-1, 'preds':None, 'labels':None} support_data = {'labels': -1, 'data':-1, 'order':-1, 'domain':-1, 'CTs':-1, 'ct_mask':-1} query_data = {'labels': -1, 'data':-1, 'order':-1, 'domain':-1, 'CTs':-1, 'ct_mask':-1} best_val_epoch = 0 for epoch in range(config.finetune_epoch): data_zip = enumerate(zip(train_class0_dataloader, train_class1_dataloader)) for step, (data_class0, data_class1) in data_zip: for key in data_class0.keys(): support_data[key] = torch.cat((data_class0[key][:-1], data_class1[key][:-1]), 0) query_data[key] = torch.cat((data_class0[key][-1].unsqueeze(0), data_class1[key][-1].unsqueeze(0)), 0) if config.CTs_augmentation==True: support_data = CTs_augmentation_batch(config, support_data) query_data = CTs_augmentation_batch(config, query_data) support_class = support_data['labels'] support_data['ct_mask'] = support_data['ct_mask'].cuda() support_data['CTs'] = support_data['CTs'].cuda() query_class = query_data['labels'] query_data['ct_mask'] = query_data['ct_mask'].cuda() query_data['CTs'] = query_data['CTs'].cuda() optimizer.zero_grad() support_class_pred, support_embed = model(support_data) query_class_pred, query_embed = model(query_data) class_avg_vectors = compute_class_vector(support_embed, support_class, config.num_classes) query_class_score, _ = compute_class_score(class_avg_vectors[0].unsqueeze(0).repeat(query_embed.shape[0],1), class_avg_vectors[1].unsqueeze(0).repeat(query_embed.shape[0],1), query_embed) y = (torch.ones(1)*(-1)).cuda() metric_loss = cos_criterion(class_avg_vectors[0].unsqueeze(0),class_avg_vectors[1].unsqueeze(0),y.unsqueeze(0)) class_loss = BCE_criterion(query_class_score[:,1], query_class.float()) loss = class_loss + config.cos_loss_gamma * metric_loss loss.backward() optimizer.step() best_model_state_dict, best_acc, best_sen, best_val_epoch, ret_dict = test(model, train_class0_dataloader, train_class1_dataloader, test_dataloader, config, epoch, best_acc, best_sen, best_val_epoch, best_model_state_dict, ret_dict, num_fold) print('Accuracy %.3f sensitivity %.3f @ epoch %d' % (best_acc,best_sen,best_val_epoch+1))
def train(model, train_data, val_data, config, num_fold): optimizer = optim.Adam(model.parameters(), lr=config.lr, weight_decay=1e-4) scheduler_StepLR = optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1) if config.enable_focal_loss == True: criterion = MultiFocalLoss(config.num_classes, alpha=config.focal_alpha, gamma=config.focal_gamma) else: criterion_CE = nn.CrossEntropyLoss() train_dataloader = DataLoader(dataset(train_data, config), batch_size=config.batch_size, shuffle=True) val_dataloader = DataLoader(dataset(val_data, config), batch_size=config.batch_size, shuffle=False) best_model_state_dict = model.state_dict() best_acc = -1 best_sen = -1 ret_dict = { 'accs': -1, 'sens': -1, 'specs': -1, 'precisions': -1, 'preds': None, 'labels': None } best_val_epoch = 0 iters = len(train_dataloader) for epoch in range(config.max_epoch): for data_src in tqdm(train_dataloader): if config.CTs_augmentation == True: data_src = CTs_augmentation_batch(config, data_src) class_src = data_src['labels'] data_src['ct_mask'] = data_src['ct_mask'].cuda() optimizer.zero_grad() src_class_output, features = model(data_src) loss = criterion_CE(src_class_output, class_src) loss.backward() optimizer.step() print('Val on source data ...') best_model_state_dict, best_acc, best_sen, best_val_epoch, ret_dict = val( model, val_dataloader, config, epoch, best_acc, best_sen, best_val_epoch, best_model_state_dict, ret_dict, 'val', num_fold) print('Accuracy %.3f sensitivity %.3f @ epoch %d' % (best_acc, best_sen, best_val_epoch + 1)) scheduler_StepLR.step() print('Current lr: ', optimizer.param_groups[0]['lr']) return model, ret_dict
def build_dataset(data_directory, img_width): X, y, tags = dataset(data_directory, int(img_width)) nb_classes = len(tags) sample_count = len(y) train_size = sample_count print("train size : {}, tags:{}, x_size:{}".format(train_size,nb_classes,X.shape)) feature = X label = np_utils.to_categorical(y, nb_classes) return feature, label, nb_classes
def train(model_type='cnn'): if model_type == 'rnn': padding = False else: padding = True model = Model(hidden_size=64) logits = model.forward(cell_type='rnn', model_type=model_type) loss = model.backword(logits) y_pred = tf.argmax(logits, axis=1, name='y_pred') y_true = tf.get_default_graph().get_tensor_by_name('y_true:0') accuracy = tf.reduce_mean(tf.cast(tf.equal(y_pred, y_true), dtype=tf.float32), name='accuracy') optimizer = tf.train.AdamOptimizer() train_step = optimizer.minimize(loss, name='train_step') init = tf.global_variables_initializer() data = dataset(padding=padding) x_train, y_train, x_val, y_val = data.x_train, data.y_train, data.x_val, data.y_val train_losses = [] train_acces = [] valid_losses = [] valid_acces = [] with tf.Session() as sess: sess.run(init) for epoch in range(1, epochs + 1): avg_train_loss, train_acc, valid_loss, valid_acc = train_per_epoch( model, sess, x_train, y_train, x_val, y_val, epoch, loss, model_type=model_type) train_losses.append(avg_train_loss) train_acces.append(train_acc) valid_losses.append(valid_loss) valid_acces.append(valid_acc) return train_losses, train_acces, valid_losses, valid_acces
def transferring_test(flow_model, data_pack, dataset, batch_size): train_data, train_labels, \ val_data, val_labels, \ train_simu_data, train_simu_labels, \ val_simu_data, val_simu_labels = data_pack wcs = train_data.shape[0] for i in range(wcs): train_set = dataset(train_data[i], train_labels[i]) val_set = dataset(val_data[i], val_labels[i]) train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True) val_loader = DataLoader(val_set, batch_size=val_data.shape[1], shuffle=False) logging.info("Training with Raw Data:") flow_model.train(train_loader, val_loader) for j in range(wcs): if i != j: test_set = dataset(val_data[j], val_labels[j]) test_loader = DataLoader(test_set, batch_size=val_data.shape[1], shuffle=False) print("from %d HP --> %d HP:" % (i, j)) flow_model.evaluation(test_loader) train_simu_set = dataset(train_simu_data[i], train_simu_labels[i]) val_simu_set = dataset(val_simu_data[i], val_simu_labels[i]) train_simu_loader = DataLoader(train_simu_set, batch_size=batch_size, shuffle=True) val_simu_loader = DataLoader(val_simu_set, batch_size=val_simu_data.shape[1], shuffle=False) logging.info("Training with Augmented Data:") flow_model.train(train_simu_loader, val_simu_loader) for j in range(wcs): if i != j: test_set = dataset(val_data[j], val_labels[j]) test_loader = DataLoader(test_set, batch_size=val_data.shape[1], shuffle=False) print("from %d HP --> %d HP:" % (i, j)) flow_model.evaluation(test_loader)
def test(args, device): def test_epoch(): model.eval() correct_avg = 0.0 test_loader_iter = iter(test_loader) for _ in range(len(test_loader_iter)): images, targets = next(test_loader_iter) images = images.to(device) targets = targets.to(device) model_output = model(images) correct_num = criteria.calculate_correct(model_output, targets) correct_avg += correct_num.item() print "Test Avg Acc: {:.4f}".format(correct_avg / float(test_set_size)) model = networks.CoPINet() model.load_state_dict(torch.load(args.model_path)) model = model.to(device) subfolders = [ os.path.join(args.dataset, folder) for folder in [ "center_single", "distribute_four", "distribute_nine", "in_center_single_out_center_single", "in_distribute_four_out_center_single", "left_center_single_right_center_single", "up_center_single_down_center_single" ] ] subfolders.append(os.path.join(args.dataset, "*")) for folder in subfolders: print "Evaluating on {}".format(folder) test_set = dataset(folder, "test", args.img_size, test=True) test_set_size = len(test_set) test_loader = DataLoader(test_set, batch_size=args.batch_size, num_workers=args.num_workers) with torch.no_grad(): test_epoch()
IMG_SIZE = hyper.IMG_SIZE if __name__ == '__main__': with tf.Session() as sess: input_tensor = tf.placeholder(tf.float32, shape=(None, IMG_SIZE[0], IMG_SIZE[1], 1)) input_label = tf.placeholder(tf.float32, shape=(None, 10)) shared_model = tf.make_template("shared_model", cnn_model) output_tensor, weights = shared_model(input_tensor) correct_prediction = tf.equal(tf.argmax(output_tensor, 1), tf.argmax(input_label, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) init = tf.global_variables_initializer() sess.run(init) # restore should behind the initialization, otherwise the model value will be rushed. ckpt = tf.train.get_checkpoint_state(CKPT_DIR) saver = tf.train.Saver(weights) saver.restore(sess, tf.train.latest_checkpoint(CKPT_DIR)) test_set = dataset(IMG_PATH, LABEL_PATH) print "use test set: ", IMG_PATH a = sess.run(accuracy, feed_dict={ input_tensor: test_set.imgs, input_label: test_set.one_hot_labels() }) print "The accuracy on test set is: %.4f" % a
for i, (idx_train, idx_val) in enumerate(kf_split): config.split_idx.append(idx_val) print('=== START training for fold %d ===' % i) train_data = { key: item[idx_train] for key, item in src_train_data.items() } val_data = {key: item[idx_val] for key, item in src_train_data.items()} train_data = data_normalize(config, train_data, method='train') val_data = data_normalize(config, val_data, method='test') # initialize training dataset and compute the mean and std of CT scans data_set = dataset(train_data, config) mean_std_dic = data_set.valid_CTs_mean_std() config.update(mean_std_dic) use_cuda = True FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor for data in [train_data, val_data]: data['labels'] = torch.from_numpy(data['labels']).type(LongTensor) data['data'] = torch.from_numpy(data['data']).type(FloatTensor) model = MedicNet(config) if use_cuda: model = model.cuda()
from __future__ import print_function import nltk import numpy as np from utils import dataset dataset = dataset() import hgtk # will use nltk.pos_tag and s_tag (StanfordPOSTagger) # tokenization will be done by nltk.word_tokenize def read_data(filename): with open(filename, 'r') as f: data = [line.split('\t') for line in f.read().splitlines()] return data train_source = [line[0] for line in dataset] train_target = ['\t' + hgtk.text.decompose(line[1]) + '\n' for line in dataset] input_characters = set() for line in train_source: for char in line: if char not in input_characters: input_characters.add(char) target_characters = set() for line in train_target: for char in line: if char not in target_characters:
os.mkdir(os.path.join('res', 'test_res_on_dataset2_' + check_dir)) # load clinical data config.data = config.data_root + 'data2' raw_data2 = load_data(config, 'train') accs_a = np.zeros(config.repeat) sens_a = np.zeros(config.repeat) specs_a = np.zeros(config.repeat) df_acc_all = [] test_data = data_normalize(config, raw_data2, method='test') use_cuda = True FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor for data in [test_data]: data['labels'] = torch.from_numpy(data['labels']).type(LongTensor) data['data'] = torch.from_numpy(data['data']).type(FloatTensor) test_dataloader = DataLoader(dataset(test_data, config), batch_size=6, shuffle=False) test(model1, test_dataloader, config, model_num=1) test(model2, test_dataloader, config, model_num=2) test(model3, test_dataloader, config, model_num=3) test(model4, test_dataloader, config, model_num=4) test(model5, test_dataloader, config, model_num=5)
def train(args, device): def train_epoch(epoch, steps): model.train() loss_avg = 0.0 acc_avg = 0.0 counter = 0 train_loader_iter = iter(train_loader) for _ in trange(len(train_loader_iter)): steps += 1 counter += 1 images, targets = next(train_loader_iter) images = images.to(device) targets = targets.to(device) model_output = model(images) loss = criterion(model_output, targets) loss_avg += loss.item() acc = criteria.calculate_acc(model_output, targets) acc_avg += acc.item() optimizer.zero_grad() loss.backward() optimizer.step() writer.add_scalar("Train Loss", loss.item(), steps) writer.add_scalar("Train Acc", acc.item(), steps) print "Epoch {}, Total Iter: {}, Train Avg Loss: {:.6f}".format( epoch, counter, loss_avg / float(counter)) return steps def validate_epoch(epoch, steps): model.eval() loss_avg = 0.0 acc_avg = 0.0 counter = 0 val_loader_iter = iter(val_loader) for _ in trange(len(val_loader_iter)): counter += 1 images, targets = next(val_loader_iter) images = images.to(device) targets = targets.to(device) model_output = model(images) loss = criterion(model_output, targets) loss_avg += loss.item() acc = criteria.calculate_acc(model_output, targets) acc_avg += acc.item() writer.add_scalar("Valid Avg Loss", loss_avg / float(counter), steps) writer.add_scalar("Valid Avg Acc", acc_avg / float(counter), steps) print "Epoch {}, Valid Avg Loss: {:.6f}, Valid Avg Acc: {:.4f}".format( epoch, loss_avg / float(counter), acc_avg / float(counter)) def test_epoch(epoch, steps): model.eval() loss_avg = 0.0 acc_avg = 0.0 counter = 0 test_loader_iter = iter(test_loader) for _ in trange(len(test_loader_iter)): counter += 1 images, targets = next(test_loader_iter) images = images.to(device) targets = targets.to(device) model_output = model(images) loss = criterion(model_output, targets) loss_avg += loss.item() acc = criteria.calculate_acc(model_output, targets) acc_avg += acc.item() writer.add_scalar("Test Avg Loss", loss_avg / float(counter), steps) writer.add_scalar("Test Avg Acc", acc_avg / float(counter), steps) print "Epoch {}, Test Avg Loss: {:.6f}, Test Avg Acc: {:.4f}".format( epoch, loss_avg / float(counter), acc_avg / float(counter)) np.random.seed(args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) model = getattr(networks, args.model_name)() if args.cuda and args.multigpu and torch.cuda.device_count() > 1: print "Running the model on {} GPUs".format(torch.cuda.device_count()) model = torch.nn.DataParallel(model) model.to(device) optimizer = optim.Adam( [param for param in model.parameters() if param.requires_grad], args.lr, weight_decay=args.weight_decay) if args.norm: transform = transforms.Compose([transforms.ToTensor()]) else: transform = None train_set = dataset(os.path.join(args.dataset, "train_set"), args.img_size, transform=transform) train_loader = DataLoader(train_set, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers) val_set = dataset(os.path.join(args.dataset, "val_set"), args.img_size, transform=transform) val_loader = DataLoader(val_set, batch_size=args.batch_size, num_workers=args.num_workers) test_set = dataset(os.path.join(args.dataset, "test_set"), args.img_size, transform=transform) test_loader = DataLoader(test_set, batch_size=args.batch_size, num_workers=args.num_workers) criterion = getattr(criteria, "cross_entropy") writer = SummaryWriter(args.log_dir) total_steps = 0 for epoch in xrange(args.epochs): total_steps = train_epoch(epoch, total_steps) with torch.no_grad(): validate_epoch(epoch, total_steps) test_epoch(epoch, total_steps) # save checkpoint model.eval().cpu() ckpt_model_name = "epoch_{}_batch_{}_seed_{}_model_{}_norm_{}_lr_{}_l2_{}.pth".format( epoch, args.batch_size, args.seed, args.model_name, args.norm, args.lr, args.weight_decay) ckpt_file_path = os.path.join(args.checkpoint_dir, ckpt_model_name) torch.save(model.state_dict(), ckpt_file_path) model.to(device) # save final model model.eval().cpu() save_model_name = "Final_epoch_{}_batch_{}_seed_{}_model_{}_norm_{}_lr_{}_l2_{}.pth".format( epoch, args.batch_size, args.seed, args.model_name, args.norm, args.lr, args.weight_decay) save_file_path = os.path.join(args.save_dir, save_model_name) torch.save(model.state_dict(), save_file_path) print "Done. Model saved."
def main(): # set GPU card os.environ["CUDA_VISIBLE_DEVICES"] = "0" # load anime face data_dir = '../anime_face/data_64/images/' data_extra_dir = '../anime_face/extra_data/images/' ds = dataset() ds.load_data(data_dir, verbose=0) ds.load_data(data_extra_dir, verbose=0) ds.shuffle() # reset graph tf.reset_default_graph() # set session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # build model model = GAN(sess, gf_dim=128) # training z_plot = sample_z(36, 100) # initial fake image z = sample_z((bs), 100) i = 1 while True: if (i == 1) or (i <= 100 and i % 20 == 0) or (i <= 200 and i % 50 == 0) or ( i <= 1000 and i % 100 == 0) or (i % 200 == 0): g_samples = model.generate(z_plot) plot_samples(g_samples, save=True, filename=str(i), folder_path='out2/', h=6, w=6) # train discriminator more for _ in range(5): real_img = ds.next_batch(bs) z = sample_z(bs, 100) fake_img = model.generate(z) # train D D_loss = model.train_D(real_img, fake_img) G_loss = model.train_G(bs) if (i % 100) == 0: model.save(model_name='WGAN_v2') z_loss = sample_z(64, 100) g_loss = model.generate(sample_z(32, 100)) g, d = model.sess.run([model.G_loss, model.D_loss], feed_dict={ model.xs: ds.random_sample(32), model.gs: g_loss, model.zs: z_loss }) print(str(i) + ' iteration:') print('D_loss:', d) print('G_loss:', g, '\n') i = i + 1
from utils import dataset from torch.utils.data import DataLoader import torch import sys from model import Model from parameters import * label = sys.argv[1] # detect gpu use_cuda = torch.cuda.is_available() data = dataset() train_size,valid_size = int(0.8*len(data)) ,int(0.2*len(data)) train_data, valid_data = torch.utils.data.random_split(data, [train_size, valid_size]) train_loader = DataLoader(train_data, batch_size = BATCH_SIZE, shuffle = True) valid_loader = DataLoader(valid_data, batch_size = BATCH_SIZE, shuffle = False) train_loader.length = len(train_data) valid_loader.length = len(valid_data) model = Model(use_cuda, label) #model.load("chooser_1223_75.pt") model.train(train_loader,valid_loader,400)
def main(): dset = dataset(dconf) # dset.set_batch_size([25, 25]) input_dim = dconf['input_dim'] output_dim = dconf['output_dim'] lr = 1e-1 # a random labeled example xl = tf.placeholder(tf.float32, input_dim) # a random neighbor xn = tf.placeholder(tf.float32, input_dim) # a random unlabeled example xu = tf.placeholder(tf.float32, input_dim) yl = forward(dconf['netName_label'], xl) yn = forward(dconf['netName_neighbor'], xn) yu = forward(dconf['netName_unlabel'], xu) yt = tf.placeholder(tf.float32, output_dim) loss_ = supervised_loss('abs_quadratic')(yl, yt) if not dconf['supervise only']: # add loss based on manifold neighbor loss_ += graph_loss('LE')(yl, yn, 1.0) loss_ += graph_loss('LE')(yl, yu, 0.0) # add loss based on cluster loss_ += cluster_loss('S3VM')(yn) opt = tf.train.AdagradOptimizer(lr) gvs = opt.compute_gradients(loss_) clipped_gvs = [(tf.clip_by_value(grad, -1.0, 1.0), var) for grad, var in gvs] train_step = opt.apply_gradients(clipped_gvs) sess = tf.InteractiveSession() tf.global_variables_initializer().run() # Train # actually this is the number of labeled samples used in training instead of epoch # because we cannot perform batch computation here nb_epoch = 100000 for i in tqdm(range(nb_epoch)): xl_, xn_, xu_, yt_ = dset.pick() sess.run(train_step, feed_dict={xl: xl_, xn: xn_, xu: xu_, yt: yt_}) # Test trained model with sess.as_default(): if dconf['multi-label']: yl_ = sess.run(yl, feed_dict={xl: dset.test['x']}) yl_ = np.argmax(np.array(yl_), axis=1) yt_ = dset.test['y'] for metric in (int_accuracy, ): m_name, m_value = metric(yl_, yt_) print(m_name, '%.6f' % m_value) for i in range(len(yt_)): print((yl_[i], yt_[i]), end='') else: for keras_metric in (int_accuracy, ): print( keras_metric(yt, yl).eval(feed_dict={ xl: dset.test['x'], yt: dset.test['y'] })) yp = sess.run(yl, feed_dict={ xl: dset.test['x'], yt: dset.test['y'] }) for i in range(20): print(yp[i], dset.test['y'][i])
def train(config, epoch_from=0): threshold = 5 threshold = threshold / 127.5 penalty = 'over' print('process before training...') train_dataset = dataset(config['path']['dataset']['train'], patch_size=config['train']['patch size'], scale=config['scale']) train_data = DataLoader(dataset=train_dataset, batch_size=config['train']['batch size'], shuffle=True, num_workers=16) valid_dataset = dataset(config['path']['dataset']['valid'], patch_size=config['train']['patch size'], scale=config['scale'], is_train=False) valid_data = DataLoader(dataset=valid_dataset, batch_size=config['valid']['batch size'], num_workers=4) # training details - epochs & iterations iterations_per_epoch = len( train_dataset) // config['train']['batch size'] + 1 n_epoch = config['train']['iterations'] // iterations_per_epoch + 1 print('epochs scheduled: %d , iterations per epoch %d...' % (n_epoch, iterations_per_epoch)) # define main SR network as generator generator = Generator(scale_by=config['scale'], n_blocks=32, n_feats=256, res_scaling=0.1).cuda() save_path_G = config['path']['ckpt'] # optimizer learning_rate = config['train']['lr'] G_optimizer = optim.Adam(generator.parameters(), lr=learning_rate) lr_scheduler = optim.lr_scheduler.StepLR(G_optimizer, config['train']['decay']['every'], config['train']['decay']['by']) # if training from scratch, remove all validation images and logs if epoch_from == 0: if os.path.exists(config['path']['validation']): shutil.rmtree(config['path']['validation']) if os.path.exists(config['path']['logs']): shutil.rmtree(config['path']['logs']) # if training not from scratch, load weights else: if os.path.exists(save_path_G): ckpt = torch.load(save_path_G) generator.load_state_dict(ckpt['model']) print('reading generator checkpoints...') G_optimizer.load_state_dict(ckpt['opt']) lr_scheduler.last_epoch = epoch_from * iterations_per_epoch del ckpt else: raise FileNotFoundError('Pretrained weight not found.') os.makedirs(config['path']['validation'], exist_ok=True) os.makedirs(config['path']['logs'], exist_ok=True) writer = SummaryWriter(config['path']['logs']) # loss functions # loss = None squared = False if config['loss'] == 'L1': # loss = nn.L1Loss() squared = False elif config['loss'] == 'MSE': # loss = nn.MSELoss() squared = True loss_fn = Thresh_Error(threshold, squared, penalty) generator = nn.DataParallel(generator).cuda() # validation valid = validation(generator, valid_data, writer, config['path']['validation']) # training print('start training...') for epoch in range(epoch_from, n_epoch): generator = generator.train() epoch_loss = 0 for i, data in enumerate(tqdm(train_data)): lr, gt, _ = data lr = lr.cuda() gt = gt.cuda() # forwarding sr = generator(lr) g_loss = loss_fn(sr, gt) # g_loss = loss(sr, gt) # distance = torch.abs(sr - gt) # shift = (distance - threshold) # L1 # thresholded_loss = relu(shift) # penalty over t only # # thresholded_loss = relu(-shift) # penalty under t only # if config['loss'] == 'MSE': # thresholded_loss = thresholded_loss ** 2 # g_loss = thresholded_loss.mean() # penalty over t only # # g_loss = -thresholded_loss.mean() # penalty under t only # back propagation G_optimizer.zero_grad() g_loss.backward() G_optimizer.step() lr_scheduler.step() epoch_loss += g_loss.item() print('Training loss at {:d} : {:.8f}\n'.format(epoch, epoch_loss)) # validation if (epoch + 1) % config['valid']['every'] == 0: is_best = valid.run(epoch + 1) # save validation image valid.save(tag='latest') if is_best: ckpt = { 'model': generator.module.state_dict(), 'opt': G_optimizer.state_dict() } torch.save(ckpt, save_path_G) torch.cuda.empty_cache() # training process finished. # final validation and save checkpoints is_best = valid.run(n_epoch) valid.save(tag='final') writer.close() if is_best: ckpt = { 'model': generator.module.state_dict(), 'opt': G_optimizer.state_dict() } torch.save(ckpt, save_path_G) print('training finished.')
def test(args, device): def test_epoch(): model.eval() correct_avg = 0.0 test_loader_iter = iter(test_loader) for _ in range(len(test_loader_iter)): images, targets = next(test_loader_iter) images = images.to(device) targets = targets.to(device) model_output = model(images) correct_num = criteria.calculate_correct(model_output, targets) correct_avg += correct_num.item() acc = correct_avg / float(test_set_size) print "Test Avg Acc: {:.4f}".format(correct_avg / float(test_set_size)) return acc if args.norm: transform = transforms.Compose([transforms.ToTensor()]) else: transform = None model = getattr(networks, args.model_name)() model.load_state_dict(torch.load(args.model_path)) model = model.to(device) # name_combinations = [["combination", "holistic"], # ["combination", "analytical"], # ["permutation", "holistic"], # ["permutation", "analytical"], # ["segmentation", "holistic"], # ["segmentation", "analytical"]] # for name_combination in name_combinations: # print "Evaluating {}".format("_".join(name_combination)) # test_set = dataset(os.path.join(args.dataset, "test_set"), args.img_size, transform=transform, names=name_combination) # test_set_size = len(test_set) # print test_set_size # test_loader = DataLoader(test_set, batch_size=args.batch_size, num_workers=args.num_workers) # with torch.no_grad(): # test_epoch() two_int = [["combination", "circle_overlap"], ["segmentation", "circle_2"]] three_int = [["combination", "circle_tangent"]] four_int = [["permutation", "circle_line"], ["segmentation", "circle_4"]] six_int = [["permutation", "circle_triangle"], ["segmentation", "circle_6"]] eight_int = [["combination", "circle_include"], ["permutation", "circle_cross"], ["permutation", "circle_square"], ["permutation", "circle_circle"], ["segmentation", "circle_8"]] number_combinations = [two_int, three_int, four_int, six_int, eight_int] for number_combination in number_combinations: total_acc = 0.0 total_num = 0 for name_combination in number_combination: acc = 0.0 print "Evaluating {}".format("_".join(name_combination)) test_set = dataset(os.path.join(args.dataset, "test_set"), args.img_size, transform=transform, names=name_combination) test_set_size = len(test_set) print test_set_size test_loader = DataLoader(test_set, batch_size=args.batch_size, num_workers=args.num_workers) with torch.no_grad(): acc = test_epoch() total_acc += acc * test_set_size total_num += test_set_size print total_acc / total_num
checkpoint_path = "checkpoints/" train_path = "fake/train.txt" dev_path = "fake/dev.txt" batch_size = 64 phase = 1 EOS_Token = 4963 use_2d = True train_steps_each_iter = 200 dev_steps_each_iter = 4 local_graph = tf.Graph() tpu_graph = tf.Graph() # dataset with local_graph.as_default(): train_data = utils.dataset(train_path, batch_size * train_steps_each_iter) dev_data = utils.dataset(dev_path, batch_size * dev_steps_each_iter) train_iter = train_data.make_one_shot_iterator() dev_iter = dev_data.make_one_shot_iterator() train_saveable = tf.data.experimental.make_saveable_from_iterator( train_iter) dev_saveable = tf.data.experimental.make_saveable_from_iterator(dev_iter) saveable = [train_saveable, dev_saveable] # model with open("model/hparams.json") as f: config = json.load(f) with tpu_graph.as_default(): encoder = Encoder(config, "encoder") decoder = Decoder(config, "decoder")
from utils import dataset import matplotlib.pyplot as plt import numpy as np import json with open('./config/houston.json', 'r') as f: dconf = json.load(f) dset = dataset(dconf) a = dconf['shape'][2] d = dset.x.reshape([dconf['shape'][0] * dconf['shape'][1], a]) l = dset.y.flatten() print(set(l)) def draw_class(c): x = np.arange(a) fig, ax = plt.subplots() for i in range(len(l)): if int(l[i]) == c: line, = ax.plot(x, d[i]) plt.savefig('/home/yangminz/Semi-supervised_Embedding/image/%s%02d.png' % (dconf['name'], c)) def main(): draw_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] for i in draw_list: draw_class(i) if __name__ == '__main__':
# net, opt = amp.initialize(net, opt, opt_level=optim_level) # load pretrained model try: net.load_state_dict(torch.load('./saved_models/HEAR_latest.pth', map_location=torch.device('cpu')), strict=False) except Exception as e: print(e) # prepare dataset # dataset = AugmentedOcclusions('../hearnet_data', # ['../ego_hands_png'], # ['../shapenet_png'], same_prob=0.5) test_data = dataset(['./aligned']) dataloader = DataLoader(test_data, batch_size=batch_size, shuffle=True, num_workers=0, drop_last=True) # define loss MSE = torch.nn.MSELoss() L1 = torch.nn.L1Loss() for epoch in range(max_epoch): # torch.cuda.empty_cache() for iter, data in enumerate(dataloader): Xs, Xt, same_person = data
self.g_loss, feed_dict={ self.z: bx, self.z: bz } ) print('Iter time: %8d, d_loss: %.4f, g_loss: %.4f' % (t, d_loss, g_loss)) if t % 10 == 0 and t > 0: save_dir = './faces2' bz = np.random.normal(size=[batch_size, self.z_dim]) bx = self.sess.run(self.x_, feed_dict={self.z: bz}) bx = np.reshape(bx, (64,96,96,3)) bx = (bx * 255).astype(np.uint8) im = Image.fromarray(bx[0]) im.save(r'./res/'+str(t)+'.png') print(bx.shape) print(bx) datapaths = glob.glob('./faces/*.jpg') data = dataset(image_paths=datapaths, batch_size=64, shape=[96,96,3], preprocess_fn=preprocess_fn ) G = generator() D = discriminator() gan = wGAN(G, D, data) gan.train()
from encoder import Encoder import utils import json import time checkpoint_path = "checkpoints/" train_path = "fake/train.txt" dev_path = "fake/dev.txt" batch_size = 64 phase = 1 EOS_Token = 4963 use_2d = False # dataset train_data = utils.dataset(train_path, batch_size) dev_data = utils.dataset(dev_path, batch_size) train_iter = train_data.make_one_shot_iterator() dev_iter = dev_data.make_one_shot_iterator() train_saveable = tf.data.experimental.make_saveable_from_iterator(train_iter) dev_saveable = tf.data.experimental.make_saveable_from_iterator(dev_iter) saveables = [train_saveable, dev_saveable] # model with open("model/hparams.json") as f: config = json.load(f) encoder = Encoder(config, "encoder") decoder = Decoder(config, "decoder") encoded_length = 1 if encoder.mode == "attention":
def main(): # define dataset object train_set = dataset(train_path, batch_size) # define input train_input = tf.placeholder( tf.float32, shape=(image_size,image_size,batch_size) ) train_label = tf.placeholder( tf.float32, shape=(image_size,image_size,batch_size) ) # the model and loss shared_model = tf.make_template('shared_model', model) train_output, weights = shared_model(train_input) loss = tf.reduce_sum(tf.nn.l2_loss(tf.substract(train_output, train_label))) # normlization weight. for w in weights: loss += tf.nn.l2_loss(w)*1e-4 # record loss tf.summary.scalar("loss", loss) # training step and learning rate global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay( base_learning_rate, global_step*batch_size, train_set.instance_num*lr_step_size, lr_decay, staircase=True ) tf.summary.scalar("learning_rate", learning_rate) # Optimizer optimizer = tf.train.AdamOptimizer(learning_rate) opt = optimizer.minimize(loss, global_step=global_step) saver = tf.train.Saver(weights, max_to_keep=0) config = tf.ConfigProto() # training with tf.Session(config=config) as sess: #TensorBoard open log with "tensorboard --logdir=logs" if not os.path.exists('logs'): os.mkdir('logs') merged = tf.summary.merge_all() file_writer = tf.summary.FileWriter('logs',sess.graph) # var initializaion tf.initialize_all_variables().run() if model_path: print "restore model..." saver.restore(sess,model_path) print "successfully restore previous model." # train for epoch in xrange(0,max_epoch): for step in range(train_set.instance_num//batch_size): data, label = train_set.next_batch() feed_dict = {train_input : data, train_label : label} _,l,output,lr,g_step = sess.run([opt, loss, train_output, learning_rate, global_step],feed_dict=feed_dict) print "[epoch %2.4f] loss %.4f\t lr %.5f" % (epoch+(float(step)*batch_size/train_set.instance_num), np.sum(l)/batch_size, lr) del data, label saver.save(sess, "./checkpoints/VDSR_const_clip_0.01_epoch_%03d.ckpt" % epoch ,global_step=global_step)
0.1 * size_data), int(0.2 * size_data) ### Get random indices for test,train,val # np.random.seed(225) # Set random seed np.random.seed(737581) # Set random seed test_idx = np.random.choice(np.arange(size_data), test_size, replace=False) # get remaining indices without test indices rem_idx1 = np.array(list(set(np.arange(size_data)) - set(test_idx))) val_idx = np.random.choice(rem_idx1, val_size, replace=False) rem_idx2 = np.array(list(set(rem_idx1) - set(val_idx))) train_idx = np.random.choice(rem_idx2, train_size, replace=False) print("Shapes of indices", train_idx.shape, test_idx.shape, val_idx.shape) #### Storing arrays into train,validation, test objects and deleting the full data dictionary train_data = dataset('training', data_dict, train_idx) val_data = dataset('validation', data_dict, val_idx) test_data = dataset('test', data_dict, test_idx) del data_dict print("\nData shapes: Train {0}, Validation {1}, Test {2}\n".format( train_data.x.shape, val_data.x.shape, test_data.x.shape)) t2 = time.time() print("Time taken to read and process input files", t2 - t1) #### ML part #### for i in model_lst: model_name = str(i)
lr_start = 100 # start learning rate # Backtracking hyper-parameters BT = 1 # using backtracking or not lr_justified = True beta = 0.5 num_iter = 20 momentum = 0.9 save_paths = ['weights/','history','history/models/'] for save_path in save_paths: if not os.path.isdir(save_path): os.mkdir(save_path) # Data trainloader, testloader, num_batches = dataset(cifar_dataset, batch_size) criterion = nn.CrossEntropyLoss() all_history = {} nets = ['ResNet18', 'MobileNetV2','SENet18', 'PreActResNet18', 'DenseNet121'] # Loop for model architetures for net_name in nets: if net_name == 'ResNet18' : net = ResNet18(num_classes) elif net_name == 'MobileNetV2' : net = MobileNetV2(num_classes=num_classes) elif net_name == 'SENet18' : net = SENet18(num_classes=num_classes) elif net_name =='PreActResNet18' : net = PreActResNet18(num_classes=num_classes) elif net_name =='DenseNet121' : net = DenseNet121(num_classes=num_classes) print('Model:',net_name)
def train(): # Define input train_set = dataset(IMG_PATH, LABEL_PATH) train_input = tf.placeholder(tf.float32, shape=(BATCH_SIZE, IMG_SIZE[0], IMG_SIZE[1], 1), name='input_tensor') train_label = tf.placeholder(tf.float32, shape=(BATCH_SIZE, 10), name='input_label') # Define model shared_model = tf.make_template('shared_model', cnn_model) train_output, weights = shared_model(train_input) loss = -tf.reduce_sum(train_label * tf.log(train_output)) for w in weights: loss += tf.nn.l2_loss(w) * 1e-4 tf.summary.scalar("loss", loss) correct_prediction = tf.equal(tf.argmax(train_output, 1), tf.argmax(train_label, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name='acc') global_step = tf.Variable(0, trainable=False) learning_rate = tf.train.exponential_decay(BASE_LR_RATE, global_step * BATCH_SIZE, STEP, DECAY_RATE) tf.summary.scalar("learning_rate", learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate) opt = optimizer.minimize(loss, global_step=global_step) saver = tf.train.Saver(weights, max_to_keep=0) # save weights config = tf.ConfigProto( allow_soft_placement=True ) # use for session initialization, as configure for session # Training with tf.Session(config=config) as sess: # Record the training process using tensorboard if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) if not os.path.exists(CKPT_DIR): os.mkdir(CKPT_DIR) merged = tf.summary.merge_all( ) # used for tensorboard, to merge all data together file_writer = tf.summary.FileWriter(LOG_DIR, sess.graph) # initialize all variables init = tf.global_variables_initializer() sess.run(init) # Start training for epoch in xrange(0, MAX_EPOCH): batch_input, batch_label = train_set.next_batch(BATCH_SIZE) _, l, output, lr, g_step, summary, acc = sess.run([ opt, loss, train_output, learning_rate, global_step, merged, accuracy ], feed_dict={ train_input: batch_input, train_label: batch_label }) print "[step %05d] loss %.4f\t lr %.6f acc %.4f" % ( g_step, np.sum(l) / BATCH_SIZE, lr, acc) file_writer.add_summary(summary, epoch) if (epoch + 1) % SAVE_EVERY == 0: print "save: ", os.path.join(CKPT_DIR, "CNN_MNIST_%05d.ckpt" % epoch) saver.save(sess, os.path.join(CKPT_DIR, "CNN_MNIST_%05d.ckpt" % epoch), global_step=global_step)