def main(): args = get_args() if args.use_dropout == 0: args.use_dropout = False if args.use_dropout ==0: args.use_dropout = False for x in vars(args).items(): print(x) #from utils import data_transforms #print(data_transforms) if args.lr_sch ==5 and torch.__version__ != '0.4.0' : print("for cosine annealing, change to torch==0.4.0 in setup.py") raise AssertionError() elif args.lr_sch !=5 and torch.__version__ == '0.4.0': print("warning : this is torch version {}! nsml report will not be recorded".format(torch.__version__)) model, optimizer, scheduler = model_all.get_model(args) if args.use_gpu: if torch.cuda.device_count() > 1: print("[gpu] Let's use", torch.cuda.device_count(), "GPUs!") # dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs model = torch.nn.DataParallel(model) elif torch.cuda.device_count() == 1: print("[gpu] Let's use", torch.cuda.device_count(), "GPUs!") else: print("[gpu] no available gpus") model = model.cuda() nsml.bind(infer=infer, model=model, optimizer=optimizer) if args.pause: nsml.paused(scope=locals()) nsml.save() if args.mode == 'train': dataloaders, dataset_sizes = utils.data_loader(args, train=True, batch_size=args.batch_size) model = train.train_test(model, optimizer, scheduler, dataloaders, dataset_sizes, args) utils.save_model(model, 'model_state') with open('args.pickle', 'wb') as farg: pickle.dump(args, farg) loader = utils.data_loader(args, train=False, batch_size=1) predict, acc = utils.get_forward_result(model, loader, args) predict = torch.cat(predict, 0) nsml.bind(save=lambda x: utils.save_csv(x, data_csv_fname=os.path.join(DATASET_PATH, 'train', 'test') + '/test_data', results=predict, test_loader=loader)) nsml.save('result')
def test(): print('Start to test...') ctx = [mx.gpu(int(i)) for i in args.gpus.split(',')] if args.gpus != '-1' else mx.cpu() _, test_iter = data_loader(args.batch_size) model = LeNetPlus() model.load_params(os.path.join(args.ckpt_dir, args.prefix + '-best.params'), ctx=ctx) start_time = timeit.default_timer() test_accuracy, features, predictions, labels = evaluate_accuracy( test_iter, model, ctx) elapsed_time = timeit.default_timer() - start_time print("Test_acc: %s, Elapsed_time: %f s" % (test_accuracy, elapsed_time)) if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) if args.plotting: plot_features(features, labels, num_classes=args.num_classes, fpath=os.path.join(args.out_dir, '%s.png' % args.prefix))
def __init__(self, config, model, experiments): self.config = config self.model = model self.experiments = experiments self.trainloader, _ = data_loader(config) # checkpoint self.checkpoint_dir = make_dirs( os.path.join(self.config.result_path, self.config.checkpoint_path)) self.ckpt = tf.train.Checkpoint(enc=self.model.enc, dec=self.model.dec, optim=self.model.optim, epoch=self.model.global_epoch) self.ckpt_manager = tf.train.CheckpointManager( self.ckpt, directory=self.checkpoint_dir, checkpoint_name='ckpt', max_to_keep=2) # tensorboard self.tensorboard_dir = make_dirs( os.path.join(self.config.result_path, self.config.tensorboard_path)) self.summary_writer = tf.summary.create_file_writer( self.tensorboard_dir)
def evalidate(flag, args): with tf.Graph().as_default(): queue_loader = data_loader(False, batch_size=args.bsize, num_epochs=args.ep, dataset_dir=args.dataset_dir) with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits, end_points = resnet_v2.resnet_v2_101(queue_loader.images, num_classes=764, is_training=flag) names_to_values, names_to_updates = slim.metrics.aggregate_metric_map({ "accuracy": slim.metrics.accuracy(np.argmax(logits, 1), queue_loader.labels) }) for name, value in names_to_values.items(): op_name = 'eval_{}'.format(name) op = tf.summary.scalar(op_name, value) tf.add_to_collection(tf.GraphKeys.SUMMARIES, op) slim.evaluation.evaluate_once( master='', checkpoint_path=args.modelpath, logdir=args.evallog, num_evals=queue_loader.num_batches, eval_op=list(names_to_updates.values()), variables_to_restore=slim.get_variables_to_restore())
def __init__(self, config, model): self.config = config self.model = model _, self.testloader = data_loader(config) # directory to save experimental results self.dir = make_dirs(os.path.join(self.config.result_path, self.config.experiment_path))
def __init__(self, dataset, data_path="data", task_type="full"): self.dataset = dataset self.data_path = data_path (self.adj, self.train_adj, self.features, self.train_features, self.labels, self.idx_train, self.idx_val, self.idx_test, self.degree, self.learning_type) = data_loader(dataset, data_path, "NoNorm", False, task_type) #convert some data to torch tensor ---- may be not the best practice here. self.features = torch.FloatTensor(self.features).float() self.train_features = torch.FloatTensor(self.train_features).float() # self.train_adj = self.train_adj.tocsr() self.labels_torch = torch.LongTensor(self.labels) self.idx_train_torch = torch.LongTensor(self.idx_train) self.idx_val_torch = torch.LongTensor(self.idx_val) self.idx_test_torch = torch.LongTensor(self.idx_test) # vertex_sampler cache # where return a tuple self.pos_train_idx = np.where(self.labels[self.idx_train] == 1)[0] self.neg_train_idx = np.where(self.labels[self.idx_train] == 0)[0] # self.pos_train_neighbor_idx = np.where self.nfeat = self.features.shape[1] self.nclass = int(self.labels.max().item() + 1) self.trainadj_cache = {} self.adj_cache = {} #print(type(self.train_adj)) self.degree_p = None
def main(): dataset_name = 'K562_USP_UUE' dataset_path = '/Users/xinzeng/Desktop/research/reproduce/' global datasets # input data from preproces datasets = data_loader(dataset_path + dataset_name+'.h5') # check whether the saved file exist! date = '5_4_2020' save_file_path = dataset_path + date mkdir(save_file_path) save_file_path = save_file_path + '/' + dataset_name mkdir(save_file_path) best_hyperparameters = input_param(dataset_path + 'K562_USP_UUE_best_model_info') # saved the hyperparameters of the model save_f = open(save_file_path + '/best_model_info', 'w') #run hyperopt to get the best hyperparameters print(best_hyperparameters) #run the best parameters on the model model = best_param_cnn(best_hyperparameters,save_f,dataset_name) model.save(save_file_path + '/best_model.h5') save_f.close()
def __init__(self, config, model, experiments): self.config = config self.model = model self.experiments = experiments self.trainloader, _ = data_loader(config) # checkpoint self.checkpoint_dir = make_dirs( os.path.join(self.config.result_path, self.config.checkpoint_path)) self.ckpt = tf.train.Checkpoint(gen=self.model.gen, g_optim=self.model.g_optim, dis=self.model.dis, d_optim=self.model.d_optim, epoch=self.model.global_epoch) self.ckpt_manager = tf.train.CheckpointManager( self.ckpt, directory=self.checkpoint_dir, checkpoint_name='ckpt', max_to_keep=1) # tensorboard self.tensorboard_dir = make_dirs( os.path.join(self.config.result_path, self.config.tensorboard_path)) self.summary_writer = tf.summary.create_file_writer( self.tensorboard_dir) tf.summary.trace_on(graph=True, profiler=True)
def test_offline(): import os os.environ['CUDA_VISIBLE_DEVICES'] = '0' model = Net() margin = 1.0 threshlod = 1.0 model_path = '/data/chenchao/reid_train/exp11.2/models/epoch_0004-000500_feat.weights' if not os.path.exists(model_path): print('huaijinhhhh') test_list = '../data/val.txt' image_root = '/home/chenchao/ReID/' test_loader = data_loader(image_root, test_list, shuffle=False, batch_size=512, is_visualization=True) acc = test_vis(model, test_loader, model_path, threshlod, margin, is_cuda=True, is_visualization=True) print('best_threshold : {:.03f}, best_accuracy:{:.03f}'.format( threshlod, acc))
def infer_step(sess, model, infer_file, jieba): iterator = data_loader(infer_file, FLAGS.batch_size, 1, False, "infer") sess.run(iterator.initializer) elems = iterator.get_next() probs = [] while True: try: try: batch_context, batch_utterance, _ = sess.run(elems) except ValueError: batch_context, batch_utterance = sess.run(elems) batch_context = list( map(lambda d: (' ').join(jieba.lcut(d.decode())), batch_context)) batch_utterance = list( map(lambda d: (' ').join(jieba.lcut(d.decode())), batch_utterance)) batch_context, batch_context_len = get_data(batch_context) batch_utterance, batch_utterance_len = get_data(batch_utterance) except tf.errors.OutOfRangeError: break feed_dict = { model.context_embedded: batch_context, model.utterance_embedded: batch_utterance, model.context_len: batch_context_len, model.utterance_len: batch_utterance_len } prob = sess.run([model.probs], feed_dict=feed_dict) prob = list(np.squeeze(prob)) probs += prob probs = np.mean(probs) return probs
def debug(model, minibatch=200, device_ids=[0]): _loader, column_path = data_loader(debug=True, refresh=True, minibatch=minibatch, shuffle=False, gpu=True, truncate=True) fields = field_size(column_path) embedding_dim = 10 model = model(fields, embedding_dim).cuda() # model = torch.nn.DataParallel(model, device_ids=device_ids) loss = torch.nn.NLLLoss().cuda() lr = 1e-3 solver = torch.optim.Adam(model.parameters(), lr=lr) for _iter in range(10): end = default_timer() _loader.create_iter() LOSS, correct = 0, 0 _sum = {} for tensor, labels in _loader._iter: tensor = tensor.long() inputs = Variable(tensor) labels = Variable(labels) outputs = model(inputs) _loss = loss(outputs, labels) LOSS = LOSS + _loss.data[0] _loss.backward() solver.step() pred = outputs.data.max(1)[1] correct += pred.eq(labels.data).sum() for x in range(pred.shape[0]): if pred[x] not in list(_sum.keys()): _sum[pred[x]] = 1 else: _sum[pred[x]] = _sum[pred[x]] + 1 break correct = 100. * correct / _loader._len LOSS = LOSS / _loader._len print("loss: {:6f}, accuracy: {:6f}, len: {}, step_time: {:4f}".format( LOSS, correct, _loader._len, default_timer() - end)) print("... dict: {}".format(_sum))
def main(preprocessed_dir_path, trained_dir_path, prefix, embedding_dim, hidden_dim, batch_size): print(20 * "=", f"Preparing for test", 20 * "=") if prefix != "": test_file_name = f"{prefix}_test" vocab_file_name = f"{prefix}_vocab" revocab_file_name = f"{prefix}_revocab" weight_file_name = f"{prefix}_weight" else: test_file_name = "test" vocab_file_name = "vocab" revocab_file_name = "revocab" weight_file_name = "weight" with open(os.path.join(preprocessed_dir_path, vocab_file_name + ".pkl"), "rb") as fp: vocab = pickle.load(fp) with open(os.path.join(preprocessed_dir_path, revocab_file_name + ".pkl"), "rb") as fp: revocab = pickle.load(fp) vocab_size = len(vocab) padding_idx = vocab[" "] encoder = Encoder(vocab_size, embedding_dim, hidden_dim, padding_idx).to(device) decoder = Decoder(vocab_size, embedding_dim, hidden_dim, padding_idx).to(device) encoder.load_state_dict( torch.load(os.path.join(trained_dir_path, f"{weight_file_name}.pth"))["encoder"]) decoder.load_state_dict( torch.load(os.path.join(trained_dir_path, f"{weight_file_name}.pth"))["decoder"]) expression_ids, answer_ids = data_loader( os.path.join(preprocessed_dir_path, test_file_name + ".pkl"), padding_idx) print(20 * "=", f"Testing", 20 * "=") predicted_answers = predict(encoder, decoder, vocab, revocab, expression_ids, batch_size, padding_idx) answers = [] for ids in answer_ids.detach().numpy().tolist(): answer = int("".join([revocab[str(id)] for id in ids])) answers.append(answer) print(20 * "=", f"Calculate Test Result", 20 * "=") score = 0 missed = [] for i, answer in enumerate(answers): if predicted_answers[i] == answer: score += 1 else: missed.append((answer, predicted_answers[i])) print(f"Accuracy: {score/len(answers) * 100} ({score}/{len(answers)})")
def __init__(self, dataset, args, data_path="data", task_type="full"): self.dataset = dataset self.data_path = data_path (self.adj, self.train_adj, self.features, self.train_features, self.labels, self.idx_train, self.idx_val, self.idx_test, self.degree, self.learning_type) = data_loader(dataset, data_path, "NoNorm", False, task_type) #convert some data to torch tensor ---- may be not the best practice here. self.train_adj = self.adj self.train_features = self.features self.learning_type = 'transductive' self.labels = self.labels.astype(np.int) if args.pca: # pca = TruncatedSVD(n_components=600, n_iter=5000, algorithm='arpack') pca = PCA(n_components=256) self.features = pca.fit_transform(self.features) self.train_features = pca.fit_transform(self.train_features) self.features = torch.FloatTensor(self.features).float() self.train_features = torch.FloatTensor(self.train_features).float() # self.train_adj = self.train_adj.tocsr() if args.train_size and not args.fastmode: from ssl_utils import get_splits_each_class self.idx_train, self.idx_val, self.idx_test = get_splits_each_class( labels=self.labels, train_size=args.train_size) # print(self.idx_train[:10]) # from ssl_utils import get_few_labeled_splits # self.idx_train, self.idx_val, self.idx_test = get_few_labeled_splits( # labels=self.labels, train_size=args.train_size) if args.fastmode: from deeprobust.graph.utils import get_train_test self.idx_train, self.idx_test = get_train_test( nnodes=self.adj.shape[0], test_size=1 - args.label_rate, stratify=self.labels) self.idx_test = self.idx_test[:1000] self.labels_torch = torch.LongTensor(self.labels) self.idx_train_torch = torch.LongTensor(self.idx_train) self.idx_val_torch = torch.LongTensor(self.idx_val) self.idx_test_torch = torch.LongTensor(self.idx_test) # vertex_sampler cache # where return a tuple self.pos_train_idx = np.where(self.labels[self.idx_train] == 1)[0] self.neg_train_idx = np.where(self.labels[self.idx_train] == 0)[0] # self.pos_train_neighbor_idx = np.where self.nfeat = self.features.shape[1] self.nclass = int(self.labels.max().item() + 1) self.trainadj_cache = {} self.adj_cache = {} #print(type(self.train_adj)) self.degree_p = None
def main(): """Main pipeline.""" # parse command line arguments pathSet = [ "./myProject/trainData/Mild/", "./myProject/trainData/Aggressive/" ] for dataPath in pathSet: # parse argument args = parse_args(dataPath) print("Constructing data loaders...") # load training set and split trainset, valset = load_data(args.dataroot, args.train_size) trainloader, validationloader = data_loader(args.dataroot, trainset, valset, args.batch_size, args.shuffle, args.num_workers) # define model print("Initialize model ...") if args.model_name == "default": model = NetworkNvidia() elif args.model_name == "modified": model = ModNet() # define optimizer and loss function optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) criterion = nn.MSELoss() # learning rate scheduler scheduler = MultiStepLR(optimizer, milestones=[30, 50], gamma=0.1) # resume if args.resume: print("Loading a model from checkpoint") # use pre-trained model checkpoint = torch.load("model.h5", map_location=lambda storage, loc: storage) args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) # cuda or cpu device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("Selected GPU: ", device) # training print("Training Neural Network...") trainer = Trainer(args.dataroot, args.ckptroot, model, device, args.epochs, criterion, optimizer, scheduler, args.start_epoch, trainloader, validationloader) trainer.train()
def main(): x_test, y_test = data_loader(clean_data_filename) x_test = data_preprocess(x_test) bd_model = keras.models.load_model(model_filename) clean_label_p = np.argmax(bd_model.predict(x_test), axis=1) class_accu = np.mean(np.equal(clean_label_p, y_test)) * 100 print('Classification accuracy:', class_accu)
def main(): """Main pipeline.""" # parse command line arguments args = parse_args() # load trainig set and split trainset, valset = load_data(args.dataroot, args.train_size) print("==> Preparing dataset ...") trainloader, validationloader = data_loader(args.dataroot, trainset, valset, args.batch_size, args.shuffle, args.num_workers) # define model print("==> Initialize model ...") if args.model_name == "nvidia": model = NetworkNvidia() elif args.model_name == "light": model = NetworkLight() # define optimizer and criterion optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=args.weight_decay) criterion = nn.MSELoss() # learning rate scheduler scheduler = MultiStepLR(optimizer, milestones=[30, 50], gamma=0.1) # resume if args.resume: print("==> Loading checkpoint ...") # use pre-trained model checkpoint = torch.load("model.h5", map_location=lambda storage, loc: storage) print("==> Loading checkpoint model successfully ...") args.start_epoch = checkpoint['epoch'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) scheduler.load_state_dict(checkpoint['scheduler']) # cuda or cpu device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print("==> Use accelerator: ", device) # training print("==> Start training ...") trainer = Trainer(args.ckptroot, model, device, args.epochs, criterion, optimizer, scheduler, args.start_epoch, trainloader, validationloader) trainer.train()
def train_cnn(cnn, cnn_cfg, rl_cfg, game_index_now): replays_paths = rl_cfg.replay_path batch_size = cnn_cfg.batch_size epoch = cnn_cfg.epoch random_samples, step_size = load_replays( replays_paths, batch_size, game_index_now, pos_sample_factor=cnn_cfg.pos_sample_factor, max_N=cnn_cfg.max_N, valid_game_index_range=cnn_cfg.valid_game_index_range) cnn_data_loader = data_loader(batch_size, random_samples, step_size) cnn.train_model(cnn_data_loader, epoch, step_size, save_chkpnt=True) print("Train CNN done!")
def train(model, training_data, dev_data, learning_rate, batch_size, max_epoch): X_train, Y_train = training_data['X'], training_data['Y'] X_dev, Y_dev = dev_data['X'], dev_data['Y'] for i in range(max_epoch): for X, Y in data_loader(X_train, Y_train, batch_size=batch_size, shuffle=True): training_loss, grad_Ws, grad_bs = model.compute_gradients(X, Y) model.update(grad_Ws, grad_bs, learning_rate) dev_acc = acc(model.predict(X_dev), Y_dev) print("Epoch {: >3d}/{}\tloss:{:.5f}\tdev_acc:{:.5f}".format( i + 1, max_epoch, training_loss, dev_acc)) return model
def visualize_label_scan_bottom_right_white_4(): print('loading dataset') X_test, Y_test = utils.data_loader('%s/%s' % (DATA_DIR, DATA_FILE)) # transform numpy arrays into data generator test_generator = build_data_loader(X_test, Y_test) print('loading model') model_file = '%s/%s' % (MODEL_DIR, MODEL_FILENAME) model = load_model(model_file) # initialize visualizer visualizer = Visualizer(model, intensity_range=INTENSITY_RANGE, regularization=REGULARIZATION, input_shape=INPUT_SHAPE, init_cost=INIT_COST, steps=STEPS, lr=LR, num_classes=NUM_CLASSES, mini_batch=MINI_BATCH, upsample_size=UPSAMPLE_SIZE, attack_succ_threshold=ATTACK_SUCC_THRESHOLD, patience=PATIENCE, cost_multiplier=COST_MULTIPLIER, img_color=IMG_COLOR, batch_size=BATCH_SIZE, verbose=2, save_last=SAVE_LAST, early_stop=EARLY_STOP, early_stop_threshold=EARLY_STOP_THRESHOLD, early_stop_patience=EARLY_STOP_PATIENCE) log_mapping = {} # y_label list to analyze y_target_list = list(range(NUM_CLASSES)) y_target_list.remove(Y_TARGET) y_target_list = [Y_TARGET] + y_target_list for y_target in y_target_list: print('processing label %d' % y_target) _, _, logs = visualize_trigger_w_mask(visualizer, test_generator, y_target=y_target, save_pattern_flag=True) log_mapping[y_target] = logs
def evaluation(): model_path = '/data/chenchao/reid_train/exp11.2/models/epoch_0004-000500_feat.weights' model = Net() model = model.cuda() model.load_full_weights(model_path) model.eval() test_list = '../data/val.txt' image_root = '/home/chenchao/ReID/' test_loader = data_loader(image_root, test_list, shuffle=True, batch_size=256) margin = 1.0 #accuracy = test(model, test_loader, 0, margin, threshlod=0.18, log_interval=10) thre, acc, mean_1, mean_2 = best_test(model, test_loader, model_path) print('accuracy = {:f}'.format(acc))
def ComputeClassificationAccuracy(images, recons, labels, args, debug=True): model_paths = { 'CIFAR': 'classifiers/model/cifar-10', 'CelebA': 'classifiers/model/celeba' } batch_size = 50 dset = utils.data_loader(images, recons, labels, batch_size) # normalization, accuracy sess = tf.Session() if args.dataset == 'CIFAR': model = CIFARModel(model_paths[args.dataset], tiny=False, mode='eval', sess=sess) # TODO: Write CelebA model class n_data = 0 n_correct_orig = 0 n_correct = 0 total = 0 for images, recons, labels in dset: total += 1 n_correct_orig += sess.run(model.num_correct, feed_dict={ model.x_input: images, model.y_input: labels }) n_correct += sess.run(model.num_correct, feed_dict={ model.x_input: recons, model.y_input: labels }) n_data += len(images) acc_orig = float(n_correct_orig) / n_data acc = float(n_correct) / n_data print('Original acc: {}'.format(acc_orig)) print('Accuracy: {}'.format(acc)) return acc
def test(): """ Test model accuracy on test dataset. 测试模型在测试集上的准确率。 """ print("Start to test...") ctx = mx.gpu() if args.use_gpu else mx.cpu() _, test_iter = data_loader(args.batch_size) model = LeNetPlus() model.load_parameters(os.path.join(args.ckpt_dir, args.prefix + "-best.params"), ctx=ctx, allow_missing=True) # center_net = CenterLoss(num_classes=args.num_classes, feature_size=args.feature_size, lmbd=args.lmbd, ctx=mx.cpu()) center_net.load_parameters(os.path.join( args.ckpt_dir, args.prefix + "-feature_matrix.params"), ctx=ctx) start_time = time.time() test_accuracy, features, predictions, labels = evaluate_accuracy( test_iter, model, center_net, args.eval_method, ctx) elapsed_time = time.time() - start_time print("Test_acc: %s, Elapsed_time: %f s" % (test_accuracy, elapsed_time)) # make directory if not os.path.exists(args.out_dir): os.makedirs(args.out_dir) # draw feature map if args.plotting: plot_features(features, labels, num_classes=args.num_classes, fpath=os.path.join(args.out_dir, "%s.png" % args.prefix))
def train_1pass(model, training_data, dev_data, learning_rate, batch_size, print_every=100, plot_every=10): X_train, Y_train = training_data['X'], training_data['Y'] X_dev, Y_dev = dev_data['X'], dev_data['Y'] num_samples = 0 print_loss_total = 0 plot_loss_total = 0 plot_losses = [] plot_num_samples = [] for idx, (X, Y) in enumerate( data_loader(X_train, Y_train, batch_size=batch_size, shuffle=True), 1): training_loss, grad_Ws, grad_bs = model.compute_gradients(X, Y) model.update(grad_Ws, grad_bs, learning_rate) num_samples += Y.shape[1] print_loss_total += training_loss plot_loss_total += training_loss if idx % print_every == 0: dev_acc = acc(model.predict(X_dev), Y_dev) print_loss_avg = print_loss_total / print_every print_loss_total = 0 print("#Samples {: >5d}\tloss:{:.5f}\tdev_acc:{:.5f}".format( num_samples, print_loss_avg, dev_acc)) if idx % plot_every == 0: plot_loss_avg = plot_loss_total / plot_every plot_loss_total = 0 plot_losses.append(plot_loss_avg) plot_num_samples.append(num_samples) return model, {"losses": plot_losses, "num_samples": plot_num_samples}
def main(args): latent_dim = ldim(args.dataset, args.is_realnvp) model = torch.load(args.generator_path) model.eval() print(model) real_image_num = 1 reconstruct_image_num = 1 _, test_loader = data_loader(args) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) for batch_idx, (data, target) in enumerate(test_loader): if data.size()[0] != args.batch_size: continue data, target = torch.tensor(data).to( args.device), torch.tensor(target).to(args.device) data = (data + 1.0) / 2.0 z = model(data, reverse=False) z = z.to(args.device) out, _ = model(z, reverse=True) out = out.to('cpu').data.numpy() out = (out + 1.0) / 2.0 out = np.clip(out, 0.0, 1.0) for sample in out: out_file = os.path.join(args.output_dir, f"{reconstruct_image_num}reconstruct.png") if sample.shape[0] == 1: sample = sample[0] scmisc.toimage(sample, cmin=0.0, cmax=1.0).save(out_file) reconstruct_image_num += 1 data = data.to('cpu').data.numpy() data = (data + 1.0) / 2.0 data = np.clip(data, 0.0, 1.0) for sample in data: out_file = os.path.join(args.output_dir, f"{real_image_num}real.png") if sample.shape[0] == 1: sample = sample[0] scmisc.toimage(sample, cmin=0.0, cmax=1.0).save(out_file) real_image_num += 1
def test_step(sess, model, valid_file, step=None, summary_op=None, summary_writer=None): iterator = data_loader(valid_file, FLAGS.batch_size, 1) sess.run(iterator.initializer) losses, iters = 0, 0 elems = iterator.get_next() while True: try: batch_context, batch_utterance, batch_labels = sess.run(elems) batch_context, batch_context_len = get_data(batch_context) batch_utterance, batch_utterance_len = get_data(batch_utterance) batch_labels = batch_labels.reshape(-1, 1) except tf.errors.OutOfRangeError: break feed_dict = { model.context_embedded: batch_context, model.utterance_embedded: batch_utterance, model.context_len: batch_context_len, model.utterance_len: batch_utterance_len, model.labels: batch_labels } if summary_op is not None: summaries, loss = sess.run([summary_op, model.loss], feed_dict=feed_dict) summary_writer.add_summary(summaries, step) else: loss = sess.run([model.loss], feed_dict=feed_dict)[0] losses += loss iters += 1 return losses / iters
def evaluation(model_path, test_list): model = Net() model = model.cuda() model.load_full_weights(model_path) model.eval() image_root = '/path/to/ReId_data/image/' if False: train_list = 'data/debug.txt' test_list = '../data/test_debug.txt' else: train_list = 'data/train_all_5set.txt' test_list = '../data/test_all_5set.txt' test_loader = data_loader(image_root, test_list, shuffle=True, batch_size=32) margin = 1 accuracy = test(model, test_loader, 0, margin, threshlod=10, log_interval=10)
def main(): torch.cuda.manual_seed(seed) cudnn.benchmark = CUDNN # model model = tiramisu.FcDnSubtle(in_channels=8, n_classes=N_CLASSES) model = model.cuda() print(' + Number of params: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) model.apply(utils.weights_init) optimizer = optim.RMSprop(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY, eps=1e-12) criterion = nn.NLLLoss2d().cuda() exp = experiment.Experiment(EXPNAME, EXPERIMENT) exp.init() START_EPOCH = exp.epoch END_EPOCH = START_EPOCH + N_EPOCHS # for epoch in range(1): for epoch in range(START_EPOCH, END_EPOCH): since = time.time() ### Collect data ### # delete existing folder and old data if os.path.exists(res_root_path): shutil.rmtree(res_root_path) utils.collect_data(ori_train_base_rp, res_train_base_rp) utils.collect_data(ori_val_base_rp, res_val_base_rp) # data loader train_loader, val_loader = utils.data_loader(res_root_path) ### Train ### trn_loss, trn_err = utils.train(model, train_loader, optimizer, criterion, epoch) print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format( epoch, trn_loss, trn_err)) time_elapsed = time.time() - since print('Train Time {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60)) ### val ### val_loss, val_err = utils.test(model, val_loader, criterion, epoch) print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err)) time_elapsed = time.time() - since print('Total Time {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60)) ### Save Metrics ### exp.save_history('train', trn_loss, trn_err) exp.save_history('val', val_loss, val_err) ### Checkpoint ### exp.save_weights(model, trn_loss, val_loss, trn_err, val_err) exp.save_optimizer(optimizer, val_loss) ## Early Stopping ## if (epoch - exp.best_val_loss_epoch) > MAX_PATIENCE: print(("Early stopping at epoch %d since no " + "better loss found since epoch %.3").format( epoch, exp.best_val_loss)) break utils.adjust_learning_rate(LEARNING_RATE, LR_DECAY, optimizer, epoch, DECAY_LR_EVERY_N_EPOCHS) exp.epoch += 1
ind = sio.loadmat(args.area+'/cva_ref.mat') cva_ind = ind['cva_ref'] cva_ind = np.reshape(cva_ind, newshape=[-1]) i1, i2 = utils.getTrainSamples(cva_ind, im1, im2, args.trn) loss_log, vpro, fcx, fcy, bval = dsfa( xtrain=i1, ytrain=i2, xtest=im1, ytest=im2, net_shape=net_shape, args=args) imm, magnitude, differ_map = utils.linear_sfa(fcx, fcy, vpro, shape=img_shape) magnitude = np.reshape(magnitude, img_shape[0:-1]) differ = differ_map change_map = np.reshape(utils.kmeans(np.reshape(magnitude, [-1])), img_shape[0:-1]) # magnitude acc_un, acc_chg, acc_all2, acc_tp = utils.metric(1-change_map, chg_map) acc_un, acc_chg, acc_all3, acc_tp = utils.metric(change_map, chg_map) plt.imsave('results.png',change_map, cmap='gray') #plt.show() return None if __name__ == '__main__': args = parser() img1, img2, chg_map = utils.data_loader(area=args.area) os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu main(img1, img2, chg_map, args=args)
from numpy.random import seed import tensorflow as tf from tensorflow import set_random_seed from args import Parser from utils import data_loader from Seq2Cluster_test import Seq2Cluster seed(1) set_random_seed(2) opts = Parser().parse() os.environ["CUDA_VISIBLE_DEVICES"] = opts['gpu'] config = tf.ConfigProto() config.gpu_options.allow_growth = True session = tf.Session(config=config) dataloader = data_loader(opts['datapath'], opts['dataset'], opts['split']) tf.reset_default_graph() model = Seq2Cluster(opts) if opts['test'] is False: model.train(dataloader['train_segdata'], dataloader['test_labels'], dataloader['test_segdata'], dataloader['train_segnum_list'], dataloader['test_segnum_list']) else: model.test(dataloader['test_segdata'], dataloader['test_labels'], dataloader['test_segnum_list'])
def main(_): VERBOSE_ = 1 parser = OptionParser() parser.add_option("--verbose", action="store_const", const=1, dest="verbose", help="verbose mode") parser.add_option("-t", "--train", dest="train_path", help="train file path", metavar="train_path") parser.add_option("-v", "--validation", dest="validation_path", help="validation file path", metavar="validation_path") parser.add_option("-m", "--model", dest="model_dir", help="dir path to save model", metavar="model_dir") parser.add_option("-i", "--iters", dest="training_iters", help="training iterations", metavar="training_iters") (options, args) = parser.parse_args() if options.verbose == 0: VERBOSE_ = 0 train_path = options.train_path if train_path == None: parser.print_help() exit(1) validation_path = options.validation_path model_dir = options.model_dir if model_dir == None: parser.print_help() exit(1) if not os.path.isdir(model_dir): os.makedirs(model_dir) training_iters = options.training_iters if not training_iters: training_iters = 30 training_iters = int(training_iters) config = config_() config.max_max_epoch = training_iters eval_config = config_() path = 'data/char2vec_50.model' sys.stderr.write("embedding:%s\n" % (path)) cmodel = CharEmbedding(path) global index2word index2word = cmodel.index_2_word() train_data, valid_data, vocab_s, max_len = data_loader(train_path, cmodel) vocab_s = cmodel.vocab_size() eval_config.batch_size = 1 config.vocab_size = vocab_s eval_config.vocab_size = vocab_s config.num_steps = max_len eval_config.num_steps = max_len # print train_data NUM_THREADS = 5 cf = tf.ConfigProto(intra_op_parallelism_threads=NUM_THREADS, inter_op_parallelism_threads=NUM_THREADS, log_device_placement=False) with tf.Graph().as_default(), tf.Session() as session: stddd = 0.1 initializer = tf.truncated_normal_initializer(config.init_scale, stddev=stddd) sys.stderr.write("stddev:%f\n" % (stddd)) with tf.variable_scope("model", reuse=None, initializer=initializer): m = CWSModel(is_training=True, config=config) sys.stderr.write( "Vocab Size: %d\nTraining Samples: %d\nValid Samples %d\n Layers:%d\n" % (vocab_s, len(train_data[0]), len( valid_data[0]), config.num_layers)) sys.stderr.write( "Hidden Size: %d\nEmbedding size: %d\nWindow Size %d\nNorm %d\n" % (config.hidden_size, config.embedding_size, config.num_input, config.max_grad_norm)) tf.initialize_all_variables().run() m.load_embedding(session, cmodel.embedding_matrix()) saver = tf.train.Saver() # save all variables for i in range(config.max_max_epoch): m.is_training = True m.config.batch_size = 20 train_accuracy = run_epoch(session, m, train_data, m.train_op, cmodel, verbose=True) sys.stderr.write("Epoch: %d Train accuracy: %.3f\n" % (i + 1, 1 - train_accuracy)) m.is_training = False m.config.batch_size = 20 valid_accuracy = run_epoch(session, m, valid_data, tf.no_op(), cmodel, verbose=False) sys.stderr.write("Epoch: %d Valid accuracy: %.3f\n" % (i + 1, 1 - valid_accuracy)) sys.stderr.write('save model(%d)\n' % (i)) saver.save(session, model_dir + '/' + 'model_' + str(i) + '.ckpt')