def evaluate(model_name, log_dir): noise_levels = np.arange(0.0,0.55,0.05) print("noise levels {}".format(noise_levels)) accuracies = np.zeros(noise_levels.shape) test_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir, 'summaries', 'test')) #restore model if model_name=="sparseCNN": model = SparseNet() elif model_name=="denseCNN": model = DenseNet() else: raise ValueError("Model name unrecognized {}".format(model_name)) ckpt = tf.train.Checkpoint(net=model) ckpt_path = os.path.join(log_dir, 'checkpoints') manager = tf.train.CheckpointManager(ckpt,ckpt_path,max_to_keep=3) ckpt.restore(manager.latest_checkpoint) test_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy() for i,noise in enumerate(noise_levels): test_dataset, _ = datasets.get_dataset("mnist", 100, subset="test", shuffle=False, noise_level=noise) with test_summary_writer.as_default(): for x_batch, y_batch in test_dataset: if len(x_batch.shape)==3: x_batch = tf.expand_dims(x_batch, 3) test_logits = model(x_batch, training=False) # Update test metrics test_acc_metric(y_batch, test_logits) test_acc = test_acc_metric.result() tf.summary.scalar("noise_accuracy", test_acc, step=i) tf.summary.image("noisy_image", x_batch, step=i) accuracies[i] = float(test_acc) test_acc_metric.reset_states() print('Model {} noise {} Test acc: {}'.format(model_name, noise, float(test_acc))) print("Average noise score accros noise level {}".format(np.mean(accuracies)))
def main(datasetname, n_classes, batch_size, model_name, epochs, lr, keep_prob, base_log_dir): #Fix TF random seed tf.random.set_seed(1777) log_dir = os.path.join(os.path.expanduser(base_log_dir), "{}".format(datasetname)) os.makedirs(log_dir, exist_ok=True) # dataset train_dataset, train_samples = datasets.get_dataset( datasetname, batch_size) test_dataset, _ = datasets.get_dataset(datasetname, batch_size, subset="test", shuffle=False) #Network if model_name == "sparseCNN": model = SparseNet() elif model_name == "denseCNN": model = DenseNet() else: raise ValueError("Model name unrecognized {}".format(model_name)) #Train optimizer, loss nrof_steps_per_epoch = (train_samples // batch_size) lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay( lr, nrof_steps_per_epoch, 0.8) optimizer = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.9) loss_fn = tf.keras.losses.SparseCategoricalCrossentropy() #metrics train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy() test_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy() #Train step @tf.function def train_step(x, labels): with tf.GradientTape() as t: logits = model(x, training=True) loss = loss_fn(labels, logits) gradients = t.gradient(loss, model.trainable_variables) optimizer.apply_gradients(zip(gradients, model.trainable_variables)) return loss, logits #Run ep_cnt = tf.Variable(initial_value=0, trainable=False, dtype=tf.int64) #Summary writers train_summary_writer = tf.summary.create_file_writer( os.path.join(log_dir, 'summaries', 'train')) test_summary_writer = tf.summary.create_file_writer( os.path.join(log_dir, 'summaries', 'test')) ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) ckpt_path = os.path.join(log_dir, 'checkpoints') manager = tf.train.CheckpointManager(ckpt, ckpt_path, max_to_keep=3) ckpt.restore(manager.latest_checkpoint) if manager.latest_checkpoint: print("Restored from {}".format(manager.latest_checkpoint)) else: print("Initializing from scratch.") for ep in tqdm.trange(epochs, desc='Epoch Loop'): if ep < ep_cnt: continue # update epoch counter ep_cnt.assign_add(1) with train_summary_writer.as_default(): # train for an epoch for step, (x, y) in enumerate(train_dataset): if len(x.shape) == 3: x = tf.expand_dims(x, 3) tf.summary.image("input_image", x, step=optimizer.iterations) loss, logits = train_step(x, y) train_acc_metric(y, logits) ckpt.step.assign_add(1) tf.summary.scalar("loss", loss, step=optimizer.iterations) if int(ckpt.step) % 1000 == 0: save_path = manager.save() print("Saved checkpoint for step {}: {}".format( int(ckpt.step), save_path)) # Log every 200 batch if step % 200 == 0: train_acc = train_acc_metric.result() print("Training loss {:1.2f}, accuracu {} at step {}".format(\ loss.numpy(), float(train_acc), step)) # Display metrics at the end of each epoch. train_acc = train_acc_metric.result() tf.summary.scalar("accuracy", train_acc, step=ep) print('Training acc over epoch: %s' % (float(train_acc), )) # Reset training metrics at the end of each epoch train_acc_metric.reset_states() ############################## Test the model ############################# with test_summary_writer.as_default(): for x_batch, y_batch in test_dataset: if len(x_batch.shape) == 3: x_batch = tf.expand_dims(x_batch, 3) test_logits = model(x_batch, training=False) # Update test metrics test_acc_metric(y_batch, test_logits) test_acc = test_acc_metric.result() tf.summary.scalar("accuracy", test_acc, step=ep) tf.summary.image("test_image", x_batch, step=ep) test_acc_metric.reset_states() print('[Epoch {}] Test acc: {}'.format(ep, float(test_acc)))
features = cuda_var_wrapper(features[:, :, :max_length], volatile=True) labels = cuda_var_wrapper(labels[:, :max_length], volatile=True) output = cnn(features) correct_batch, total_batch = get_batch_accuracy( labels, output, lengths) correct += correct_batch total += total_batch print("{} out of {} label predictions are correct".format(correct, total)) return correct / total writer = SummaryWriter(log_dir=writer_path) if args.model == 'mufold': cnn = MUFold_ss(dropout=args.dropout) else: cnn = DenseNet(drop_rate=args.dropout) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(cnn.parameters(), lr=args.lr) # optimizer = torch.optim.SGD(cnn.parameters(), lr=args.lr, momentum=0.9, nesterov=True) scheduler = ReduceLROnPlateau(optimizer, mode='max', patience=args.patience, factor=0.1, min_lr=args.min_lr, verbose=True) if use_cuda: cnn.cuda() criterion.cuda() load_trained_model = os.path.join(
if not args.train and not args.test: print("You should train or test your network. Please check params.") exit() # some default params dataset/architecture related train_params = get_train_params() print("Params:") for k, v in model_params.items(): print("\t%s: %s" % (k, v)) print("Train params:") for k, v in train_params.items(): print("\t%s: %s" % (k, v)) print("Initialize the model..") model = DenseNet(**model_params) if args.train: # model.load_model() w1, w2 = model.train_all_epochs(train_params) if args.test: w1 = 0.5 w2 = 0.5 if not args.train: model.load_model() print("Testing...") _, _, _,_,_, _, _, _,_,_, _, acc1, acc2, accuracy = model.test(data_provider.test, w1, w2, batch_size=8) print("mean accuracy1: %f" % (acc1)) print("mean accuracy2: %f" % (acc2)) print("mean accuracy: %f" % (accuracy))
args.bc_mode = True model_params = vars(args) if not args.train and not args.test: print("You should train or test your network. Please check params.") exit() # some default params dataset/architecture related train_params = get_train_params_by_name(args.dataset) print("Params:") for k, v in model_params.items(): print("\t%s: %s" % (k, v)) print("Train params:") for k, v in train_params.items(): print("\t%s: %s" % (k, v)) print("Prepare training data...") data_provider = get_data_provider_by_name(args.dataset, train_params) print("Initialize the model..") model = DenseNet(data_provider=data_provider, **model_params) if args.train: print("Data provider train images: ", data_provider.train.num_examples) model.train_all_epochs(train_params) if args.test: if not args.train: model.load_model() print("Data provider test images: ", data_provider.test.num_examples) print("Testing...") loss, accuracy = model.test(data_provider.test, batch_size=200) print("mean cross_entropy: %f, mean accuracy: %f" % (loss, accuracy))
class Tool(): @staticmethod def edit_distance(s1, s2): size = len(s1) match = 0 for i in range(0, size - 1): if s1[i] == s2[i]: match += 1 return match / size cnn = DenseNet(channel_size, batch_size, drop_rate=0, num_init_features=512) criterion = nn.L1Loss() optimizer = torch.optim.Adam(cnn.parameters(), lr=0.001) if use_cuda: cnn.cuda() criterion.cuda() train_dataset = Protein_Dataset(0, 5800) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=False, num_workers=16) # Train the Model
train = [(k, v, w) for k, v, w in zip(x_train, y_train, w_train)] val = [(k, v, w) for k, v, w in zip(x_val, y_val, w_val)] for iteration in range(0, 10): train_dl = torch.utils.data.DataLoader(train, batch_size=BATCH_SIZE, shuffle=True) val_dl = torch.utils.data.DataLoader(val, batch_size=BATCH_SIZE, shuffle=True) # Model setup # device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") device = torch.device("cpu") model = DenseNet().to(device) criterion = torch.nn.MSELoss(reduction='mean') optimizer = torch.optim.Adam(model.parameters(), lr=LR) best_model_wts_mae, best_model_wts_loss, val_losses_plot = train_weighted_model( model, train_dl, val_dl, criterion, optimizer, EPOCHS, BATCH_SIZE, device, LR) model.load_state_dict(best_model_wts_loss) train_dl = torch.utils.data.DataLoader(train, batch_size=1, shuffle=True) val_dl = torch.utils.data.DataLoader(val, batch_size=1, shuffle=True) print("Evaluating.") def eval(data, model, device):
def nn_train_test_frame(train_x, train_y, orig_x, orig_y, new_x, new_y, frames): train_x = train_x.reshape([train_x.shape[0], train_x.shape[1], -1]) orig_x = orig_x.reshape([orig_x.shape[0], orig_x.shape[1], -1]) new_x = new_x.reshape([new_x.shape[0], -1]) n_frames = len(frames) n_sets = len(train_x) # orig_loss = np.zeros([n_sets, n_frames]) # new_loss = np.zeros([n_sets, n_frames]) D_in = train_x.shape[2] H1 = 10 H2 = 5 D_out = 1 new_dataset = DataSet(new_x, new_y).normalize() n_epochs = 60 loss_fn = nn.BCELoss() optimizer_type = optim.Adam scheduler_type = optim.lr_scheduler.MultiStepLR lr = 0.01 train_losses = torch.zeros([n_sets, n_epochs]) orig_losses = torch.zeros([n_sets, n_epochs]) orig_accuracies = torch.zeros([n_sets, n_epochs]) new_losses = torch.zeros([n_sets, n_epochs]) new_accuracies = torch.zeros([n_sets, n_epochs]) for idx, one_train_x, one_orig_x in enumerate(zip(train_x, orig_x)): print(idx) train_dataset = DataSet(one_train_x, train_y).normalize() orig_dataset = DataSet(one_orig_x, orig_y).normalize() net = DenseNet(D_in, H1, H2, D_out) net = net.double() optimizer = optimizer_type(net.parameters(), lr=lr, weight_decay=0.1) scheduler = scheduler_type(optimizer, [20], gamma=0.1) train_loader = data_utils.DataLoader(train_dataset, batch_size=16, shuffle=True) orig_y_int = orig_y.int() new_y_int = new_y.int() for e in range(n_epochs): scheduler.step() epoch_train_loss = [] net.train() for x, y in train_loader: optimizer.zero_grad() y_pred = net(x) y_pred = y_pred.view(y_pred.numel()) loss = loss_fn(y_pred, y) loss.backward() optimizer.step() epoch_train_loss.append(loss.item()) train_losses[idx, e] = sum(epoch_train_loss) / len(epoch_train_loss) net.eval() with torch.no_grad(): outputs = net(orig_dataset.all_x) outputs = outputs.view(outputs.numel()) orig_losses[idx, e] = loss_fn(outputs, orig_dataset.all_y).item() predictions = torch.round(outputs).int() orig_accuracies[idx, e] = ( predictions == orig_y_int).sum().item() / len(orig_y_int) outputs = net(new_dataset.all_x) outputs = outputs.view(outputs.numel()) new_losses[idx, e] = loss_fn(outputs, new_dataset.all_y).item() predictions = torch.round(outputs).int() new_accuracies[idx, e] = ( predictions == new_y_int).sum().item() / len(new_y_int) # if orig_losses[-1] < 0.01: # return net, train_losses, orig_losses, orig_accuracies # if e > 18 and np.mean(orig_losses[-5:]) > np.mean(orig_losses[-10:-5]): # print('finished at epoch {}'.format(e)) # return net, train_losses, orig_losses, orig_accuracies if e % 5 == 0: print( "{}. train loss: {} orig loss: {} new loss: {}".format( e, train_losses[idx, e], orig_losses[idx, e], new_losses[idx, e])) np.savez('diff_sess_loss', train_losses=train_losses, orig_losses=orig_losses, orig_accuracies=orig_accuracies, new_losses=new_losses, new_accuracies=new_accuracies)