def evaluate(model_name, log_dir):
noise_levels = np.arange(0.0,0.55,0.05)
print("noise levels {}".format(noise_levels))
accuracies = np.zeros(noise_levels.shape)
test_summary_writer = tf.summary.create_file_writer(os.path.join(log_dir,
'summaries',
'test'))
#restore model
if model_name=="sparseCNN":
model = SparseNet()
elif model_name=="denseCNN":
model = DenseNet()
else:
raise ValueError("Model name unrecognized {}".format(model_name))
ckpt = tf.train.Checkpoint(net=model)
ckpt_path = os.path.join(log_dir, 'checkpoints')
manager = tf.train.CheckpointManager(ckpt,ckpt_path,max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
test_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
for i,noise in enumerate(noise_levels):
test_dataset, _ = datasets.get_dataset("mnist",
100,
subset="test",
shuffle=False,
noise_level=noise)
with test_summary_writer.as_default():
for x_batch, y_batch in test_dataset:
if len(x_batch.shape)==3:
x_batch = tf.expand_dims(x_batch, 3)
test_logits = model(x_batch, training=False)
# Update test metrics
test_acc_metric(y_batch, test_logits)
test_acc = test_acc_metric.result()
tf.summary.scalar("noise_accuracy", test_acc, step=i)
tf.summary.image("noisy_image", x_batch, step=i)
accuracies[i] = float(test_acc)
test_acc_metric.reset_states()
print('Model {} noise {} Test acc: {}'.format(model_name, noise, float(test_acc)))
print("Average noise score accros noise level {}".format(np.mean(accuracies)))
def main(datasetname, n_classes, batch_size, model_name, epochs, lr, keep_prob,
base_log_dir):
#Fix TF random seed
tf.random.set_seed(1777)
log_dir = os.path.join(os.path.expanduser(base_log_dir),
"{}".format(datasetname))
os.makedirs(log_dir, exist_ok=True)
# dataset
train_dataset, train_samples = datasets.get_dataset(
datasetname, batch_size)
test_dataset, _ = datasets.get_dataset(datasetname,
batch_size,
subset="test",
shuffle=False)
#Network
if model_name == "sparseCNN":
model = SparseNet()
elif model_name == "denseCNN":
model = DenseNet()
else:
raise ValueError("Model name unrecognized {}".format(model_name))
#Train optimizer, loss
nrof_steps_per_epoch = (train_samples // batch_size)
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
lr, nrof_steps_per_epoch, 0.8)
optimizer = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.9)
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy()
#metrics
train_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
test_acc_metric = tf.keras.metrics.SparseCategoricalAccuracy()
#Train step
@tf.function
def train_step(x, labels):
with tf.GradientTape() as t:
logits = model(x, training=True)
loss = loss_fn(labels, logits)
gradients = t.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(gradients, model.trainable_variables))
return loss, logits
#Run
ep_cnt = tf.Variable(initial_value=0, trainable=False, dtype=tf.int64)
#Summary writers
train_summary_writer = tf.summary.create_file_writer(
os.path.join(log_dir, 'summaries', 'train'))
test_summary_writer = tf.summary.create_file_writer(
os.path.join(log_dir, 'summaries', 'test'))
ckpt = tf.train.Checkpoint(step=tf.Variable(1),
optimizer=optimizer,
net=model)
ckpt_path = os.path.join(log_dir, 'checkpoints')
manager = tf.train.CheckpointManager(ckpt, ckpt_path, max_to_keep=3)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
else:
print("Initializing from scratch.")
for ep in tqdm.trange(epochs, desc='Epoch Loop'):
if ep < ep_cnt:
continue
# update epoch counter
ep_cnt.assign_add(1)
with train_summary_writer.as_default():
# train for an epoch
for step, (x, y) in enumerate(train_dataset):
if len(x.shape) == 3:
x = tf.expand_dims(x, 3)
tf.summary.image("input_image", x, step=optimizer.iterations)
loss, logits = train_step(x, y)
train_acc_metric(y, logits)
ckpt.step.assign_add(1)
tf.summary.scalar("loss", loss, step=optimizer.iterations)
if int(ckpt.step) % 1000 == 0:
save_path = manager.save()
print("Saved checkpoint for step {}: {}".format(
int(ckpt.step), save_path))
# Log every 200 batch
if step % 200 == 0:
train_acc = train_acc_metric.result()
print("Training loss {:1.2f}, accuracu {} at step {}".format(\
loss.numpy(),
float(train_acc),
step))
# Display metrics at the end of each epoch.
train_acc = train_acc_metric.result()
tf.summary.scalar("accuracy", train_acc, step=ep)
print('Training acc over epoch: %s' % (float(train_acc), ))
# Reset training metrics at the end of each epoch
train_acc_metric.reset_states()
############################## Test the model #############################
with test_summary_writer.as_default():
for x_batch, y_batch in test_dataset:
if len(x_batch.shape) == 3:
x_batch = tf.expand_dims(x_batch, 3)
test_logits = model(x_batch, training=False)
# Update test metrics
test_acc_metric(y_batch, test_logits)
test_acc = test_acc_metric.result()
tf.summary.scalar("accuracy", test_acc, step=ep)
tf.summary.image("test_image", x_batch, step=ep)
test_acc_metric.reset_states()
print('[Epoch {}] Test acc: {}'.format(ep, float(test_acc)))
features = cuda_var_wrapper(features[:, :, :max_length], volatile=True)
labels = cuda_var_wrapper(labels[:, :max_length], volatile=True)
output = cnn(features)
correct_batch, total_batch = get_batch_accuracy(
labels, output, lengths)
correct += correct_batch
total += total_batch
print("{} out of {} label predictions are correct".format(correct, total))
return correct / total
writer = SummaryWriter(log_dir=writer_path)
if args.model == 'mufold':
cnn = MUFold_ss(dropout=args.dropout)
else:
cnn = DenseNet(drop_rate=args.dropout)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=args.lr)
# optimizer = torch.optim.SGD(cnn.parameters(), lr=args.lr, momentum=0.9, nesterov=True)
scheduler = ReduceLROnPlateau(optimizer,
mode='max',
patience=args.patience,
factor=0.1,
min_lr=args.min_lr,
verbose=True)
if use_cuda:
cnn.cuda()
criterion.cuda()
load_trained_model = os.path.join(
if not args.train and not args.test:
print("You should train or test your network. Please check params.")
exit()
# some default params dataset/architecture related
train_params = get_train_params()
print("Params:")
for k, v in model_params.items():
print("\t%s: %s" % (k, v))
print("Train params:")
for k, v in train_params.items():
print("\t%s: %s" % (k, v))
print("Initialize the model..")
model = DenseNet(**model_params)
if args.train:
# model.load_model()
w1, w2 = model.train_all_epochs(train_params)
if args.test:
w1 = 0.5
w2 = 0.5
if not args.train:
model.load_model()
print("Testing...")
_, _, _,_,_, _, _, _,_,_, _, acc1, acc2, accuracy = model.test(data_provider.test, w1, w2, batch_size=8)
print("mean accuracy1: %f" % (acc1))
print("mean accuracy2: %f" % (acc2))
print("mean accuracy: %f" % (accuracy))
args.bc_mode = True
model_params = vars(args)
if not args.train and not args.test:
print("You should train or test your network. Please check params.")
exit()
# some default params dataset/architecture related
train_params = get_train_params_by_name(args.dataset)
print("Params:")
for k, v in model_params.items():
print("\t%s: %s" % (k, v))
print("Train params:")
for k, v in train_params.items():
print("\t%s: %s" % (k, v))
print("Prepare training data...")
data_provider = get_data_provider_by_name(args.dataset, train_params)
print("Initialize the model..")
model = DenseNet(data_provider=data_provider, **model_params)
if args.train:
print("Data provider train images: ", data_provider.train.num_examples)
model.train_all_epochs(train_params)
if args.test:
if not args.train:
model.load_model()
print("Data provider test images: ", data_provider.test.num_examples)
print("Testing...")
loss, accuracy = model.test(data_provider.test, batch_size=200)
print("mean cross_entropy: %f, mean accuracy: %f" % (loss, accuracy))
class Tool():
@staticmethod
def edit_distance(s1, s2):
size = len(s1)
match = 0
for i in range(0, size - 1):
if s1[i] == s2[i]:
match += 1
return match / size
cnn = DenseNet(channel_size, batch_size, drop_rate=0, num_init_features=512)
criterion = nn.L1Loss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=0.001)
if use_cuda:
cnn.cuda()
criterion.cuda()
train_dataset = Protein_Dataset(0, 5800)
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=16)
# Train the Model
train = [(k, v, w) for k, v, w in zip(x_train, y_train, w_train)]
val = [(k, v, w) for k, v, w in zip(x_val, y_val, w_val)]
for iteration in range(0, 10):
train_dl = torch.utils.data.DataLoader(train,
batch_size=BATCH_SIZE,
shuffle=True)
val_dl = torch.utils.data.DataLoader(val,
batch_size=BATCH_SIZE,
shuffle=True)
# Model setup
# device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
device = torch.device("cpu")
model = DenseNet().to(device)
criterion = torch.nn.MSELoss(reduction='mean')
optimizer = torch.optim.Adam(model.parameters(), lr=LR)
best_model_wts_mae, best_model_wts_loss, val_losses_plot = train_weighted_model(
model, train_dl, val_dl, criterion, optimizer, EPOCHS, BATCH_SIZE,
device, LR)
model.load_state_dict(best_model_wts_loss)
train_dl = torch.utils.data.DataLoader(train, batch_size=1, shuffle=True)
val_dl = torch.utils.data.DataLoader(val, batch_size=1, shuffle=True)
print("Evaluating.")
def eval(data, model, device):
def nn_train_test_frame(train_x, train_y, orig_x, orig_y, new_x, new_y,
frames):
train_x = train_x.reshape([train_x.shape[0], train_x.shape[1], -1])
orig_x = orig_x.reshape([orig_x.shape[0], orig_x.shape[1], -1])
new_x = new_x.reshape([new_x.shape[0], -1])
n_frames = len(frames)
n_sets = len(train_x)
# orig_loss = np.zeros([n_sets, n_frames])
# new_loss = np.zeros([n_sets, n_frames])
D_in = train_x.shape[2]
H1 = 10
H2 = 5
D_out = 1
new_dataset = DataSet(new_x, new_y).normalize()
n_epochs = 60
loss_fn = nn.BCELoss()
optimizer_type = optim.Adam
scheduler_type = optim.lr_scheduler.MultiStepLR
lr = 0.01
train_losses = torch.zeros([n_sets, n_epochs])
orig_losses = torch.zeros([n_sets, n_epochs])
orig_accuracies = torch.zeros([n_sets, n_epochs])
new_losses = torch.zeros([n_sets, n_epochs])
new_accuracies = torch.zeros([n_sets, n_epochs])
for idx, one_train_x, one_orig_x in enumerate(zip(train_x, orig_x)):
print(idx)
train_dataset = DataSet(one_train_x, train_y).normalize()
orig_dataset = DataSet(one_orig_x, orig_y).normalize()
net = DenseNet(D_in, H1, H2, D_out)
net = net.double()
optimizer = optimizer_type(net.parameters(), lr=lr, weight_decay=0.1)
scheduler = scheduler_type(optimizer, [20], gamma=0.1)
train_loader = data_utils.DataLoader(train_dataset,
batch_size=16,
shuffle=True)
orig_y_int = orig_y.int()
new_y_int = new_y.int()
for e in range(n_epochs):
scheduler.step()
epoch_train_loss = []
net.train()
for x, y in train_loader:
optimizer.zero_grad()
y_pred = net(x)
y_pred = y_pred.view(y_pred.numel())
loss = loss_fn(y_pred, y)
loss.backward()
optimizer.step()
epoch_train_loss.append(loss.item())
train_losses[idx,
e] = sum(epoch_train_loss) / len(epoch_train_loss)
net.eval()
with torch.no_grad():
outputs = net(orig_dataset.all_x)
outputs = outputs.view(outputs.numel())
orig_losses[idx, e] = loss_fn(outputs,
orig_dataset.all_y).item()
predictions = torch.round(outputs).int()
orig_accuracies[idx, e] = (
predictions == orig_y_int).sum().item() / len(orig_y_int)
outputs = net(new_dataset.all_x)
outputs = outputs.view(outputs.numel())
new_losses[idx, e] = loss_fn(outputs, new_dataset.all_y).item()
predictions = torch.round(outputs).int()
new_accuracies[idx, e] = (
predictions == new_y_int).sum().item() / len(new_y_int)
# if orig_losses[-1] < 0.01:
# return net, train_losses, orig_losses, orig_accuracies
# if e > 18 and np.mean(orig_losses[-5:]) > np.mean(orig_losses[-10:-5]):
# print('finished at epoch {}'.format(e))
# return net, train_losses, orig_losses, orig_accuracies
if e % 5 == 0:
print(
"{}. train loss: {} orig loss: {} new loss: {}".format(
e, train_losses[idx, e], orig_losses[idx, e],
new_losses[idx, e]))
np.savez('diff_sess_loss',
train_losses=train_losses,
orig_losses=orig_losses,
orig_accuracies=orig_accuracies,
new_losses=new_losses,
new_accuracies=new_accuracies)