def score(self, X, y, tag_inverse_transformer):
test_pred = self.predict(X)
pred_labels, test_labels = pred2label(test_pred, tag_inverse_transformer), pred2label(y, tag_inverse_transformer)
print utils.get_accuracy(test_labels, pred_labels)
print
return utils.get_classification_score(test_labels, pred_labels)
def valid_and_test(test_program, test_pyreader, get_valid_examples, process, mode, exe, fetch_list):
"""
return auc and acc
"""
# Get Batch Data
batch_data = fluid.io.batch(get_valid_examples, args.batch_size, drop_last=False)
test_pyreader.decorate_paddle_reader(batch_data)
test_pyreader.start()
pred_list = []
while True:
try:
_pred = exe.run(program=test_program,fetch_list=[pred.name])
pred_list += list(_pred)
except fluid.core.EOFException:
test_pyreader.reset()
break
pred_list = np.vstack(pred_list)
if mode == "test":
label_list = process.get_test_label()
elif mode == "valid":
label_list = process.get_valid_label()
if args.task_mode == "pairwise":
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
metric.reset()
metric.update(pred_list, label_list)
auc = metric.eval()
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, label_list, args.task_mode,
args.lamda)
return auc, acc
else:
return auc
def test(self, loader, model, mask, epoch_or_phase):
"""Tests the model and return the accuracy"""
criterion = torch.nn.CrossEntropyLoss().to(self.device)
model.eval()
losses, batch_time, accuracy = AverageMeter(), AverageMeter(), AverageMeter()
mask = to_device(mask, self.device)
with torch.no_grad():
start = time.time()
for inputs, labels in loader:
# Get outputs
inputs, labels = to_device((inputs, labels), self.device)
outputs = model(inputs)
loss = criterion(outputs, labels)
losses.update(loss.data, inputs.size(0))
# Measure accuracy
prob = torch.softmax(outputs, dim=1)
acc = get_accuracy(prob, labels, mask)
accuracy.update(acc, labels.size(0))
batch_time.update(time.time() - start)
start = time.time()
self.logger.info(
f'==> Test [{epoch_or_phase}]:\tTime:{batch_time.sum:.4f}\tLoss:{losses.avg:.4f}\tAcc:{accuracy.avg:.4f}')
return accuracy.avg
def evaluate(self, x, y, cm=None):
y_pred = self.predict(x)
if cm == None:
return utils.get_accuracy(y_pred, y)
labels = np.unique(y)
confusion_matrix = sklearn.metrics.confusion_matrix(y,
y_pred,
labels,
normalize='true')
fig, ax = plt.subplots()
fig.set_size_inches(6, 6)
sklearn.metrics.ConfusionMatrixDisplay(confusion_matrix, labels).plot(
ax=ax, cmap='YlOrBr', values_format='.2%')
plt.savefig(cm, dpi=200)
fig.clf()
return utils.get_accuracy(y_pred, y)
def add_train_ops(self, num_classes, joint_rep, minibatch):
"""Add ops for training in the computation graph.
Args:
num_classes: number of classes to predict in the task.
joint_rep: the joint sentence representation if the input is sentence
pairs or the representation for the sentence if the input is a single
sentence.
minibatch: a minibatch of sequences of embeddings.
Returns:
train_accuracy: the accuracy on the training dataset
loss: training loss.
opt_step: training op.
"""
if self.linear_classifier is None:
classifier_layers = []
classifier_layers.append(snt.Linear(num_classes))
self.linear_classifier = snt.Sequential(classifier_layers)
logits = self.linear_classifier(joint_rep)
# Losses and optimizer.
def get_loss(logits, labels):
return tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=labels,
logits=logits))
loss = get_loss(logits, minibatch.sentiment)
train_accuracy = utils.get_accuracy(logits, minibatch.sentiment)
opt_step = self._add_optimize_op(loss)
return train_accuracy, loss, opt_step
def valid_and_test(program, feeder, reader, process, mode="test"):
"""
return auc and acc
"""
# Get Batch Data
batch_data = paddle.batch(reader, args.batch_size, drop_last=False)
pred_list = []
for data in batch_data():
_pred = executor.run(program=program,
feed=feeder.feed(data),
fetch_list=[pred.name])
pred_list += list(_pred)
pred_list = np.vstack(pred_list)
if mode == "test":
label_list = process.get_test_label()
elif mode == "valid":
label_list = process.get_valid_label()
if args.task_mode == "pairwise":
pred_list = (pred_list + 1) / 2
pred_list = np.hstack(
(np.ones_like(pred_list) - pred_list, pred_list))
metric.reset()
metric.update(pred_list, label_list)
auc = metric.eval()
if args.compute_accuracy:
acc = utils.get_accuracy(pred_list, label_list, args.task_mode,
args.lamda)
return auc, acc
else:
return auc
def train(epoch, max_epoch):
""" train model
"""
# if epoch % 10 == 0:
# for param_group in solver.param_groups:
# param_group['lr'] *= 0.1
loss_meter = AverageMeter()
acc_meter = AverageMeter()
ind = 0
for x, target in tqdm(training_loader, desc="TRAINING", leave=False):
act_lengths = get_seq_length(x)
flatten_target, target_lengths = preprocess_target(target)
x = x.to(Config.device)
x = Variable(x)
act_lengths = Variable(act_lengths)
flatten_target = Variable(flatten_target)
target_lengths = Variable(target_lengths)
output = model(x)
loss = criterion(output, flatten_target, act_lengths, target_lengths)
solver.zero_grad()
loss.backward()
# nn.utils.clip_grad_norm(model.parameters(), 10)
solver.step()
loss_meter.update(loss.data[0])
acc = get_accuracy(output, target)
acc_meter.update(acc)
train_result(epoch, max_epoch, ind, len(training_loader), loss_meter,
acc_meter)
train_result(epoch, max_epoch, ind, len(training_loader), loss_meter,
acc_meter, log_file)
return acc_meter.avg
def go_through_by_batch(model, X, y, training):
''' Given training examples, pass them forward/backward through the network in batches '''
# If no examples, return invalid loss and accuracy
if len(X) == 0:
return float('inf'), float('inf')
assert len(X) == len(y) # Sanity check
total_loss = total_acc = 0
num_batches = len(X) // config.BATCH_SIZE
for idx in range(num_batches):
# Get mini-batches
start_idx = config.BATCH_SIZE * idx
end_idx = start_idx + config.BATCH_SIZE
x_batch = X[start_idx:end_idx]
y_batch = y[start_idx:end_idx]
# Forward pass through the model
x_batch = forward_pass(model, x_batch, training=training)
y_batch_pred = sce_gate.softmax(x_batch)
loss = sce_gate.cross_entropy(y_batch_pred, y_batch)
grad = sce_gate.backward()
total_loss += np.mean(loss)
total_acc += utils.get_accuracy(y_batch_pred, y_batch)
if training:
backward_pass(model, grad)
update_model(model)
# Get average loss and accuracy
avg_loss = total_loss / num_batches
avg_acc = total_acc / num_batches
return avg_loss, avg_acc
def test_SCU(cnn, X_test, y_test):
# convert NumPy Array to Torch Tensor
test_input = torch.from_numpy(X_test)
test_label = torch.from_numpy(y_test)
# create the data loader for the test set
testset = torch.utils.data.TensorDataset(test_input, test_label)
testloader = torch.utils.data.DataLoader(testset,
batch_size=opt.batch_size,
shuffle=False,
num_workers=4)
cnn.eval()
test_cumulative_accuracy = 0
for i, data in enumerate(testloader, 0):
# format the data from the dataloader
test_inputs, test_labels = data
test_inputs, test_labels = test_inputs.to(device), test_labels.to(
device)
test_inputs = test_inputs.float()
test_outputs = cnn(test_inputs)
_, test_predicted = torch.max(test_outputs, 1)
test_acc = get_accuracy(test_labels, test_predicted)
test_cumulative_accuracy += test_acc
return test_cumulative_accuracy, len(testloader)
def train(net, loader, crit, optim, lr_adjuster=None, augmentor=None):
net.train()
n_batches = len(loader)
total_loss = 0
total_acc = 0
for inputs, targets in loader:
inputs = Variable(inputs.cuda())
targets = Variable(targets.cuda())
if augmentor is not None:
inputs = augmentor.transform(inputs)
output = net(inputs)
loss = crit(output, targets)
optim.zero_grad()
loss.backward()
optim.step()
preds = utils.get_argmax(output)
accuracy = utils.get_accuracy(preds, targets.data.cpu().numpy())
total_loss += loss.data[0]
total_acc += accuracy
if lr_adjuster is not None:
lr_adjuster.step()
mean_loss = total_loss / n_batches
mean_acc = total_acc / n_batches
return mean_loss, mean_acc
def test(loader, model, criterion, class_mask, logger, epoch):
model.eval()
losses, batch_time, accuracy, task_accuracy = AverageMeter(), AverageMeter(
), AverageMeter(), AverageMeter()
with torch.no_grad():
start = time.time()
for inputs, labels in loader:
# Get outputs
inputs, labels = inputs.half().cuda(
non_blocking=True), labels.cuda(non_blocking=True)
outputs = model(inputs)
loss = criterion(outputs, labels)
losses.update(loss.data, inputs.size(0))
# Measure accuracy and task accuracy
prob = torch.nn.functional.softmax(outputs, dim=1)
acc, task_acc = get_accuracy(prob, labels, class_mask)
accuracy.update(acc, labels.size(0))
task_accuracy.update(task_acc, labels.size(0))
batch_time.update(time.time() - start)
start = time.time()
logger.info(
'==> Test: [{0}]\tTime:{batch_time.sum:.4f}\tLoss:{losses.avg:.4f}\tAcc:{acc.avg:.4f}\tTask Acc:{task_acc.avg:.4f}\t'
.format(epoch,
batch_time=batch_time,
losses=losses,
acc=accuracy,
task_acc=task_accuracy))
return accuracy.avg
def train(args):
logger.warning('This script is an example to showcase the extensions and '
'data-loading features of Torchmeta, and as such has been '
'very lightly tested.')
dataset = omniglot(args.folder,
shots=args.num_shots,
ways=args.num_ways,
shuffle=True,
test_shots=15,
meta_train=True,
download=args.download)
dataloader = BatchMetaDataLoader(dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
model = PrototypicalNetwork(1,
args.embedding_size,
hidden_size=args.hidden_size)
model.to(device=args.device)
model.train()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
# Training loop
with tqdm(dataloader, total=args.num_batches) as pbar:
for batch_idx, batch in enumerate(pbar):
model.zero_grad()
train_inputs, train_targets = batch['train']
train_inputs = train_inputs.to(device=args.device)
train_targets = train_targets.to(device=args.device)
train_embeddings = model(train_inputs)
test_inputs, test_targets = batch['test']
test_inputs = test_inputs.to(device=args.device)
test_targets = test_targets.to(device=args.device)
test_embeddings = model(test_inputs)
prototypes = get_prototypes(train_embeddings, train_targets,
dataset.num_classes_per_task)
loss = prototypical_loss(prototypes, test_embeddings, test_targets)
loss.backward()
optimizer.step()
with torch.no_grad():
accuracy = get_accuracy(prototypes, test_embeddings, test_targets)
pbar.set_postfix(accuracy='{0:.4f}'.format(accuracy.item()))
if batch_idx >= args.num_batches:
break
# Save model
if args.output_folder is not None:
filename = os.path.join(args.output_folder, 'protonet_omniglot_'
'{0}shot_{1}way.pt'.format(args.num_shots, args.num_ways))
with open(filename, 'wb') as f:
state_dict = model.state_dict()
torch.save(state_dict, f)
def evaluate(self, key):
args = self.param.args
dm = self.param.volatile.dm
dm.restart(key, args.batch_size, shuffle=False)
result_arr = []
while True:
incoming = self.get_next_batch(dm, key, restart=False)
if incoming is None:
break
incoming.args = Storage()
with torch.no_grad():
self.net.forward(incoming)
result_arr.append(incoming.result)
detail_arr = Storage()
for i in args.show_sample:
index = [i * args.batch_size + j for j in range(args.batch_size)]
incoming = self.get_select_batch(dm, key, index)
incoming.args = Storage()
with torch.no_grad():
self.net.forward(incoming)
detail_arr["show_str%d" % i] = incoming.result.show_str
detail_arr.update({'loss':get_mean(result_arr, 'loss'), \
'accuracy':get_accuracy(result_arr, label_key='label', prediction_key='prediction')})
return detail_arr
def outer_loop(self, batch, is_train):
train_inputs, train_targets, test_inputs, test_targets = self.unpack_batch(
batch)
loss_log = 0
acc_log = 0
grad_list = []
loss_list = []
for (train_input, train_target, test_input,
test_target) in zip(train_inputs, train_targets, test_inputs,
test_targets):
with higher.innerloop_ctx(self.network,
self.inner_optimizer,
track_higher_grads=False) as (fmodel,
diffopt):
for step in range(self.args.n_inner):
self.inner_loop(fmodel, diffopt, train_input, train_target)
train_logit = fmodel(train_input)
in_loss = F.cross_entropy(train_logit, train_target)
test_logit = fmodel(test_input)
outer_loss = F.cross_entropy(test_logit, test_target)
loss_log += outer_loss.item() / self.batch_size
with torch.no_grad():
acc_log += get_accuracy(
test_logit, test_target).item() / self.batch_size
if is_train:
params = list(fmodel.parameters(time=-1))
in_grad = torch.nn.utils.parameters_to_vector(
torch.autograd.grad(in_loss, params,
create_graph=True))
outer_grad = torch.nn.utils.parameters_to_vector(
torch.autograd.grad(outer_loss, params))
implicit_grad = self.neumann_approx(
in_grad, outer_grad, params)
grad_list.append(implicit_grad)
loss_list.append(outer_loss.item())
if is_train:
self.outer_optimizer.zero_grad()
weight = torch.ones(len(grad_list))
weight = weight / torch.sum(weight)
grad = mix_grad(grad_list, weight)
grad_log = apply_grad(self.network, grad)
self.outer_optimizer.step()
return loss_log, acc_log, grad_log
else:
return loss_log, acc_log
def test(epoch, max_epoch):
acc_meter = AverageMeter()
for x, target in tqdm(testing_loader, desc="TESTING", leave=False):
x = Variable(x).to(Config.device)
output = model(x)
acc = get_accuracy(output, target)
acc_meter.update(acc)
test_result(epoch, max_epoch, acc_meter)
test_result(epoch, max_epoch, acc_meter, log_file)
return acc_meter.avg
def test(model, dataloader, args):
model.eval()
epoch_stats = EpochStats()
for batch_idx, batch in enumerate(tqdm(dataloader, desc="Batch")):
_, test_batch, _ = concurrent_multi_task_train_test_split(
batch, False, tasks=args.tasks)
test_batch = test_batch[0]
test_batch = test_batch.to(args.device)
with torch.no_grad():
gc_test_logit, nc_test_logit, lp_test_logit = model(test_batch)
# GC
if "gc" in args.tasks:
gc_loss = F.cross_entropy(gc_test_logit, test_batch.y)
with torch.no_grad():
gc_acc = ut.get_accuracy(gc_test_logit, test_batch.y)
epoch_stats.update("gc", test_batch, gc_loss, gc_acc, False)
#NC
if "nc" in args.tasks:
node_labels = test_batch.node_y.argmax(1)
train_mask = test_batch.train_mask.squeeze()
test_mask = (train_mask == 0).float()
nc_loss = F.cross_entropy(nc_test_logit[test_mask == 1],
node_labels[test_mask == 1])
with torch.no_grad():
nc_acc = ut.get_accuracy(nc_test_logit[test_mask == 1],
node_labels[test_mask == 1])
epoch_stats.update("nc", test_batch, nc_loss, nc_acc, False)
# LP
if "lp" in args.tasks:
test_link_labels = data_utils.get_link_labels(
test_batch.pos_edge_index, test_batch.neg_edge_index)
lp_loss = F.binary_cross_entropy_with_logits(
lp_test_logit.squeeze(), test_link_labels)
with torch.no_grad():
test_labels = test_link_labels.detach().cpu().numpy()
test_predictions = lp_test_logit.detach().cpu().numpy()
lp_acc = roc_auc_score(test_labels,
test_predictions.squeeze())
epoch_stats.update("lp", test_batch, lp_loss, lp_acc, False)
tasks_test_stats = epoch_stats.get_average_stats()
bl_ut.print_test_stats(tasks_test_stats)
return tasks_test_stats
def test(model, k, num_each_class):
model.eval() # TODO: notice there is a difference in L0 gate
base, test, label = utils.get_test_data(num_each_class=num_each_class)
latent_test = get_latent(test)
latent_base = get_latent(base)
pred = kNNClassifer(latent_base, latent_test, k)
acc = utils.get_accuracy(pred, label).tolist()
print('test acc of {}-NN is {}:'.format(k, acc))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--input_file', help="Input csv file with data.")
args = parser.parse_args()
dataset = utils.load_csv(args.input_file)
train_set, test_set = corpus.split_dataset(dataset, 0.67)
separated = corpus.separate_by_class(train_set)
summaries = corpus.summarize_by_class(train_set)
predictions = predict_set(summaries, test_set)
accuracy = utils.get_accuracy(test_set, predictions)
print('Accuracy: {0}%').format(accuracy)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-i', '--input_file', help="Input csv file with data.")
args = parser.parse_args()
dataset = utils.load_csv(args.input_file)
train_set, test_set = corpus.split_dataset(dataset, 0.67)
separated = corpus.separate_by_class(train_set)
summaries = corpus.summarize_by_class(train_set)
predictions = predict_set(summaries, test_set)
accuracy = utils.get_accuracy(test_set, predictions)
print('Accuracy: {0}%').format(accuracy)
def compare_crf():
reader = Reader()
crf_rnn = CRF_RNN(input_shape=[512, 512, 1],
batch_size=1,
l2_weight=0.001,
input_dim=262144,
learning_rate=6e-4,
pre_train=True)
show_all_variables()
iter = 30
acc_without_crf = 0.0
acc_with_crf = 0.0
for _ in range(iter):
xs, ys = reader.next_batch(1)
pred = crf_rnn.predict(xs, as_list=False)
infe = crf_rnn.predict_n_inference(xs, as_list=False)
tmp_without_crf = get_accuracy(pred, ys)
tmp_with_crf = get_accuracy(infe, ys)
print('--Precision without CRF: %g --Precision with CRF: %g' %
(tmp_without_crf, tmp_with_crf))
acc_with_crf += tmp_with_crf
acc_without_crf += tmp_without_crf
plt.figure()
plt.subplot(121)
plt.imshow(pred.reshape((512, 512)), cmap='gray')
plt.subplot(122)
plt.imshow(infe.reshape((512, 512)), cmap='gray')
plt.show()
plt.close()
plt.show()
print(
'\nTotal Precision:\n\t--Precision without CRF: %g\n\t--Precision with CRF: %g'
% (acc_without_crf / float(iter), acc_with_crf / float(iter)))
print('\nFinished!!!')
def eval_baseline_nn_output_model(output_model, dataloader, output_task, device="cpu"):
output_model.eval()
epoch_stats = EpochStats()
for batch_idx, batch in enumerate(tqdm(dataloader, desc="Eval Batch")):
batch = prepare_batch_for_task(batch, output_task, train=False)
batch = batch.to(device)
with torch.no_grad():
# Forward pass
if output_task == "gc":
test_logit = output_model(batch.node_embeddings, batch.batch)
elif output_task == "nc":
test_logit = output_model(batch.node_embeddings)
elif output_task == "lp":
test_logit = output_model(batch.node_embeddings, batch.pos_edge_index, batch.neg_edge_index)
# Evaluate Loss and Accuracy
if output_task == "gc":
loss = F.cross_entropy(test_logit, batch.y)
with torch.no_grad():
acc = ut.get_accuracy(test_logit, batch.y)
elif output_task == "nc":
node_labels = batch.node_y.argmax(1)
train_mask = batch.train_mask.squeeze()
test_mask = (train_mask==0).float()
loss = F.cross_entropy(test_logit[test_mask==1], node_labels[test_mask==1])
with torch.no_grad():
acc = ut.get_accuracy(test_logit[test_mask==1], node_labels[test_mask==1])
elif output_task == "lp":
test_link_labels = data_utils.get_link_labels(batch.pos_edge_index, batch.neg_edge_index)
loss = F.binary_cross_entropy_with_logits(test_logit.squeeze(), test_link_labels)
with torch.no_grad():
test_labels = test_link_labels.detach().cpu().numpy()
test_predictions = test_logit.detach().cpu().numpy()
acc = roc_auc_score(test_labels, test_predictions.squeeze())
epoch_stats.update(output_task, batch, loss, acc, False)
task_test_stats = epoch_stats.get_average_stats()
bl_ut.print_test_stats(task_test_stats)
return task_test_stats
def train_loop(loader, model, epochs = 3, start_epoch = 0, params = None, device = None, loss_func = torch.nn.CrossEntropyLoss, n_tops = [1, 5]):
L_RATE, DECAY_RATE, DECAY_EPOCHS, WEIGHT_DECAY, SAVE_MODEL, SAVE_MODEL_N, SAVE_MODEL_DIR, MODEL, N_LAYERS = params
optimizer = optim.Adam(model.parameters(), lr = L_RATE, weight_decay = WEIGHT_DECAY)
if SAVE_MODEL:
if MODEL == 'Darknet':
path = '{}{}'.format(MODEL, N_LAYERS)
else:
path = MODEL
if not os.path.exists('{}/{}'.format(SAVE_MODEL_DIR, path)):
os.makedirs('{}/{}'.format(SAVE_MODEL_DIR, path))
losses, accuracies = {'train': [], 'validate': []}, {'train': [], 'validate': []}
for epoch in range(start_epoch, epochs + start_epoch):
t = time()
if (epoch + 1) % DECAY_EPOCHS == 0:
L_RATE *= (1 - DECAY_RATE)
optimizer = optim.Adam(model.parameters(), lr=L_RATE, weight_decay=WEIGHT_DECAY)
# print epoch number
print_report(part = 'start', epoch = epoch)
# train loop
train_epoch(loader['train'], model, optimizer, device, loss_func)
# print metrics
val_acc, val_loss = get_accuracy(loader['val'], model, device, dtype, loss_func, n_tops)
train_acc, train_loss = get_accuracy(loader['train'], model, device, dtype, loss_func, n_tops)
metrics = train_loss, val_loss, train_acc, val_acc, n_tops
print_report(part='accuracy', metrics = metrics)
# collect metrics
losses['train'].append(train_loss)
losses['validate'].append(val_loss)
accuracies['train'].append(train_acc)
accuracies['validate'].append(val_acc)
# save models
if SAVE_MODEL:
save_checkpoint(model = model, cfg = cfg, epoch = epoch, loss = round(val_loss, 3))
# print time
print_report(part='end', t = int(time() - t))
def test(net, tst_loader, criterion):
net.eval()
test_loss = 0
test_acc = 0
for data in tst_loader:
inputs = Variable(data[0].cuda(), volatile=True)
target = Variable(data[1].cuda())
output = net(inputs)
test_loss += criterion(output, target).data[0]
pred = utils.get_argmax(output)
test_acc += utils.get_accuracy(pred, target.data.cpu().numpy())
test_loss /= len(tst_loader)
test_acc /= len(tst_loader)
return test_loss, test_acc
def test(model, dataloader, args):
model.eval()
epoch_stats = EpochStats()
for batch_idx, batch in enumerate(tqdm(dataloader, desc="Batch")):
test_batch = prepare_batch_for_task(batch, args.task, train=False)
test_batch = test_batch.to(args.device)
with torch.no_grad():
test_logit = model(test_batch)
if args.task == "gc":
loss = F.cross_entropy(test_logit, test_batch.y)
with torch.no_grad():
acc = ut.get_accuracy(test_logit, test_batch.y)
elif args.task == "nc":
node_labels = test_batch.node_y.argmax(1)
train_mask = test_batch.train_mask.squeeze()
test_mask = (train_mask == 0).float()
loss = F.cross_entropy(test_logit[test_mask == 1],
node_labels[test_mask == 1])
with torch.no_grad():
acc = ut.get_accuracy(test_logit[test_mask == 1],
node_labels[test_mask == 1])
elif args.task == "lp":
test_link_labels = data_utils.get_link_labels(
test_batch.pos_edge_index, test_batch.neg_edge_index)
loss = F.binary_cross_entropy_with_logits(
test_logit.squeeze(), test_link_labels)
with torch.no_grad():
test_labels = test_link_labels.detach().cpu().numpy()
test_predictions = test_logit.detach().cpu().numpy()
acc = roc_auc_score(test_labels,
test_predictions.squeeze())
epoch_stats.update(args.task, test_batch, loss, acc, False)
task_test_stats = epoch_stats.get_average_stats()
bl_ut.print_test_stats(task_test_stats)
return task_test_stats
def test(test_loader, model_base, model_gaze, model_attn, num_action):
model_base.eval()
model_gaze.eval()
model_attn.eval()
list_true = []
list_pred = []
start_time = time.time()
with torch.no_grad():
for i, (rgb, flow, label) in enumerate(test_loader, 1):
label = label.to(device)
len_video, height, width = rgb.shape[2:]
top, left = (height - args.crop) // 2, (width - args.crop) // 2
jump = args.trange
if args.test_sparse:
if len_video > args.trange * 10:
jump = len_video // 10
list_start_idx = list(range(0, len_video - args.trange + 1, jump))
list_y = []
for t in list_start_idx:
t_rgb = rgb[..., t:t + args.trange, top:top + args.crop,
left:left + args.crop].cuda()
t_flow = flow[..., t:t + args.trange, top:top + args.crop,
left:left + args.crop].cuda()
pi, h = model_base(t_rgb, t_flow)
pi = model_gaze(pi)
z_hard, pi_g = make_hard_decision(pi, device)
y = compute_cross_entropy(z_hard, h, model_attn, label)[0]
list_y.append(y)
y_avg = torch.cat(list_y, 0).mean(0, keepdim=True)
list_true.append(label.item())
list_pred.append(torch.argmax(y_avg, 1).item())
print('step: %04d, %s' %
(i, timedelta(seconds=int(time.time() - start_time))),
flush=True)
mean_class_acc, acc = get_accuracy(
confusion_matrix(list_true, list_pred, labels=list(range(num_action))))
print('acc: %.2f, %.2f / %s' %
(mean_class_acc, acc,
timedelta(seconds=int(time.time() - start_time))),
flush=True)
def outer_loop(self, batch, is_train):
train_inputs, train_targets, test_inputs, test_targets = self.unpack_batch(
batch)
test_losses = [0 for _ in range(self.args.n_inner)]
test_corrects = [0 for _ in range(self.args.n_inner)]
test_accs = [0 for _ in range(self.args.n_inner)]
for (train_input, train_target, test_input,
test_target) in zip(train_inputs, train_targets, test_inputs,
test_targets):
fast_weights = OrderedDict(self.network.named_parameters())
for i in range(self.args.n_inner):
train_logit = self.network.functional_forward(
train_input, fast_weights)
train_loss = F.cross_entropy(train_logit, train_target)
train_grad = torch.autograd.grad(train_loss,
fast_weights.values(),
create_graph=True)
# Update weights manually
fast_weights = OrderedDict(
(name, param - self.args.inner_lr * grad)
for ((name, param),
grad) in zip(fast_weights.items(), train_grad))
test_logit = self.network.functional_forward(
test_input, fast_weights)
test_loss = F.cross_entropy(test_logit, test_target)
test_loss.backward(retain_graph=True)
test_losses[i] += test_loss
with torch.no_grad():
test_acc = get_accuracy(test_logit, test_target).item()
test_accs[i] += test_acc
acc_log = test_accs[-1] / self.batch_size
loss_log = test_losses[-1] / self.args.batch_size
if is_train:
self.outer_optimizer.zero_grad()
loss_log.backward()
self.outer_optimizer.step()
return loss_log.item(), acc_log, loss_log.item()
else:
return loss_log.item(), acc_log
def train(df, selected_features=utils.SELECTED_FEATURES, alpha=0.1, epsilon=0.0001, reg_param=100, train_size=0.8,
mode="gradient", batch_size=None, iterations=None):
train_df = df.sample(frac=train_size, random_state=7)
test_df = df.drop(train_df.index)
thetas_dict, cost_list_dict = get_thetas_train(train_df, selected_features=selected_features, alpha=alpha,
epsilon=epsilon, reg_param=reg_param, mode=mode,
batch_size=batch_size, iterations=iterations)
if thetas_dict is None:
return None, None, None
prediction = predict(test_df, thetas_dict)
truth = test_df[["Index", "Hogwarts House"]]
accuracy = utils.get_accuracy(prediction, truth)
return thetas_dict, cost_list_dict, accuracy
def __init__(self, input_dim, output_dim, init_noisevar=None):
self.input_dim = input_dim
self.output_dim = output_dim
self.true_outputs = tf.placeholder(tf.float32, [
None,
output_dim,
],
name='trueY')
self.iblayerobj = iblayer.NoisyIBLayer(init_noisevar=init_noisevar,
name='noisy_ib_layer')
self.layers = []
self.layers.append(
tf.placeholder(tf.float32, [
None,
input_dim,
], name='X'))
self.layers.append(
tf.keras.layers.Dense(cfg['n_wide'],
activation=tf.nn.relu)(self.layers[-1]))
self.layers.append(
tf.keras.layers.Dense(cfg['n_wide'],
activation=tf.nn.relu)(self.layers[-1]))
self.layers.append(
tf.keras.layers.Dense(cfg['n_hidden'],
activation=None)(self.layers[-1]))
self.layers.append(self.iblayerobj(self.layers[-1]))
self.layers.append(
tf.keras.layers.Dense(cfg['n_wide'],
activation=tf.nn.relu)(self.layers[-1]))
self.layers.append(
tf.keras.layers.Dense(output_dim, activation=None,
name='Y')(self.layers[-1]))
self.inputs = self.layers[0]
self.predictions = self.layers[-1]
self.entropyY = tf.placeholder(dtype=tf.float32,
shape=(),
name='entropyY')
self.cross_entropy = utils.get_error(errtype=data['err'],
y_true=self.true_outputs,
y_pred=self.predictions)
self.accuracy = utils.get_accuracy(errtype=data['err'],
y_true=self.true_outputs,
y_pred=self.predictions)
def evaluate(model, test_dataloader, device, criterior):
test_loss, accuracy, count = 0, 0, 0
# treshold = args.trs_hold
model.eval()
with torch.no_grad():
for batch_ind, (input, target) in enumerate(test_dataloader):
input, target = input.to(device), target.to(device)
output = model(input)
output_after_tresh = utils.apply_trashold(output, 0.1)
tmp, length = utils.get_accuracy(output_after_tresh, target)
accuracy+=tmp
count+=length
loss = criterior(output, target)
test_loss+=loss.item()
print(f'Test loss is:{test_loss/len(test_dataloader):.5}')
print(f'Accuracy is: {accuracy/count:.5}')
def outer_loop(self, batch, is_train):
self.network.zero_grad()
train_inputs, train_targets, test_inputs, test_targets = self.unpack_batch(
batch)
loss_log = 0
acc_log = 0
grad_list = []
loss_list = []
for (train_input, train_target, test_input,
test_target) in zip(train_inputs, train_targets, test_inputs,
test_targets):
with higher.innerloop_ctx(
self.network,
self.inner_optimizer,
track_higher_grads=is_train) as (fmodel, diffopt):
for step in range(self.args.n_inner):
self.inner_loop(fmodel, diffopt, train_input, train_target)
test_logit = fmodel(test_input)
outer_loss = F.cross_entropy(test_logit, test_target)
loss_log += outer_loss.item() / self.batch_size
with torch.no_grad():
acc_log += get_accuracy(
test_logit, test_target).item() / self.batch_size
if is_train:
outer_grad = torch.autograd.grad(outer_loss,
fmodel.parameters(time=0))
grad_list.append(outer_grad)
loss_list.append(outer_loss.item())
if is_train:
weight = torch.ones(len(grad_list))
weight = weight / torch.sum(weight)
grad = mix_grad(grad_list, weight)
grad_log = apply_grad(self.network, grad)
self.outer_optimizer.step()
return loss_log, acc_log, grad_log
else:
return loss_log, acc_log
def train(model,
dataloader,
epochs,
optimizer,
criterion,
save_output_every=10,
save_model_every=50,
only_show=False):
device = utils.get_device()
for epoch in tqdm(range(1, epochs + 1)):
losses_per_epoch = []
accuracies_per_epoch = []
# go over all batches
for step, (input, target) in enumerate(dataloader):
model.train()
input, target = input.to(device), target.to(device)
target = utils.preprocess_target(target)
pred = model(input)
optimizer.zero_grad()
loss = criterion(pred, target)
loss.backward()
optimizer.step()
acc = utils.get_accuracy(pred, target)
losses_per_epoch.append(loss.item())
accuracies_per_epoch.append(acc)
mean_loss = np.mean(losses_per_epoch)
mean_acc = np.mean(accuracies_per_epoch)
print(f'{"-"*30} Epoch {epoch} {"-"*30}')
print('Loss: %.3f Accuracy: %.3f' % (mean_loss, mean_acc))
if epoch % save_output_every == 0:
utils.save_result(epoch,
input,
pred,
target,
name='epoch',
only_show=only_show)
if epoch % save_model_every == 0:
utils.save_model(epoch, model)
def predict(in_file, out_file):
xvals, yvals = utils.load_data(in_file)
network = build_network()
predictions = utils.predict(xvals, network, 'circle.tflearn')
print('Accuracy: {}%'.format(utils.get_accuracy(yvals, predictions)))
utils.write_predictions(xvals, predictions, out_file)
