def evaluate_current_config(docs, labels, tvt_idx, verbose=True):
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
# paras = count_parameters(model)
# print("\t\"", config["test_name"], "\" : ", paras, sep="")
# return 0
trainer = Trainer(dataset, model)
best_val_loss = float('inf')
best_model_test = 0
time_since_best = 0
for i in range(config["epochs"]):
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
if verbose:
print(
config["indices"] +
" [epoch %02d] Train loss %.4f, Val loss %.4f, Test Acc %.4f" %
(i, train_loss, val_loss, test_acc))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
best_model_test = test_acc
time_since_best = 0
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
print(
"\n ## [RESULT!] %s achieved final test score at epoch %i: %.4f ## \n"
% (config["test_name"], i - time_since_best, best_model_test))
break
del dataset
del model
del trainer
gc.collect()
torch.cuda.empty_cache()
return best_model_test
def main():
docs, labels, (t_idx, v_idx, test_idx) = load_data()
in_training_test = test_idx[:len(test_idx) // 2]
out_training_test = test_idx[len(test_idx) // 2:]
# train on (train, val, in_training_test)
train_docs = docs[t_idx + v_idx + in_training_test]
train_labels = labels[t_idx + v_idx + in_training_test]
train_and_intest_indices = (t_idx, v_idx,
range(
len(t_idx) + len(v_idx),
len(train_labels)))
a = 0
b = 0
# test on (in_training_test)
only_in_training_docs = docs[t_idx[:a] + v_idx[:b] + in_training_test]
only_in_training_labels = labels[t_idx[:a] + v_idx[:b] + in_training_test]
only_in_training_indices = ([t_idx[:a]], [v_idx[:b]],
range(a + b, len(only_in_training_labels)))
# test on (out_training_test)
only_out_training_docs = docs[t_idx[:a] + v_idx[:b] + out_training_test]
only_out_training_labels = labels[t_idx[:a] + v_idx[:b] +
out_training_test]
only_out_training_indices = ([t_idx[:a]], [v_idx[:b]],
range(a + b, len(only_out_training_labels)))
# test on (train, val, out_training_test)
train_and_outtest_docs = docs[t_idx + v_idx + out_training_test]
train_and_outtest_labels = labels[t_idx + v_idx + out_training_test]
train_and_outtest_indices = (t_idx, v_idx,
range(
len(t_idx) + len(v_idx),
len(train_and_outtest_labels)))
train_and_intest_dataset = DocumentGraphDataset(train_docs, train_labels,
train_and_intest_indices)
only_in_training_dataset = DocumentGraphDataset(
only_in_training_docs,
only_in_training_labels,
only_in_training_indices,
force_vocab=train_and_intest_dataset.vocab)
only_out_training_dataset = DocumentGraphDataset(
only_out_training_docs,
only_out_training_labels,
only_out_training_indices,
force_vocab=train_and_intest_dataset.vocab)
train_and_outtest_dataset = DocumentGraphDataset(
train_and_outtest_docs,
train_and_outtest_labels,
train_and_outtest_indices,
force_vocab=train_and_intest_dataset.vocab)
model = None
best_val_loss = float('inf')
time_since_best = 0
for i in range(config["epochs"]):
if model is None:
model = create_model(train_and_intest_dataset)
trainer = Trainer(train_and_intest_dataset, model)
else:
trainer.update_data(train_and_intest_dataset)
train_loss, val_loss = trainer.train_epoch()
test_acc_train_and_intest = trainer.test()
trainer.update_data(only_in_training_dataset)
test_acc_only_in_training = trainer.test()
trainer.update_data(only_out_training_dataset)
test_acc_only_out_training = trainer.test()
trainer.update_data(train_and_outtest_dataset)
test_acc_train_and_outtest = trainer.test()
print("[epoch %02d] Train loss %.4f, Val loss %.4f" %
(i, train_loss, val_loss))
print(" acc on in training test, with training docs in graph: %.4f" %
(test_acc_train_and_intest))
print(
" acc on in training test, WITHOUT training docs in graph: %.4f" %
(test_acc_only_in_training))
print(
" acc on out of training test, WITHOUT training docs in graph: %.4f"
% (test_acc_only_out_training))
print(
" acc on out of training test, with training docs in graph: %.4f" %
(test_acc_train_and_outtest))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
time_since_best = 0
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
print("\n[RESULT!] Final test score: see above")
break
from options.test_options import TestOptions
from data.data_loader import CreateDataLoader
from models.create_model import create_model
from utils import visualizer
opt = TestOptions().parse()
### set dataloader ###
print('### prepare DataLoader ###')
data_loader = CreateDataLoader(opt)
test_loader = data_loader.load_data()
print('test images : {}'.format(len(data_loader)))
print('numof_iteration : {}'.format(len(test_loader)))
### define model ###
model = create_model(opt)
model.gen.eval()
print('### start test ! ###')
for iter, data in enumerate(test_loader):
model.set_variables(data)
fake_image = model.forward()
fake_image = (fake_image + 1.0) / 2.0
visualizer.save_test_images(opt, iter, data['label'], fake_image, nrow=7)
def main():
update_config(quick_config)
docs, labels, tvt_idx = load_data()
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
trainer = Trainer(dataset, model)
# trainer.save_initial_reps()
best_val_loss = float('inf')
time_since_best = 0
best_val_loss_acc = 0
high_score = 0
for i in range(config["epochs"]):
# split for special debug printing
if config["sampled_training"] and config["unsupervised_loss"]:
# trainer.save_sage_reps()
train_loss, val_loss, unsup_train_loss_pos, unsup_train_loss_neg, unsup_val_loss_pos, unsup_val_loss_neg, unsup_test_pos, unsup_test_neg = trainer.train_epoch(
)
test_acc, test_loss, unsup_test_loss_pos, unsup_test_loss_neg = trainer.test(
)
total_train = train_loss + unsup_train_loss_pos + unsup_train_loss_neg
total_val = val_loss + unsup_val_loss_pos + unsup_val_loss_neg
total_test = test_loss + unsup_test_loss_pos + unsup_test_loss_neg
print("[epoch %02d] Test Acc %.4f (Trained %s-supervised)" %
(i, test_acc, config['sup_mode']))
print("\t Train Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" %
(total_train, train_loss, unsup_train_loss_pos,
unsup_train_loss_neg, train_loss / total_train * 100))
print("\t Val Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" %
(total_val, val_loss, unsup_val_loss_pos, unsup_val_loss_neg,
val_loss / total_val * 100))
print("\t Training on test Losses: %.4f, %.4f, %.1f%% of total" %
(unsup_test_pos, unsup_test_neg,
(unsup_test_pos + unsup_test_neg) /
(unsup_test_pos + unsup_test_neg + total_train) * 100))
# print("\t Test Loss: %.4f (%.4f / %.4f / %.4f) (%.0f%% sup)" % (total_test, test_loss, unsup_test_loss_pos, unsup_test_loss_neg, test_loss / total_test))
val_loss = total_val
else:
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
high_score = max(test_acc, high_score)
print(
"[epoch %02d] Train loss %.4f, Val loss %.4f, Test Acc %.4f, Highscore: %.4f"
% (i, train_loss, val_loss, test_acc, high_score))
# Early stopping
time_since_best += 1
if val_loss < best_val_loss:
best_val_loss = val_loss
time_since_best = 0
best_val_loss_acc = test_acc
if config["terminate_early"] and time_since_best >= config[
"terminate_patience"]:
if config["sampled_training"] and config["unsupervised_loss"]:
test_acc, test_loss, unsup_test_loss_pos, unsup_test_loss_neg = trainer.test(
)
print("\n[RESULT!] Final test score: ", best_val_loss_acc)
else:
test_acc = trainer.test()
break
print("\n[RESULT!] Final test score: ", best_val_loss_acc)
def main():
# load data
docs, labels, tvt_idx = load_data()
train_idx, val_idx, test_idx = tvt_idx
model = None
results = {
"epochs": [],
"average_train_loss": [],
"average_val_loss": [],
"average_test_acc": [],
"split_amount": [],
"acc": []
}
# create model with vocab for entire dataset
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
model = create_model(dataset)
trainer = Trainer(dataset, model)
for i in range(config["epochs"]):
number_of_splits = random.randint(1, 40)
# split in x random divisions
split_train_amount = len(train_idx) // number_of_splits
split_val_amount = len(val_idx) // number_of_splits
split_test_amount = len(test_idx) // number_of_splits
print("Splitting %i segments, %i %i %i" %
(number_of_splits, split_train_amount, split_val_amount,
split_test_amount))
# train on different random splits
random.shuffle(train_idx)
random.shuffle(val_idx)
random.shuffle(test_idx)
train_losses = []
val_losses = []
test_accs = []
for split_i in range(number_of_splits):
split_train_idx = train_idx[split_i *
split_train_amount:(split_i + 1) *
split_train_amount]
split_val_idx = val_idx[split_i * split_val_amount:(split_i + 1) *
split_val_amount]
split_test_idx = test_idx[split_i *
split_test_amount:(split_i + 1) *
split_test_amount]
# this ain't right
dataset = DocumentGraphDataset(
docs, labels, (split_train_idx, split_val_idx, split_test_idx))
if model is None:
model = create_model(dataset)
trainer = Trainer(dataset, model)
else:
trainer.update_data(dataset)
train_loss, val_loss = trainer.train_epoch()
test_acc = trainer.test()
print("split %02d: (%.4f, %.4f, ! %.4f !), " %
(split_i, train_loss, val_loss, test_acc),
end="")
train_losses.append(train_loss)
val_losses.append(val_loss)
test_accs.append(test_acc)
# test on entire graph
dataset = DocumentGraphDataset(docs, labels, tvt_idx)
trainer.update_data(dataset)
test_acc = trainer.test()
print("\n\n[epoch %02d] Test Acc on entire dataset %.4f\n\n" %
(i, test_acc))
# summary
results["epochs"].append(i)
results["average_train_loss"].append(
float(sum(train_losses) / len(train_losses)))
results["average_val_loss"].append(
float(sum(val_losses) / len(val_losses)))
results["average_test_acc"].append(
float(sum(test_accs) / len(test_accs)))
results["split_amount"].append(number_of_splits)
results["acc"].append(test_acc)
df = pd.DataFrame(results)
df.to_csv('./results/' + config['experiment_name'] + '.csv')