def main(args):
world = worlds.get_world(args.world)
agent = ActiveLearningAgent(world, None, faiss_nprobe=10)
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
if args.world == 'inat' or args.world == 'celeba': # images
agent.trained_model = ImageEmbedder(
args.world,
None,
loss_type=args.loss_type,
distance_metric=args.distance_metric,
logit_scale=args.logit_scale)
else:
agent.trained_model = SentenceEmbedder(
None,
min_epochs=args.min_epochs,
batch_size=args.batch_size,
pooling_mode=args.pooling_mode,
loss_type=args.loss_type,
distance_metric=args.distance_metric,
logit_scale=args.logit_scale,
bias_scale=args.bias_scale)
agent.trained_model.load(args.load_dir)
if args.paws_qqp or args.evaluation_file:
if args.paws_qqp:
dataset = util.load_paws_qqp_data()
else:
dataset = util.load_custom_data(world, args.evaluation_file)
preds = agent.trained_model.predict(dataset) # Returns logits
hard_preds = {k: int(v > 0) for k, v in preds.items()}
results = evaluation.evaluate_accuracy(dataset, hard_preds)
print(json.dumps(results))
if args.paws_qqp:
# Evaluate average precision
y_true = [ex.label for ex in dataset]
y_score = [preds[tuple(ex.sent_ids)] for ex in dataset]
print('Average Precision: {}'.format(
average_precision_score(y_true, y_score)))
else:
agent.Evaluate(args.output_dir, args.distance_metric, test=args.test)
print("Initializing VIN...")
network = algorithms.RMSProp(
create_VIN(
env['input_image_shape'],
n_hidden_filters=150,
n_state_filters=10,
k=env['k'],
),
verbose=True,
error=loss_function,
epoch_end_signal=on_epoch_end_from_steps(env['steps']),
**env['training_options']
)
print("Training VIN...")
network.train(
(x_train, s1_train, s2_train), y_train,
(x_test, s1_test, s2_test), y_test,
epochs=env['epochs'],
)
if not os.path.exists(MODELS_DIR):
os.mkdir(MODELS_DIR)
print("Saving pre-trained VIN model...")
storage.save(network, env['pretrained_network_file'])
print("Evaluating accuracy on test set...")
evaluate_accuracy(network.predict, x_test, s1_test, s2_test)
x_train, s1_train, s2_train, y_train = load_data(env['train_data_file'])
x_test, s1_test, s2_test, y_test = load_data(env['test_data_file'])
network = algorithms.RMSProp(
create_VIN(
env['input_image_shape'],
n_hidden_filters=150,
n_state_filters=10,
k=env['k'],
),
step=0.01,
verbose=True,
batch_size=12,
error=loss_function,
epoch_end_signal=on_epoch_end,
decay=0.9,
epsilon=1e-6,
)
network.train((x_train, s1_train, s2_train), y_train,
(x_test, s1_test, s2_test), y_test,
epochs=120)
if not os.path.exists(MODELS_DIR):
os.mkdir(MODELS_DIR)
storage.save(network, env['pretrained_network_file'])
evaluate_accuracy(network.connection.compile(), x_test, s1_test, s2_test)
loss.backward()
if iternum % args.gradient_accumulation_steps == 0:
optimizer.step()
scheduler.step() # Update learning rate schedule
# scheduler.step() # Update learning rate schedule
optimizer.zero_grad()
# optimizer.zero_grad()
# optimizer.step()
writer.add_scalar("loss", loss.item(), iternum)
writer.add_scalar("lr", scheduler.get_lr()[0], iternum)
if np.mod(iternum, args.val_freq) == 0 and iternum > 0:
acc, scores_test, labels_test = evaluate_accuracy(
model, val_loader, device
)
tqdm.write(f"Accuracy at {iternum} is {acc:.2f}")
writer.add_scalar("acc", acc, iternum)
model.train()
result_filename = os.path.join(result_dir, f"{iternum}.npy")
np.save(result_filename, acc)
if wandb is not None:
wandb.run.summary["best_accuracy"] = best_perf
wandb.run.summary["best_iter"] = best_iter
# Save For Debug Info
"""
f = open(result_filename.replace("npy", "json"), "w")
prediction = scores_test.argmax(-1)
sep_token_id = text_encoder.sep_token_id