def main(args): world = worlds.get_world(args.world) agent = ActiveLearningAgent(world, None, faiss_nprobe=10) if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) if args.world == 'inat' or args.world == 'celeba': # images agent.trained_model = ImageEmbedder( args.world, None, loss_type=args.loss_type, distance_metric=args.distance_metric, logit_scale=args.logit_scale) else: agent.trained_model = SentenceEmbedder( None, min_epochs=args.min_epochs, batch_size=args.batch_size, pooling_mode=args.pooling_mode, loss_type=args.loss_type, distance_metric=args.distance_metric, logit_scale=args.logit_scale, bias_scale=args.bias_scale) agent.trained_model.load(args.load_dir) if args.paws_qqp or args.evaluation_file: if args.paws_qqp: dataset = util.load_paws_qqp_data() else: dataset = util.load_custom_data(world, args.evaluation_file) preds = agent.trained_model.predict(dataset) # Returns logits hard_preds = {k: int(v > 0) for k, v in preds.items()} results = evaluation.evaluate_accuracy(dataset, hard_preds) print(json.dumps(results)) if args.paws_qqp: # Evaluate average precision y_true = [ex.label for ex in dataset] y_score = [preds[tuple(ex.sent_ids)] for ex in dataset] print('Average Precision: {}'.format( average_precision_score(y_true, y_score))) else: agent.Evaluate(args.output_dir, args.distance_metric, test=args.test)
print("Initializing VIN...") network = algorithms.RMSProp( create_VIN( env['input_image_shape'], n_hidden_filters=150, n_state_filters=10, k=env['k'], ), verbose=True, error=loss_function, epoch_end_signal=on_epoch_end_from_steps(env['steps']), **env['training_options'] ) print("Training VIN...") network.train( (x_train, s1_train, s2_train), y_train, (x_test, s1_test, s2_test), y_test, epochs=env['epochs'], ) if not os.path.exists(MODELS_DIR): os.mkdir(MODELS_DIR) print("Saving pre-trained VIN model...") storage.save(network, env['pretrained_network_file']) print("Evaluating accuracy on test set...") evaluate_accuracy(network.predict, x_test, s1_test, s2_test)
x_train, s1_train, s2_train, y_train = load_data(env['train_data_file']) x_test, s1_test, s2_test, y_test = load_data(env['test_data_file']) network = algorithms.RMSProp( create_VIN( env['input_image_shape'], n_hidden_filters=150, n_state_filters=10, k=env['k'], ), step=0.01, verbose=True, batch_size=12, error=loss_function, epoch_end_signal=on_epoch_end, decay=0.9, epsilon=1e-6, ) network.train((x_train, s1_train, s2_train), y_train, (x_test, s1_test, s2_test), y_test, epochs=120) if not os.path.exists(MODELS_DIR): os.mkdir(MODELS_DIR) storage.save(network, env['pretrained_network_file']) evaluate_accuracy(network.connection.compile(), x_test, s1_test, s2_test)
loss.backward() if iternum % args.gradient_accumulation_steps == 0: optimizer.step() scheduler.step() # Update learning rate schedule # scheduler.step() # Update learning rate schedule optimizer.zero_grad() # optimizer.zero_grad() # optimizer.step() writer.add_scalar("loss", loss.item(), iternum) writer.add_scalar("lr", scheduler.get_lr()[0], iternum) if np.mod(iternum, args.val_freq) == 0 and iternum > 0: acc, scores_test, labels_test = evaluate_accuracy( model, val_loader, device ) tqdm.write(f"Accuracy at {iternum} is {acc:.2f}") writer.add_scalar("acc", acc, iternum) model.train() result_filename = os.path.join(result_dir, f"{iternum}.npy") np.save(result_filename, acc) if wandb is not None: wandb.run.summary["best_accuracy"] = best_perf wandb.run.summary["best_iter"] = best_iter # Save For Debug Info """ f = open(result_filename.replace("npy", "json"), "w") prediction = scores_test.argmax(-1) sep_token_id = text_encoder.sep_token_id