def main(): # Example options: # train ./Configs/Base.yaml # test ./Configs/Base.yaml ap = argparse.ArgumentParser("Progressive Transformers") # Choose between Train and Test ap.add_argument("mode", choices=["train", "test"], help="train a model or test") # Path to Config ap.add_argument("config_path", type=str, help="path to YAML config file") args = ap.parse_args() # If Train if args.mode == "train": train(cfg_file=args.config_path) # If Test elif args.mode == "test": test(cfg_file=args.config_path, ckpt=args.ckpt) else: raise ValueError("Unknown mode")
def main(): # prepare dataset train_adversarial = 1 use_cuda = True epochs = 2000 lr = 0.0005 train_set, test_set = get_datasets(balance_train=True) num_features = train_set[0][0].shape[0] batch_size = 400 test_batch_size = 50 torch.manual_seed(7347) device = 'cuda' if torch.cuda.is_available() else 'cpu' print('using device {0}'.format(device)) train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, shuffle=True) model = Net(activation=nn.LeakyReLU(), num_features=num_features, embed_size=80).to(device) PATH = None # model.load_state_dict(torch.load(PATH, map_location=device), strict=False) reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr) discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1) encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1) if train_adversarial: compute_loss = compute_loss_adversarial_enc optimizer = {'rec': reconstruction_optimizer, 'dis': discriminative_optimizer, 'enc': encoder_optimizer} tmp = [reconstruction_optimizer, discriminative_optimizer, encoder_optimizer] schedulers = [StepLR(x, step_size=70, gamma=0.9) for x in tmp] else: compute_loss = compute_loss_autoenc optimizer = {'rec': reconstruction_optimizer} schedulers = [StepLR(reconstruction_optimizer, step_size=50, gamma=0.9)] for epoch in range(1, epochs + 1): if epoch % 50 == 0: test(model, compute_loss, device, test_loader) train(model, compute_loss, device, train_loader, optimizer, epoch) for scheduler in schedulers: scheduler.step() if epoch % 100 == 0 and epoch: torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch)) print('learning rate: {0}'.format(scheduler.get_lr()))
def main(): embed_size = 80 use_embed = True use_cuda = True epochs = 2000 lr = 0.0001 train_set, test_set = get_datasets(label_columns=['posOutcome']) # process with feature extractor # create new test and train sets num_features = train_set[0][0].shape[0] batch_size = 100 test_batch_size = 50 torch.manual_seed(7347) device = 'cuda' if torch.cuda.is_available() else 'cpu' print('using device {0}'.format(device)) train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) test_loader = torch.utils.data.DataLoader(test_set, batch_size=test_batch_size, shuffle=True) extractor = Net(activation=nn.LeakyReLU(), num_features=num_features, embed_size=embed_size).to(device) extractor_path = 'extractor.pt' extractor.load_state_dict(torch.load(extractor_path, map_location=device), strict=False) model = ClassifierNet( num_features=embed_size if use_embed else num_features, activation=nn.LeakyReLU()).to(device) classifier_optimizer = optim.AdamW(model.parameters(), lr=lr) compute_loss = compute_classifier_loss optimizer = {'opt': classifier_optimizer} schedulers = [StepLR(classifier_optimizer, step_size=50, gamma=0.9)] callback = compute_classifier_loss if use_embed: callback = functools.partial(loss_callback, extractor=extractor) for epoch in range(1, epochs + 1): if epoch % 50 == 0 or epoch == 1: test(model, callback, device, test_loader) train(model, callback, device, train_loader, optimizer, epoch) for scheduler in schedulers: scheduler.step() if epoch % 100 == 0 and epoch: torch.save(model.state_dict(), "classifier{0}.pt".format(epoch)) print('learning rate: {0}'.format(scheduler.get_lr()))
def result_for_manifest(model, criterion, manifest, decoder, target_decoder, batch_size, num_workers): ### LOADER test_dataset = MultiDataset(manifest, model._meta['labels'], use_mfcc_in=model._meta['use_mfcc_in'], use_ivectors_in=model._meta['use_ivectors_in'], use_embeddings_in=model._meta['use_embeddings_in'], embedding_size=model._meta['embedding_size'], use_transcripts_out=model._meta['use_transcripts_out'], use_accents_out=model._meta['use_accents_out']) test_loader = MultiDataLoader(test_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) ### TEST test_results = test(model, test_loader, criterion, decoder, target_decoder) test_loss, test_loss_text, test_loss_accent, test_wer, test_accent_acc = test_results results_dict = {} if test_wer != -1: results_dict['WER'] = test_wer if test_accent_acc != -1: results_dict['Accent accuracy'] = test_accent_acc return results_dict
def main(): global args args = parser.parse_args() print() print('Command-line argument values:') for key, value in vars(args).items(): print('-', key, ':', value) print() test_params = [ args.model, path_to_save_string(args.dataset), path_to_save_string(args.test_dataset), args.viewpoint_modulo, args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed, args.routing_iters ] test_name = '_'.join([str(x) for x in test_params]) + '.pth' model_params = [ args.model, path_to_save_string(args.dataset), args.viewpoint_modulo, args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed, args.routing_iters ] model_name = '_'.join([str(x) for x in model_params]) + '.pth' header = 'model,training-dataset,test-dataset,viewpoint_modulo,' \ 'batch_size,epochs,lr,weight_decay,seed,em_iters,accuracy' snapshot_path = os.path.join('.', 'snapshots', model_name) result_path = os.path.join('.', 'results', 'pytorch_test.csv') make_dirs_if_not_exist([snapshot_path, result_path]) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) model, criterion, optimizer, scheduler = load_model( args.model, device_ids=args.device_ids, lr=args.lr, routing_iters=args.routing_iters) num_class, train_loader, test_loader = load_datasets( args.test_dataset, args.batch_size, args.test_batch_size, args.test_viewpoint_modulo) model.load_state_dict(torch.load(snapshot_path)) acc, predictions, labels, logits = test(test_loader, model, criterion, chunk=1) print(f'Accuracy: {acc:.2f}%') print(f'Memory usage: {gpu_memory_usage()}') to_write = test_params + [acc.cpu().numpy()] append_to_csv(result_path, to_write, header=header) if args.roc != '': make_dirs_if_not_exist(args.roc) torch.save((predictions, labels, logits), args.roc)
def test_train_bilm_chars(self): vocab, data, options = self._get_vocab_data_options(True, True) train(options, data, 1, self.tmp_dir, self.tmp_dir) # now test tf.reset_default_graph() options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir) data_test, vocab_test = self._get_data(True, True, True) perplexity = test(options, ckpt_file, data_test, batch_size=1) self.assertTrue(perplexity < 20.0)
def main(args): options, ckpt_file = load_options_latest_checkpoint(args.save_dir) # load the vocab if 'char_cnn' in options: max_word_length = options['char_cnn']['max_characters_per_token'] else: max_word_length = None vocab = load_vocab(args.vocab_file, max_word_length) test_prefix = args.test_prefix kwargs = { 'test': True, 'shuffle_on_load': False, } if options.get('bidirectional'): data = BidirectionalLMDataset(test_prefix, vocab, **kwargs) else: data = LMDataset(test_prefix, vocab, **kwargs) test(options, ckpt_file, data, batch_size=args.batch_size)
def test_train_skip_connections(self): bidirectional = True use_chars = False vocab, data, options = self._get_vocab_data_options( bidirectional, use_chars) options['lstm']['use_skip_connections'] = True train(options, data, 1, self.tmp_dir, self.tmp_dir) # now test tf.reset_default_graph() options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir) data_test, vocab_test = self._get_data( bidirectional, use_chars, test=True) perplexity = test(options, ckpt_file, data_test, batch_size=1) self.assertTrue(perplexity < 20.0)
def test_train_shared_softmax_embedding(self): bidirectional = True use_chars = False vocab, data, options = self._get_vocab_data_options( bidirectional, use_chars, share_embedding_softmax=True) train(options, data, 1, self.tmp_dir, self.tmp_dir) # now test tf.reset_default_graph() options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir) data_test, vocab_test = self._get_data( bidirectional, use_chars, test=True) perplexity = test(options, ckpt_file, data_test, batch_size=1) self.assertTrue(perplexity < 20.0)
def main(): train_adversarial = 1 use_cuda = True epochs = 200 lr = 0.005 batch_size = 100 test_batch_size = 100 torch.manual_seed(7347) device = 'cuda' if torch.cuda.is_available() else 'cpu' print('using device {0}'.format(device)) kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), ])), batch_size=batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, transform=transforms.Compose([ transforms.ToTensor(), ])), batch_size=test_batch_size, shuffle=True, **kwargs) model = Net(activation=nn.LeakyReLU()).to(device) PATH = 'mnist_cnn100.pt.bak' # model.load_state_dict(torch.load(PATH, map_location=device), strict=False) reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr) discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1) encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1) if train_adversarial: compute_loss = compute_loss_adversarial_enc optimizer = { 'rec': reconstruction_optimizer, 'dis': discriminative_optimizer, 'enc': encoder_optimizer } tmp = [ reconstruction_optimizer, discriminative_optimizer, encoder_optimizer ] schedulers = [StepLR(x, step_size=5, gamma=0.9) for x in tmp] else: compute_loss = compute_loss_autoenc optimizer = {'rec': reconstruction_optimizer} schedulers = [StepLR(reconstruction_optimizer, step_size=5, gamma=0.9)] for epoch in range(1, epochs + 1): if epoch % 5 == 0: test(model, compute_loss, device, test_loader) train(model, compute_loss, device, train_loader, optimizer, epoch) for scheduler in schedulers: scheduler.step() if epoch % 10 == 0 and epoch: torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch)) print('learning rate: {0}'.format(scheduler.get_lr()))
exit(0) train_scores = np.zeros(args.epochs) validation_scores = np.zeros(args.epochs) with torch.cuda.device(args.device): try: t = trange(args.start_epoch, args.epochs) for epoch in t: if args.train: train_scores[epoch] = train(train_loader, model, optimizer, args) else: train_scores[epoch] = 0 if args.test: validation_scores[epoch] = test(test_loader, model, args) else: validation_scores[epoch] = 0 results(test_loader.dataset, model, vis, epoch + 1) save_checkpoint( { 'args': args, 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'optimizer': optimizer.state_dict(), 'losses': losses, }, filename=args.checkpoint)
for i in range(epoch): scheduler.step() for i in range(epoch, 400): epoch += 1 scheduler.step() train_loader = get_split_dataloader(f'{args.dataset}/train_k{i % 99}.csv', width=width, batch_size=batch_size, transform=transform, num_workers=num_workers) train(model, train_loader, optimizer=optimizer, epoch=epoch, half=args.half) if epoch % evaluate_interval == 0: mean_loss, mean_map, t = test(model, valid_loader, half=args.half) if hvd.rank() == 0: checkpoint_path = save_checkpoint(model, optimizer, test_acc=mean_map, tag=epoch) mean_loss, mean_map, t = test(model, valid_loader, half=args.half) if hvd.rank() == 0: checkpoint_path = save_checkpoint(model, optimizer, test_acc=mean_map, tag=epoch)
verbose=False, version_tag="RUN_TO_GET_UNNORMALIZED_INFO", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='multi_vae', data_subdir=data_subdir, n_epochs_pred_only=N_EPOCHS_PRED_ONLY, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, max_kl=0.2, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=0.01, evaluation_metric="DCG", batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, version_tag="RUN_TO_GET_UNNORMALIZED_INFO", ) print("On to round 2! Now we'll do the critic.") train( model_class='multi_vae', data_subdir=data_subdir, n_epochs_pred_only=0, n_epochs_ac_only=N_EPOCHS_AC_ONLY,
positive_weights=5.0, version_tag="VWMF_JUST_ACTOR", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='variational_wmf', # data_subdir=data_subdir, n_epochs_pred_only=200, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, max_kl=0.05, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=0.0001, evaluation_metric="NDCG", batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, positive_weights=5.0, version_tag="VWMF_JUST_ACTOR", ) print("On to round 2! Now we'll do the critic.") train( model_class='variational_wmf', # data_subdir=data_subdir, n_epochs_pred_only=0, n_epochs_ac_only=50,
break_early=BREAK_EARLY, verbose=False, version_tag="ProperlyShapedMultiDAE_ONLY_ACTOR", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='new_multi_dae', n_epochs_pred_only=200, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, max_kl=0.2, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=1e-05, evaluation_metric="NDCG", batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, version_tag="ProperlyShapedMultiDAE_ONLY_ACTOR", ) print("On to round 2! Now we'll do the critic.") train( model_class='new_multi_dae', n_epochs_pred_only=0, n_epochs_ac_only=50, n_epochs_pred_and_ac=50, max_kl=0.2,
0.8, 'addit', test=True ) #both work also, because of test=true, the mode option is useless here use_gpu = True n_epoch = 100 learning_rate = 0.002 history = train(model, train_loader, valid_loader, n_epoch, learning_rate, use_gpu=use_gpu) history.display() test_acc, test_loss = test(model, test_loader, use_gpu=use_gpu) print('Test:\n\tLoss: {}\n\tAccuracy: {}'.format(test_loss, test_acc)) #Train with Input Dropout in addit mode model = Net() model.type(torch.cuda.FloatTensor) train_loader, valid_loader, test_loader = pixar_dataset(0.8, 'addit', test=False) use_gpu = True n_epoch = 100 learning_rate = 0.002 history = train(model, train_loader,
def anomaly(experiment_name, dataset = "mnist", bayesian_approximation = "dropout", inside_labels = [0, 1], num_epochs = 50, batch_size = 128, acc_threshold = 0.6, weight_decay = 1e-5, dropout_p = 0.5, fc_layers = [512, 512], plot = True): """ This methods trains a neural network classifier on a subset of classes. After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies. The anomalous classes are the ones that are not part of the training subset. dataset = "mnist" or "cifar" For MNIST we use a fully-connected MLP. For CIFAR10 we use a convolutional net (similar to LeNet) bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST. inside_labels are the subset of trained classes, the other classes are only used for testing. """ n_out = len(inside_labels) # Prepare Theano variables for inputs and targets # Load the dataset print("Loading data...") if dataset == "mnist": input_var = T.matrix('inputs') target_var = T.ivector('targets') n_in = [28*28] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels) if bayesian_approximation == "dropout": model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay) elif bayesian_approximation == "variational": model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size)) elif dataset == "cifar": input_var = T.tensor4('inputs') target_var = T.ivector('targets') n_in = [3, 32, 32] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10(inside_labels) model = models.convnet_dropout(input_var, target_var, n_in, n_out, dropout_p, weight_decay) df = pd.DataFrame() # Mini-batch training with ADAM epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold) # Mini-batch testing acc, bayes_acc = training.test(model, X_test, y_test, batch_size) df.set_value(experiment_name, "test_acc", acc) df.set_value(experiment_name, "bayes_test_acc", bayes_acc) # Uncertainty prediction test_mean_std_bayesian = {x:[] for x in range(10)} test_mean_std_deterministic = {x:[] for x in range(10)} test_entropy_bayesian = {x:[] for x in range(10)} test_entropy_deterministic = {x:[] for x in range(10)} for i in range(len(X_test_all)): bayesian_probs = model.probabilities(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) bayesian_entropy = model.entropy_bayesian(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) classical_probs = model.probabilities_deterministic(X_test_all[i][np.newaxis,:])[0] classical_entropy = model.entropy_deterministic(X_test_all[i][np.newaxis,:]) predictive_mean = np.mean(bayesian_probs, axis=0) predictive_std = np.std(bayesian_probs, axis=0) test_mean_std_bayesian[y_test_all[i]].append(np.concatenate((predictive_mean, predictive_std))) test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy) test_entropy_deterministic[y_test_all[i]].append(classical_entropy) test_mean_std_deterministic[y_test_all[i]].append(classical_probs) # Plotting if plot: for k in sorted(test_mean_std_bayesian.keys()): sns.plt.figure() #sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k)) sns.plt.hist(test_entropy_bayesian[k], label = "Bayesian Entropy v1 for " + str(k)) #sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k)) #sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k)) sns.plt.legend() sns.plt.show() # Anomaly detection using simple threshold def anomaly_detection_old(anomaly_score_dict, name, df): threshold = np.logspace(-30, 1.0, 1000) acc = {} for t in threshold: tp = 0.0 tn = 0.0 for l in anomaly_score_dict: if l in inside_labels: tp += (np.array(anomaly_score_dict[l]) < t).mean() else: tn += (np.array(anomaly_score_dict[l]) >= t).mean() tp /= len(inside_labels) tn /= 10.0 - len(inside_labels) bal_acc = (tp + tn)/2.0 f1_score = 2.0*tp/(2.0 + tp - tn) acc[t] = [bal_acc, f1_score, tp, tn] print("{}\tscore\tthreshold\tTP\tTN".format(name)) sorted_acc = sorted(acc.items(), key= lambda x : x[1][0], reverse = True) df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0]) df.set_value(experiment_name, name + ' bal_acc_threshold', sorted_acc[0][0]) print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) sorted_acc = sorted(acc.items(), key= lambda x : x[1][1], reverse = True) df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1]) df.set_value(experiment_name, name + ' f1_score_threshold', sorted_acc[0][0]) print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) return df # Anomaly detection using logistic regression def anomaly_detection(anomaly_score_dict, name, df): X = [] y = [] for l in anomaly_score_dict: X += anomaly_score_dict[l] if l in inside_labels: y += [0]*len(anomaly_score_dict[l]) else: y += [1]*len(anomaly_score_dict[l]) X = np.array(X) y = np.array(y) X, y = utils.shuffle(X, y, random_state=0) X_train = X[:len(X)/2] X_test = X[len(X)/2:] y_train = y[:len(y)/2] y_test = y[len(y)/2:] clf = linear_model.LogisticRegression(C=1.0) clf.fit(X_train, y_train) auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1]) print("AUC", auc) df.set_value(experiment_name, name + ' AUC', auc) if plot: # Plot ROC curve fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1], pos_label=1) sns.plt.figure() sns.plt.plot(fpr, tpr, label='ROC curve') sns.plt.plot([0, 1], [0, 1], 'k--') sns.plt.xlim([0.0, 1.0]) sns.plt.ylim([0.0, 1.05]) sns.plt.xlabel('False Positive Rate') sns.plt.ylabel('True Positive Rate') sns.plt.title('Receiver operating characteristic example') sns.plt.legend(loc="lower right") sns.plt.show() return df df.set_value(experiment_name, 'dataset', dataset) df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation) df.set_value(experiment_name, 'inside_labels', str(inside_labels)) df.set_value(experiment_name, 'epochs', epochs) df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df) df = anomaly_detection(test_mean_std_deterministic, "Classical prediction", df) df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df) df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df) return df
# 'logging_frequency': 50, # 'logging_frequency': 50, # 'batch_size': 500, # 'batch_size': 5, # 'break_early': True, # 'break_early': False, 'verbose': True, # 'path_to_save_actor': "best_ndcg_trained_150_epochs", # 'path_to_save_last_actor': "last_actor_after_150_trained_epochs", 'version_tag': "PHASE_4_LAMBDARANK", 'path_to_save_actor': "MSD_BEST_ACTOR_FROM_LAMBDARANK_PHASE_4", 'restore_trained_actor_path': "MSD_BEST_ACTOR_FROM_WARP_CRITIC_ONLY_PHASE_1_FULL" } # jUST PHASE 4 THIS TIME FIRST_TRAIN_KWARGS = dict(DEFAULT_KWARGS) # NOW FOR TESTING FIRST_TEST_KWARGS = dict(DEFAULT_KWARGS) del FIRST_TEST_KWARGS['logging_frequency'] del FIRST_TEST_KWARGS['path_to_save_actor'] train(**FIRST_TRAIN_KWARGS) tf.reset_default_graph() test(**FIRST_TEST_KWARGS) exit()
verbose=False, version_tag="FULL_RUN_ON_MSD_ONLY", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='multi_vae', data_subdir=data_subdir, n_epochs_pred_only=100, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, # max_kl=0.2, max_kl=0.1, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=0.01, evaluation_metric="NDCG", batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, version_tag="FULL_RUN_ON_MSD_ONLY", ) print("On to round 2! Now we'll do the critic.") train( model_class='multi_vae', data_subdir=data_subdir, n_epochs_pred_only=0, n_epochs_ac_only=50,
def main(): global args args = parser.parse_args() print() print('Command-line argument values:') for key, value in vars(args).items(): print('-', key, ':', value) print() params = [ args.model, path_to_save_string(args.dataset), args.viewpoint_modulo, args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed, args.routing_iters ] model_name = '_'.join([str(x) for x in params]) + '.pth' header = 'model,dataset,viewpoint_modulo,batch_size,epochs,lr,weight_decay,seed,em_iters,accuracy' snapshot_path = os.path.join('.', 'snapshots', model_name) data_path = os.path.join('.', 'results', 'training_data', model_name) result_path = os.path.join('.', 'results', 'pytorch_train.csv') make_dirs_if_not_exist([snapshot_path, data_path, result_path]) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) model, criterion, optimizer, scheduler = load_model( args.model, device_ids=args.device_ids, lr=args.lr, routing_iters=args.routing_iters) num_class, train_loader, test_loader = load_datasets( args.dataset, args.batch_size, args.test_batch_size, args.viewpoint_modulo) best_acc = 0 training_accuracies = [] test_accuracies = [] if args.append: model.load_state_dict(torch.load(snapshot_path)) try: for epoch in range(1, args.epochs + 1): print() acc = train(train_loader, model, criterion, optimizer, epoch, epochs=args.epochs, log_interval=args.log_interval) training_accuracies.append(acc) scheduler.step(acc) print('Epoch accuracy was %.1f%%. Learning rate is %.9f.' % (acc, optimizer.state_dict()['param_groups'][0]['lr'])) if epoch % args.test_interval == 0: test_acc, __, __, __ = test(test_loader, model, criterion, chunk=args.test_size) test_accuracies.append(test_acc) if test_acc > best_acc: best_acc = test_acc except KeyboardInterrupt: print('Cancelled training after %d epochs' % (epoch - 1)) args.epochs = epoch - 1 acc, predictions, labels, logits = test(test_loader, model, criterion, chunk=1) print(f'Accuracy: {acc:.2f}% (best: {best_acc:.2f}%)') to_write = params + [acc.cpu().numpy()] append_to_csv(result_path, to_write, header=header) snapshot(snapshot_path, model) #torch.save((accuracies, labels, predictions), data_path) if args.learn_curve != '': make_dirs_if_not_exist(args.learn_curve) torch.save((training_accuracies, test_accuracies), args.learn_curve)
def main(): num_gpus = torch.cuda.device_count() dist.init_process_group(backend=args.backend, init_method="env://") # Hyperparameters hparams = { "n_cnn_layers": 3, "n_rnn_layers": 5, "rnn_dim": 512, "n_class": 29, "n_feats": 128, "stride": 2, "dropout": 0.1, "learning_rate": 5e-4, "batch_size": 10, "epochs": 20 } # Load train and test data train_dataset = torchaudio.datasets.LIBRISPEECH("./data", url=args.train_url, download=False) train_sampler = DistributedSampler(dataset=train_dataset) train_loader = data.DataLoader( dataset=train_dataset, batch_size=hparams['batch_size'], sampler=train_sampler, collate_fn=lambda x: data_processing(x, 'train'), num_workers=num_gpus, pin_memory=True) test_dataset = torchaudio.datasets.LIBRISPEECH("./data", url="test-clean", download=False) test_loader = data.DataLoader( dataset=test_dataset, batch_size=hparams['batch_size'], shuffle=False, collate_fn=lambda x: data_processing(x, 'valid'), num_workers=num_gpus, pin_memory=True) # Set seed for reproducibility across all GPUs torch.manual_seed(args.seed) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False torch.backends.cudnn.enabled = False # Instantiate model if args.model == "baseline": model = DeepSpeech2(hparams['n_cnn_layers'], hparams['n_rnn_layers'], hparams['rnn_dim'], hparams['n_class'], hparams['n_feats'], hparams['stride'], hparams['dropout']) elif args.model == "improved": model = ImprovedDeepSpeech2(hparams['n_cnn_layers'], hparams['n_rnn_layers'], hparams['rnn_dim'], hparams['n_class'], hparams['n_feats'], hparams['stride'], hparams['dropout']) device = torch.device('cuda:{}'.format(args.local_rank)) model.to(device) model = DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank) optimizer = optim.AdamW(model.parameters(), hparams['learning_rate']) criterion = nn.CTCLoss(blank=28) scheduler = optim.lr_scheduler.OneCycleLR(optimizer, max_lr=hparams['learning_rate'], steps_per_epoch=int( len(train_loader)), epochs=hparams['epochs'], anneal_strategy='linear') results = [] start = time.perf_counter() for epoch in range(1, hparams['epochs'] + 1): print(f"Local Rank: {args.local_rank}, Epoch: {epoch}, Training...") train(model, device, train_loader, criterion, optimizer, scheduler, epoch) if args.local_rank == 0: avg_cer, avg_wer = test(model, device, test_loader, criterion, epoch) print( f"Epoch: {epoch:2} Average CER: {avg_cer:.4f}, Average WER: {avg_wer:.4f}\n" ) results.append([avg_cer, avg_wer]) end = time.perf_counter() print(f"Execution time: {end - start:0.2f} seconds") df = pd.DataFrame(results, columns=['avg_cer', 'avg_wer']) df.to_csv(args.model + '_' + args.train_url + '.csv', index=False)
n_in = [28*28] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels) # Load model if bayesian_approximation = 'dropout': model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay) else bayesian_approximation = "variational": model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size)) pd = pd.DataFrame() # mini-batch training epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold) # mini-batch testing acc, bayes_acc = training.test(model, X_test, y_test, batch_size) df.set_value(experiment_name, 'test_acc', acc) df.set_value(experiment_name, 'bayes_test_acc', bayes_acc) # uncertainty prediction test_mean_std_bayesian = {x:[] for x in range(10)} test_mean_std_deterministic = {x:[] for x in range(10)} test_entropy_bayesian = {x:[] for x in range(10)} test_entropy_deterministic = {x:[] for x in range(10)} for i in range(len(X_test_all)): bayesian_probs = model.probabilities(np.title(X_test_all[i], batch_size).reshape([-1]+n_in)) bayesian_entropy = model.entropy_bayesian(np.title(X_terst_all[i], batch_size).reshape([-1]+n_in)) predictve_mean = np.mean(bayesian_probs, axis=0) predictve_std = np.std(bayesian_probs, axis=0)
hidden_size=50, embedding_dim=300, dropout=0) net.to(device) if checkpoint: print('==> Resuming from checkpoint..') print(checkpoint) checkpoint = torch.load(checkpoint) net.load_state_dict(checkpoint['net_state_dicts']) torch_rng_state = checkpoint['torch_rng_state'] torch.set_rng_state(torch_rng_state) numpy_rng_state = checkpoint['numpy_rng_state'] np.random.set_state(numpy_rng_state) eval_iter = enumerate(eval_loader) for i in range(100): net.eval() print(training.test(net, eval_iter, device)) batch_size = 100 eval_data = util.SICKData('validation') eval_loader = DataLoader(eval_data, batch_size=batch_size, shuffle=True, drop_last=True) eval_iter = enumerate(eval_loader) step, (cla_label, reg_label, senA, senB) = next(eval_iter) net.eval() output_1hot = net.forward(senA, senB) output = util.onehot2class(output_1hot) (output == cla_label.to(device)).sum().item() / 100
positive_weights=5.0, version_tag="FULL_RUN_ON_OTHER_DATASETS", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='wmf_vae', data_subdir=data_subdir, n_epochs_pred_only=100, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, max_kl=0.05, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=0.0001, evaluation_metric="NDCG", batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, positive_weights=5.0, version_tag="FULL_RUN_ON_OTHER_DATASETS", ) print("On to round 2! Now we'll do the critic.") train( model_class='wmf_vae', data_subdir=data_subdir, n_epochs_pred_only=0, n_epochs_ac_only=50,
print("On to testing.") test( # model_class="wmf", # model_class='multi_vae', model_class='warp_encoder', n_epochs_pred_only=0, n_epochs_ac_only=10, n_epochs_pred_and_ac=10, epochs_to_anneal_over=100, # min_kl=0.0001, max_kl=0.0, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=1e-5, # positive_weights=5, # evaluation_metric='AP', evaluation_metric="NDCG", # logging_frequency=25, # logging_frequency=50, # logging_frequency=50, batch_size=500, # batch_size=25, break_early=False, verbose=False, # path_to_save_actor="best_ndcg_trained_150_epochs", # path_to_save_last_actor="last_actor_after_150_trained_epochs", version_tag="WARP_WITH_CRITIC", # path_to_save_actor="BEST_WARP_RUN_15_EPOCHS_TRUTHFUL_LOSS", restore_trained_actor_path="BEST_WARP_RUN_15_EPOCHS_TRUTHFUL_LOSS") exit()
from tensorflow import keras import data_preparation as dp import training as tr if __name__ == '__main__': model = keras.models.load_model('models/updated_model') df = dp.make_dataframe_and_add_y() print(df.head()) print(df.shape) print("data frame is ready!") scaled_df, scaler = dp.scale_data_for_lstm(df) X_train, y_train, X_test, y_test = dp.create_train_test_sets(scaled_df, 100, 0.8) tr.test(model, scaler, X_test, y_test)
def run_experiment(_exp_name, _epochs, _train_manifest, _test_manifest, _labels, _use_mfcc_in, _use_ivectors_in, _use_embeddings_in, _use_transcripts_out, _use_accents_out, _batch_size, _num_workers, _mfcc_size, _ivector_size, _embedding_size, _rnn_type, _rnn_hidden_size, _nb_head_layers, _nb_speech_layers, _nb_accents_layers, _bidirectional, _losses_mix, _learning_rate, _lm_path, _decoder_alpha, _decoder_beta, _decoder_cutoff_top_n, _decoder_beam_width, _cuda, _tensorboard_path, _saved_models_path, _bottleneck_size, _accent_loss): print(f'\n##### Running experiment {_exp_name} #####') # Tools to log values results_dict = {} results_dict['train_loss'] = [] results_dict['train_loss_text'] = [] results_dict['train_loss_accent'] = [] results_dict['test_loss'] = [] results_dict['test_loss_text'] = [] results_dict['test_loss_accent'] = [] results_dict['test_wer'] = [] results_dict['test_accent_acc'] = [] tb_path = Path(_tensorboard_path) / _exp_name makedirs(tb_path, exist_ok=True) tb_writer = SummaryWriter(tb_path) ### DATA LOADING # Training set train_dataset = MultiDataset(_train_manifest, _labels, use_mfcc_in=_use_mfcc_in, use_ivectors_in=_use_ivectors_in, use_embeddings_in=_use_embeddings_in, embedding_size=_embedding_size, use_transcripts_out=_use_transcripts_out, use_accents_out=_use_accents_out) train_loader = MultiDataLoader(train_dataset, batch_size=_batch_size, shuffle=True, num_workers=_num_workers) # Testing set test_dataset = MultiDataset(_test_manifest, _labels, use_mfcc_in=_use_mfcc_in, use_ivectors_in=_use_ivectors_in, use_embeddings_in=_use_embeddings_in, embedding_size=_embedding_size, use_transcripts_out=_use_transcripts_out, use_accents_out=_use_accents_out) test_loader = MultiDataLoader(test_dataset, batch_size=_batch_size, shuffle=True, num_workers=_num_workers) ### CREATE MODEL model = MultiTask(use_mfcc_in=_use_mfcc_in, use_ivectors_in=_use_ivectors_in, use_embeddings_in=_use_embeddings_in, use_transcripts_out=_use_transcripts_out, use_accents_out=_use_accents_out, mfcc_size=_mfcc_size, ivector_size=_ivector_size, embedding_size=_embedding_size, rnn_type=_rnn_type, labels=_labels, accents_dict=train_dataset.accent_dict, rnn_hidden_size=_rnn_hidden_size, nb_head_layers=_nb_head_layers, nb_speech_layers=_nb_speech_layers, nb_accents_layers=_nb_accents_layers, bidirectional=_bidirectional, bottleneck_size=_bottleneck_size, DEBUG=False) if _cuda: model = model.cuda() print(model, '\n') print('Model parameters counts:', MultiTask.get_param_size(model), '\n') ### OPTIMIZER, CRITERION, DECODER # Optimizer optimizer = torch.optim.Adam(model.parameters(), lr=_learning_rate) # Criterion if _use_accents_out: if _accent_loss == 'focal': AccLoss = FocalLoss() elif _accent_loss == 'CE': AccLoss = nn.CrossEntropyLoss() else: raise ValueError( f'Loss {_accent_loss} for accent_loss is unknown. Please use either "focal" or "CE".' ) if not _use_transcripts_out: # only accent classification criterion = AccLoss elif not _use_accents_out: # only text recognition criterion = nn.CTCLoss() else: # both tasks criterion = (nn.CTCLoss(), FocalLoss()) # Decoder if _use_transcripts_out: decoder = BeamCTCDecoder(_labels, lm_path=_lm_path, alpha=_decoder_alpha, beta=_decoder_beta, cutoff_top_n=_decoder_cutoff_top_n, cutoff_prob=_decoder_cutoff_top_n, beam_width=_decoder_beam_width, num_processes=_num_workers) target_decoder = GreedyDecoder(_labels) else: decoder, target_decoder = None, None ### EPOCHS best_wer = math.inf best_acc = 0 for epoch in range(1, _epochs + 1): ### TRAIN print(f'Epoch {epoch} training: {exp_name}') train_results = train(model, train_loader, criterion, optimizer, losses_mix=_losses_mix) train_loss, train_loss_text, train_loss_accent = train_results results_dict['train_loss'].append(train_loss) results_dict['train_loss_text'].append(train_loss_text) results_dict['train_loss_accent'].append(train_loss_accent) print(f'Epoch {epoch} training loss: {train_loss}') ### TEST print(f'Epoch {epoch} testing') test_results = test(model, test_loader, criterion, decoder, target_decoder, losses_mix=_losses_mix) test_loss, test_loss_text, test_loss_accent, test_wer, test_accent_acc = test_results results_dict['test_loss'].append(test_loss) results_dict['test_loss_text'].append(test_loss_text) results_dict['test_loss_accent'].append(test_loss_accent) results_dict['test_wer'].append(test_wer) results_dict['test_accent_acc'].append(test_accent_acc) print(f'Epoch {epoch} testing loss: {test_loss}') # Add values to tensorboard for key, results in results_dict.items(): tb_writer.add_scalar(key, results[-1], epoch) #Save model if it is best save_new = False if _use_transcripts_out: if test_wer < best_wer: save_new = True best_wer = test_wer else: if test_accent_acc > best_acc: save_new = True best_acc = test_accent_acc if save_new: MultiTask.serialize( model, Path(_saved_models_path) / _exp_name, save=True, exp_name=_exp_name, optimizer=optimizer, epoch=epoch, train_losses=results_dict['train_loss'], test_losses=results_dict['test_loss'], text_train_losses=results_dict['train_loss_text'], text_test_losses=results_dict['test_loss_text'], text_wers=results_dict['test_wer'], accent_train_losses=results_dict['train_loss_accent'], accent_test_losses=results_dict['test_loss_accent'], accent_accuracies=results_dict['test_accent_acc']) del model gc.collect() torch.cuda.empty_cache()
order = parse_order(args.order) if depth and rgb: model = RGBDNet.double_combined18(classes_list.values(), index, order, args.pretrained) else: model = cbnet.combined_net18(classes_list.values(), pre_imagenet=True, pretrained=args.pretrained, order=order) model.set_index(index) else: raise(Exception("Error in parameters. Network should be one between " + str(network_list))) if not TEST: accuracy = training.train(args.network, model, train_loader, test_loader, prefix=prefix, visdom_env=visdom, step=step, batch=batch, epochs=epochs, lr=lr, decay=decay, adamlr=adamlr, momentum=0.9, freeze=args.frozen, cost_function=cost_function, metric=metric) else: accuracy = training.test(model, test_loader, cost_function, metric) print("Saving results in RESULTS.txt") out = open("RESULTS.txt", "a") output = { "date" : str(date.today()), "name" : visdom, "task" : args.task, "net" : args.network, "rgb" : rgb, "depth" : depth, "epochs" : epochs, "adamlr" : adamlr, "lr" : lr, "max_acc": "{:.2f}".format(accuracy[0]),
def anomaly(experiment_name, dataset="mnist", bayesian_approximation="dropout", inside_labels=[0, 1], num_epochs=50, batch_size=128, acc_threshold=0.6, weight_decay=1e-5, dropout_p=0.5, fc_layers=[512, 512], plot=True): """ This methods trains a neural network classifier on a subset of classes. After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies. The anomalous classes are the ones that are not part of the training subset. dataset = "mnist" or "cifar" For MNIST we use a fully-connected MLP. For CIFAR10 we use a convolutional net (similar to LeNet) bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST. inside_labels are the subset of trained classes, the other classes are only used for testing. """ n_out = len(inside_labels) # Prepare Theano variables for inputs and targets # Load the dataset print("Loading data...") if dataset == "mnist": input_var = T.matrix('inputs') target_var = T.ivector('targets') n_in = [28 * 28] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST( inside_labels) if bayesian_approximation == "dropout": model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay) elif bayesian_approximation == "variational": model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train) / float(batch_size)) elif dataset == "cifar": input_var = T.tensor4('inputs') target_var = T.ivector('targets') n_in = [3, 32, 32] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10( inside_labels) model = models.convnet_dropout(input_var, target_var, n_in, n_out, dropout_p, weight_decay) df = pd.DataFrame() # Mini-batch training with ADAM epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold) # Mini-batch testing acc, bayes_acc = training.test(model, X_test, y_test, batch_size) df.set_value(experiment_name, "test_acc", acc) df.set_value(experiment_name, "bayes_test_acc", bayes_acc) # Uncertainty prediction test_mean_std_bayesian = {x: [] for x in range(10)} test_mean_std_deterministic = {x: [] for x in range(10)} test_entropy_bayesian = {x: [] for x in range(10)} test_entropy_deterministic = {x: [] for x in range(10)} for i in range(len(X_test_all)): bayesian_probs = model.probabilities( np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) bayesian_entropy = model.entropy_bayesian( np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) classical_probs = model.probabilities_deterministic( X_test_all[i][np.newaxis, :])[0] classical_entropy = model.entropy_deterministic( X_test_all[i][np.newaxis, :]) predictive_mean = np.mean(bayesian_probs, axis=0) predictive_std = np.std(bayesian_probs, axis=0) test_mean_std_bayesian[y_test_all[i]].append( np.concatenate((predictive_mean, predictive_std))) test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy) test_entropy_deterministic[y_test_all[i]].append(classical_entropy) test_mean_std_deterministic[y_test_all[i]].append(classical_probs) # Plotting if plot: for k in sorted(test_mean_std_bayesian.keys()): sns.plt.figure() #sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k)) sns.plt.hist(test_entropy_bayesian[k], label="Bayesian Entropy v1 for " + str(k)) #sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k)) #sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k)) sns.plt.legend() sns.plt.show() # Anomaly detection using simple threshold def anomaly_detection_old(anomaly_score_dict, name, df): threshold = np.logspace(-30, 1.0, 1000) acc = {} for t in threshold: tp = 0.0 tn = 0.0 for l in anomaly_score_dict: if l in inside_labels: tp += (np.array(anomaly_score_dict[l]) < t).mean() else: tn += (np.array(anomaly_score_dict[l]) >= t).mean() tp /= len(inside_labels) tn /= 10.0 - len(inside_labels) bal_acc = (tp + tn) / 2.0 f1_score = 2.0 * tp / (2.0 + tp - tn) acc[t] = [bal_acc, f1_score, tp, tn] print("{}\tscore\tthreshold\tTP\tTN".format(name)) sorted_acc = sorted(acc.items(), key=lambda x: x[1][0], reverse=True) df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0]) df.set_value(experiment_name, name + ' bal_acc_threshold', sorted_acc[0][0]) print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format( sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) sorted_acc = sorted(acc.items(), key=lambda x: x[1][1], reverse=True) df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1]) df.set_value(experiment_name, name + ' f1_score_threshold', sorted_acc[0][0]) print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format( sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) return df # Anomaly detection using logistic regression def anomaly_detection(anomaly_score_dict, name, df): X = [] y = [] for l in anomaly_score_dict: X += anomaly_score_dict[l] if l in inside_labels: y += [0] * len(anomaly_score_dict[l]) else: y += [1] * len(anomaly_score_dict[l]) X = np.array(X) y = np.array(y) X, y = utils.shuffle(X, y, random_state=0) X_train = X[:len(X) / 2] X_test = X[len(X) / 2:] y_train = y[:len(y) / 2] y_test = y[len(y) / 2:] clf = linear_model.LogisticRegression(C=1.0) clf.fit(X_train, y_train) auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proba(np.array(X_test))[:, 1]) print("AUC", auc) df.set_value(experiment_name, name + ' AUC', auc) if plot: # Plot ROC curve fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test), clf.predict_proba( np.array(X_test))[:, 1], pos_label=1) sns.plt.figure() sns.plt.plot(fpr, tpr, label='ROC curve') sns.plt.plot([0, 1], [0, 1], 'k--') sns.plt.xlim([0.0, 1.0]) sns.plt.ylim([0.0, 1.05]) sns.plt.xlabel('False Positive Rate') sns.plt.ylabel('True Positive Rate') sns.plt.title('Receiver operating characteristic example') sns.plt.legend(loc="lower right") sns.plt.show() return df df.set_value(experiment_name, 'dataset', dataset) df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation) df.set_value(experiment_name, 'inside_labels', str(inside_labels)) df.set_value(experiment_name, 'epochs', epochs) df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df) df = anomaly_detection(test_mean_std_deterministic, "Classical prediction", df) df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df) df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df) return df
# Finally, we do the remainder of phase 1. THIRD_TRAIN_KWARGS = dict(DEFAULT_KWARGS) # THIRD_TRAIN_KWARGS['n_epochs_pred_only']=1 THIRD_TRAIN_KWARGS['n_epochs_pred_only'] = 50 THIRD_TRAIN_KWARGS['restore_trained_actor_path'] = FIRST_TRAIN_KWARGS[ 'path_to_save_actor'] THIRD_TRAIN_KWARGS[ 'path_to_save_actor'] = 'BEST_ACTOR_FROM_WARP_CRITIC_AFTER_ADDITIONAL_PHASE_1' THIRD_TEST_KWARGS = dict(THIRD_TRAIN_KWARGS) del THIRD_TEST_KWARGS['logging_frequency'] del THIRD_TEST_KWARGS['path_to_save_actor'] train(**FIRST_TRAIN_KWARGS) tf.reset_default_graph() test(**FIRST_TEST_KWARGS) tf.reset_default_graph() train(**SECOND_TRAIN_KWARGS) tf.reset_default_graph() test(**SECOND_TEST_KWARGS) tf.reset_default_graph() train(**THIRD_TRAIN_KWARGS) tf.reset_default_graph() test(**THIRD_TEST_KWARGS) tf.reset_default_graph() exit() # print('\n\n\nchange from multivae!!!!!!\n\n\n') print("First, train just the actor") train(
if args.net == 'resnet': model = net.wide_resnet(d_names.values(), args.pretrained, args.frozen, args.old) elif args.net == 'piggyback': model = net.piggyback_net(d_names.values(), args.pretrained, args.old) else: raise ValueError( 'The network you specified is not included in our model.' 'Please specify one of the following: resnet, piggyback or quantized.' ) accuracy = 0 if args.dataset in d_names.keys(): if not args.test: accuracy = training.train(model, args.dataset, args.prefix, mirror=args.mirror, bn=args.bn, scaling=args.scaling, epochs=args.epochs, visdom_env=args.visdom) else: accuracy = training.test(model, args.dataset) if args.output: file = open(args.output, "a") file.write("Dataset: " + args.dataset + " " + "Top1= " + str(accuracy) + "\n") file.close()