def main():
# Example options:
# train ./Configs/Base.yaml
# test ./Configs/Base.yaml
ap = argparse.ArgumentParser("Progressive Transformers")
# Choose between Train and Test
ap.add_argument("mode", choices=["train", "test"],
help="train a model or test")
# Path to Config
ap.add_argument("config_path", type=str,
help="path to YAML config file")
args = ap.parse_args()
# If Train
if args.mode == "train":
train(cfg_file=args.config_path)
# If Test
elif args.mode == "test":
test(cfg_file=args.config_path, ckpt=args.ckpt)
else:
raise ValueError("Unknown mode")
def main():
# prepare dataset
train_adversarial = 1
use_cuda = True
epochs = 2000
lr = 0.0005
train_set, test_set = get_datasets(balance_train=True)
num_features = train_set[0][0].shape[0]
batch_size = 400
test_batch_size = 50
torch.manual_seed(7347)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('using device {0}'.format(device))
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=test_batch_size, shuffle=True)
model = Net(activation=nn.LeakyReLU(),
num_features=num_features,
embed_size=80).to(device)
PATH = None
# model.load_state_dict(torch.load(PATH, map_location=device), strict=False)
reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr)
discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1)
encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1)
if train_adversarial:
compute_loss = compute_loss_adversarial_enc
optimizer = {'rec': reconstruction_optimizer,
'dis': discriminative_optimizer,
'enc': encoder_optimizer}
tmp = [reconstruction_optimizer,
discriminative_optimizer,
encoder_optimizer]
schedulers = [StepLR(x, step_size=70, gamma=0.9) for x in tmp]
else:
compute_loss = compute_loss_autoenc
optimizer = {'rec': reconstruction_optimizer}
schedulers = [StepLR(reconstruction_optimizer, step_size=50, gamma=0.9)]
for epoch in range(1, epochs + 1):
if epoch % 50 == 0:
test(model, compute_loss, device, test_loader)
train(model, compute_loss, device, train_loader, optimizer, epoch)
for scheduler in schedulers:
scheduler.step()
if epoch % 100 == 0 and epoch:
torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch))
print('learning rate: {0}'.format(scheduler.get_lr()))
def main():
embed_size = 80
use_embed = True
use_cuda = True
epochs = 2000
lr = 0.0001
train_set, test_set = get_datasets(label_columns=['posOutcome'])
# process with feature extractor
# create new test and train sets
num_features = train_set[0][0].shape[0]
batch_size = 100
test_batch_size = 50
torch.manual_seed(7347)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('using device {0}'.format(device))
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(test_set,
batch_size=test_batch_size,
shuffle=True)
extractor = Net(activation=nn.LeakyReLU(),
num_features=num_features,
embed_size=embed_size).to(device)
extractor_path = 'extractor.pt'
extractor.load_state_dict(torch.load(extractor_path, map_location=device),
strict=False)
model = ClassifierNet(
num_features=embed_size if use_embed else num_features,
activation=nn.LeakyReLU()).to(device)
classifier_optimizer = optim.AdamW(model.parameters(), lr=lr)
compute_loss = compute_classifier_loss
optimizer = {'opt': classifier_optimizer}
schedulers = [StepLR(classifier_optimizer, step_size=50, gamma=0.9)]
callback = compute_classifier_loss
if use_embed:
callback = functools.partial(loss_callback, extractor=extractor)
for epoch in range(1, epochs + 1):
if epoch % 50 == 0 or epoch == 1:
test(model, callback, device, test_loader)
train(model, callback, device, train_loader, optimizer, epoch)
for scheduler in schedulers:
scheduler.step()
if epoch % 100 == 0 and epoch:
torch.save(model.state_dict(), "classifier{0}.pt".format(epoch))
print('learning rate: {0}'.format(scheduler.get_lr()))
def result_for_manifest(model, criterion, manifest, decoder, target_decoder, batch_size, num_workers):
### LOADER
test_dataset = MultiDataset(manifest,
model._meta['labels'],
use_mfcc_in=model._meta['use_mfcc_in'],
use_ivectors_in=model._meta['use_ivectors_in'],
use_embeddings_in=model._meta['use_embeddings_in'],
embedding_size=model._meta['embedding_size'],
use_transcripts_out=model._meta['use_transcripts_out'],
use_accents_out=model._meta['use_accents_out'])
test_loader = MultiDataLoader(test_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=num_workers)
### TEST
test_results = test(model, test_loader, criterion, decoder, target_decoder)
test_loss, test_loss_text, test_loss_accent, test_wer, test_accent_acc = test_results
results_dict = {}
if test_wer != -1:
results_dict['WER'] = test_wer
if test_accent_acc != -1:
results_dict['Accent accuracy'] = test_accent_acc
return results_dict
def main():
global args
args = parser.parse_args()
print()
print('Command-line argument values:')
for key, value in vars(args).items():
print('-', key, ':', value)
print()
test_params = [
args.model,
path_to_save_string(args.dataset),
path_to_save_string(args.test_dataset), args.viewpoint_modulo,
args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed,
args.routing_iters
]
test_name = '_'.join([str(x) for x in test_params]) + '.pth'
model_params = [
args.model,
path_to_save_string(args.dataset), args.viewpoint_modulo,
args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed,
args.routing_iters
]
model_name = '_'.join([str(x) for x in model_params]) + '.pth'
header = 'model,training-dataset,test-dataset,viewpoint_modulo,' \
'batch_size,epochs,lr,weight_decay,seed,em_iters,accuracy'
snapshot_path = os.path.join('.', 'snapshots', model_name)
result_path = os.path.join('.', 'results', 'pytorch_test.csv')
make_dirs_if_not_exist([snapshot_path, result_path])
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
model, criterion, optimizer, scheduler = load_model(
args.model,
device_ids=args.device_ids,
lr=args.lr,
routing_iters=args.routing_iters)
num_class, train_loader, test_loader = load_datasets(
args.test_dataset, args.batch_size, args.test_batch_size,
args.test_viewpoint_modulo)
model.load_state_dict(torch.load(snapshot_path))
acc, predictions, labels, logits = test(test_loader,
model,
criterion,
chunk=1)
print(f'Accuracy: {acc:.2f}%')
print(f'Memory usage: {gpu_memory_usage()}')
to_write = test_params + [acc.cpu().numpy()]
append_to_csv(result_path, to_write, header=header)
if args.roc != '':
make_dirs_if_not_exist(args.roc)
torch.save((predictions, labels, logits), args.roc)
def test_train_bilm_chars(self):
vocab, data, options = self._get_vocab_data_options(True, True)
train(options, data, 1, self.tmp_dir, self.tmp_dir)
# now test
tf.reset_default_graph()
options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir)
data_test, vocab_test = self._get_data(True, True, True)
perplexity = test(options, ckpt_file, data_test, batch_size=1)
self.assertTrue(perplexity < 20.0)
def main(args):
options, ckpt_file = load_options_latest_checkpoint(args.save_dir)
# load the vocab
if 'char_cnn' in options:
max_word_length = options['char_cnn']['max_characters_per_token']
else:
max_word_length = None
vocab = load_vocab(args.vocab_file, max_word_length)
test_prefix = args.test_prefix
kwargs = {
'test': True,
'shuffle_on_load': False,
}
if options.get('bidirectional'):
data = BidirectionalLMDataset(test_prefix, vocab, **kwargs)
else:
data = LMDataset(test_prefix, vocab, **kwargs)
test(options, ckpt_file, data, batch_size=args.batch_size)
def test_train_skip_connections(self):
bidirectional = True
use_chars = False
vocab, data, options = self._get_vocab_data_options(
bidirectional, use_chars)
options['lstm']['use_skip_connections'] = True
train(options, data, 1, self.tmp_dir, self.tmp_dir)
# now test
tf.reset_default_graph()
options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir)
data_test, vocab_test = self._get_data(
bidirectional, use_chars, test=True)
perplexity = test(options, ckpt_file, data_test, batch_size=1)
self.assertTrue(perplexity < 20.0)
def test_train_shared_softmax_embedding(self):
bidirectional = True
use_chars = False
vocab, data, options = self._get_vocab_data_options(
bidirectional, use_chars, share_embedding_softmax=True)
train(options, data, 1, self.tmp_dir, self.tmp_dir)
# now test
tf.reset_default_graph()
options, ckpt_file = load_options_latest_checkpoint(self.tmp_dir)
data_test, vocab_test = self._get_data(
bidirectional, use_chars, test=True)
perplexity = test(options, ckpt_file, data_test, batch_size=1)
self.assertTrue(perplexity < 20.0)
def main():
train_adversarial = 1
use_cuda = True
epochs = 200
lr = 0.005
batch_size = 100
test_batch_size = 100
torch.manual_seed(7347)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('using device {0}'.format(device))
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
])),
batch_size=batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
transform=transforms.Compose([
transforms.ToTensor(),
])),
batch_size=test_batch_size,
shuffle=True,
**kwargs)
model = Net(activation=nn.LeakyReLU()).to(device)
PATH = 'mnist_cnn100.pt.bak'
# model.load_state_dict(torch.load(PATH, map_location=device), strict=False)
reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr)
discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1)
encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1)
if train_adversarial:
compute_loss = compute_loss_adversarial_enc
optimizer = {
'rec': reconstruction_optimizer,
'dis': discriminative_optimizer,
'enc': encoder_optimizer
}
tmp = [
reconstruction_optimizer, discriminative_optimizer,
encoder_optimizer
]
schedulers = [StepLR(x, step_size=5, gamma=0.9) for x in tmp]
else:
compute_loss = compute_loss_autoenc
optimizer = {'rec': reconstruction_optimizer}
schedulers = [StepLR(reconstruction_optimizer, step_size=5, gamma=0.9)]
for epoch in range(1, epochs + 1):
if epoch % 5 == 0:
test(model, compute_loss, device, test_loader)
train(model, compute_loss, device, train_loader, optimizer, epoch)
for scheduler in schedulers:
scheduler.step()
if epoch % 10 == 0 and epoch:
torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch))
print('learning rate: {0}'.format(scheduler.get_lr()))
exit(0)
train_scores = np.zeros(args.epochs)
validation_scores = np.zeros(args.epochs)
with torch.cuda.device(args.device):
try:
t = trange(args.start_epoch, args.epochs)
for epoch in t:
if args.train:
train_scores[epoch] = train(train_loader, model, optimizer,
args)
else:
train_scores[epoch] = 0
if args.test:
validation_scores[epoch] = test(test_loader, model, args)
else:
validation_scores[epoch] = 0
results(test_loader.dataset, model, vis, epoch + 1)
save_checkpoint(
{
'args': args,
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
'losses': losses,
},
filename=args.checkpoint)
for i in range(epoch):
scheduler.step()
for i in range(epoch, 400):
epoch += 1
scheduler.step()
train_loader = get_split_dataloader(f'{args.dataset}/train_k{i % 99}.csv',
width=width,
batch_size=batch_size,
transform=transform,
num_workers=num_workers)
train(model,
train_loader,
optimizer=optimizer,
epoch=epoch,
half=args.half)
if epoch % evaluate_interval == 0:
mean_loss, mean_map, t = test(model, valid_loader, half=args.half)
if hvd.rank() == 0:
checkpoint_path = save_checkpoint(model,
optimizer,
test_acc=mean_map,
tag=epoch)
mean_loss, mean_map, t = test(model, valid_loader, half=args.half)
if hvd.rank() == 0:
checkpoint_path = save_checkpoint(model,
optimizer,
test_acc=mean_map,
tag=epoch)
verbose=False,
version_tag="RUN_TO_GET_UNNORMALIZED_INFO",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='multi_vae',
data_subdir=data_subdir,
n_epochs_pred_only=N_EPOCHS_PRED_ONLY,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
max_kl=0.2,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=0.01,
evaluation_metric="DCG",
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
version_tag="RUN_TO_GET_UNNORMALIZED_INFO",
)
print("On to round 2! Now we'll do the critic.")
train(
model_class='multi_vae',
data_subdir=data_subdir,
n_epochs_pred_only=0,
n_epochs_ac_only=N_EPOCHS_AC_ONLY,
positive_weights=5.0,
version_tag="VWMF_JUST_ACTOR",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='variational_wmf',
# data_subdir=data_subdir,
n_epochs_pred_only=200,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
max_kl=0.05,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=0.0001,
evaluation_metric="NDCG",
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
positive_weights=5.0,
version_tag="VWMF_JUST_ACTOR",
)
print("On to round 2! Now we'll do the critic.")
train(
model_class='variational_wmf',
# data_subdir=data_subdir,
n_epochs_pred_only=0,
n_epochs_ac_only=50,
break_early=BREAK_EARLY,
verbose=False,
version_tag="ProperlyShapedMultiDAE_ONLY_ACTOR",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='new_multi_dae',
n_epochs_pred_only=200,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
max_kl=0.2,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=1e-05,
evaluation_metric="NDCG",
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
version_tag="ProperlyShapedMultiDAE_ONLY_ACTOR",
)
print("On to round 2! Now we'll do the critic.")
train(
model_class='new_multi_dae',
n_epochs_pred_only=0,
n_epochs_ac_only=50,
n_epochs_pred_and_ac=50,
max_kl=0.2,
0.8, 'addit', test=True
) #both work also, because of test=true, the mode option is useless here
use_gpu = True
n_epoch = 100
learning_rate = 0.002
history = train(model,
train_loader,
valid_loader,
n_epoch,
learning_rate,
use_gpu=use_gpu)
history.display()
test_acc, test_loss = test(model, test_loader, use_gpu=use_gpu)
print('Test:\n\tLoss: {}\n\tAccuracy: {}'.format(test_loss, test_acc))
#Train with Input Dropout in addit mode
model = Net()
model.type(torch.cuda.FloatTensor)
train_loader, valid_loader, test_loader = pixar_dataset(0.8,
'addit',
test=False)
use_gpu = True
n_epoch = 100
learning_rate = 0.002
history = train(model,
train_loader,
def anomaly(experiment_name,
dataset = "mnist",
bayesian_approximation = "dropout",
inside_labels = [0, 1],
num_epochs = 50,
batch_size = 128,
acc_threshold = 0.6,
weight_decay = 1e-5,
dropout_p = 0.5,
fc_layers = [512, 512],
plot = True):
"""
This methods trains a neural network classifier on a subset of classes.
After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies.
The anomalous classes are the ones that are not part of the training subset.
dataset = "mnist" or "cifar"
For MNIST we use a fully-connected MLP.
For CIFAR10 we use a convolutional net (similar to LeNet)
bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST
bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST.
inside_labels are the subset of trained classes, the other classes are only used for testing.
"""
n_out = len(inside_labels)
# Prepare Theano variables for inputs and targets
# Load the dataset
print("Loading data...")
if dataset == "mnist":
input_var = T.matrix('inputs')
target_var = T.ivector('targets')
n_in = [28*28]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels)
if bayesian_approximation == "dropout":
model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay)
elif bayesian_approximation == "variational":
model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size))
elif dataset == "cifar":
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
n_in = [3, 32, 32]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10(inside_labels)
model = models.convnet_dropout(input_var, target_var, n_in, n_out, dropout_p, weight_decay)
df = pd.DataFrame()
# Mini-batch training with ADAM
epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold)
# Mini-batch testing
acc, bayes_acc = training.test(model, X_test, y_test, batch_size)
df.set_value(experiment_name, "test_acc", acc)
df.set_value(experiment_name, "bayes_test_acc", bayes_acc)
# Uncertainty prediction
test_mean_std_bayesian = {x:[] for x in range(10)}
test_mean_std_deterministic = {x:[] for x in range(10)}
test_entropy_bayesian = {x:[] for x in range(10)}
test_entropy_deterministic = {x:[] for x in range(10)}
for i in range(len(X_test_all)):
bayesian_probs = model.probabilities(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
bayesian_entropy = model.entropy_bayesian(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
classical_probs = model.probabilities_deterministic(X_test_all[i][np.newaxis,:])[0]
classical_entropy = model.entropy_deterministic(X_test_all[i][np.newaxis,:])
predictive_mean = np.mean(bayesian_probs, axis=0)
predictive_std = np.std(bayesian_probs, axis=0)
test_mean_std_bayesian[y_test_all[i]].append(np.concatenate((predictive_mean, predictive_std)))
test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy)
test_entropy_deterministic[y_test_all[i]].append(classical_entropy)
test_mean_std_deterministic[y_test_all[i]].append(classical_probs)
# Plotting
if plot:
for k in sorted(test_mean_std_bayesian.keys()):
sns.plt.figure()
#sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k))
sns.plt.hist(test_entropy_bayesian[k], label = "Bayesian Entropy v1 for " + str(k))
#sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k))
#sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k))
sns.plt.legend()
sns.plt.show()
# Anomaly detection using simple threshold
def anomaly_detection_old(anomaly_score_dict, name, df):
threshold = np.logspace(-30, 1.0, 1000)
acc = {}
for t in threshold:
tp = 0.0
tn = 0.0
for l in anomaly_score_dict:
if l in inside_labels:
tp += (np.array(anomaly_score_dict[l]) < t).mean()
else:
tn += (np.array(anomaly_score_dict[l]) >= t).mean()
tp /= len(inside_labels)
tn /= 10.0 - len(inside_labels)
bal_acc = (tp + tn)/2.0
f1_score = 2.0*tp/(2.0 + tp - tn)
acc[t] = [bal_acc, f1_score, tp, tn]
print("{}\tscore\tthreshold\tTP\tTN".format(name))
sorted_acc = sorted(acc.items(), key= lambda x : x[1][0], reverse = True)
df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0])
df.set_value(experiment_name, name + ' bal_acc_threshold', sorted_acc[0][0])
print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3]))
sorted_acc = sorted(acc.items(), key= lambda x : x[1][1], reverse = True)
df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1])
df.set_value(experiment_name, name + ' f1_score_threshold', sorted_acc[0][0])
print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3]))
return df
# Anomaly detection using logistic regression
def anomaly_detection(anomaly_score_dict, name, df):
X = []
y = []
for l in anomaly_score_dict:
X += anomaly_score_dict[l]
if l in inside_labels:
y += [0]*len(anomaly_score_dict[l])
else:
y += [1]*len(anomaly_score_dict[l])
X = np.array(X)
y = np.array(y)
X, y = utils.shuffle(X, y, random_state=0)
X_train = X[:len(X)/2]
X_test = X[len(X)/2:]
y_train = y[:len(y)/2]
y_test = y[len(y)/2:]
clf = linear_model.LogisticRegression(C=1.0)
clf.fit(X_train, y_train)
auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1])
print("AUC", auc)
df.set_value(experiment_name, name + ' AUC', auc)
if plot: # Plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1], pos_label=1)
sns.plt.figure()
sns.plt.plot(fpr, tpr, label='ROC curve')
sns.plt.plot([0, 1], [0, 1], 'k--')
sns.plt.xlim([0.0, 1.0])
sns.plt.ylim([0.0, 1.05])
sns.plt.xlabel('False Positive Rate')
sns.plt.ylabel('True Positive Rate')
sns.plt.title('Receiver operating characteristic example')
sns.plt.legend(loc="lower right")
sns.plt.show()
return df
df.set_value(experiment_name, 'dataset', dataset)
df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation)
df.set_value(experiment_name, 'inside_labels', str(inside_labels))
df.set_value(experiment_name, 'epochs', epochs)
df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df)
df = anomaly_detection(test_mean_std_deterministic, "Classical prediction", df)
df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df)
df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df)
return df
# 'logging_frequency': 50,
# 'logging_frequency': 50,
# 'batch_size': 500,
# 'batch_size': 5,
# 'break_early': True,
# 'break_early': False,
'verbose':
True,
# 'path_to_save_actor': "best_ndcg_trained_150_epochs",
# 'path_to_save_last_actor': "last_actor_after_150_trained_epochs",
'version_tag':
"PHASE_4_LAMBDARANK",
'path_to_save_actor':
"MSD_BEST_ACTOR_FROM_LAMBDARANK_PHASE_4",
'restore_trained_actor_path':
"MSD_BEST_ACTOR_FROM_WARP_CRITIC_ONLY_PHASE_1_FULL"
}
# jUST PHASE 4 THIS TIME
FIRST_TRAIN_KWARGS = dict(DEFAULT_KWARGS)
# NOW FOR TESTING
FIRST_TEST_KWARGS = dict(DEFAULT_KWARGS)
del FIRST_TEST_KWARGS['logging_frequency']
del FIRST_TEST_KWARGS['path_to_save_actor']
train(**FIRST_TRAIN_KWARGS)
tf.reset_default_graph()
test(**FIRST_TEST_KWARGS)
exit()
verbose=False,
version_tag="FULL_RUN_ON_MSD_ONLY",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='multi_vae',
data_subdir=data_subdir,
n_epochs_pred_only=100,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
# max_kl=0.2,
max_kl=0.1,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=0.01,
evaluation_metric="NDCG",
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
version_tag="FULL_RUN_ON_MSD_ONLY",
)
print("On to round 2! Now we'll do the critic.")
train(
model_class='multi_vae',
data_subdir=data_subdir,
n_epochs_pred_only=0,
n_epochs_ac_only=50,
def main():
global args
args = parser.parse_args()
print()
print('Command-line argument values:')
for key, value in vars(args).items():
print('-', key, ':', value)
print()
params = [
args.model,
path_to_save_string(args.dataset), args.viewpoint_modulo,
args.batch_size, args.epochs, args.lr, args.weight_decay, args.seed,
args.routing_iters
]
model_name = '_'.join([str(x) for x in params]) + '.pth'
header = 'model,dataset,viewpoint_modulo,batch_size,epochs,lr,weight_decay,seed,em_iters,accuracy'
snapshot_path = os.path.join('.', 'snapshots', model_name)
data_path = os.path.join('.', 'results', 'training_data', model_name)
result_path = os.path.join('.', 'results', 'pytorch_train.csv')
make_dirs_if_not_exist([snapshot_path, data_path, result_path])
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
model, criterion, optimizer, scheduler = load_model(
args.model,
device_ids=args.device_ids,
lr=args.lr,
routing_iters=args.routing_iters)
num_class, train_loader, test_loader = load_datasets(
args.dataset, args.batch_size, args.test_batch_size,
args.viewpoint_modulo)
best_acc = 0
training_accuracies = []
test_accuracies = []
if args.append:
model.load_state_dict(torch.load(snapshot_path))
try:
for epoch in range(1, args.epochs + 1):
print()
acc = train(train_loader,
model,
criterion,
optimizer,
epoch,
epochs=args.epochs,
log_interval=args.log_interval)
training_accuracies.append(acc)
scheduler.step(acc)
print('Epoch accuracy was %.1f%%. Learning rate is %.9f.' %
(acc, optimizer.state_dict()['param_groups'][0]['lr']))
if epoch % args.test_interval == 0:
test_acc, __, __, __ = test(test_loader,
model,
criterion,
chunk=args.test_size)
test_accuracies.append(test_acc)
if test_acc > best_acc:
best_acc = test_acc
except KeyboardInterrupt:
print('Cancelled training after %d epochs' % (epoch - 1))
args.epochs = epoch - 1
acc, predictions, labels, logits = test(test_loader,
model,
criterion,
chunk=1)
print(f'Accuracy: {acc:.2f}% (best: {best_acc:.2f}%)')
to_write = params + [acc.cpu().numpy()]
append_to_csv(result_path, to_write, header=header)
snapshot(snapshot_path, model)
#torch.save((accuracies, labels, predictions), data_path)
if args.learn_curve != '':
make_dirs_if_not_exist(args.learn_curve)
torch.save((training_accuracies, test_accuracies), args.learn_curve)
def main():
num_gpus = torch.cuda.device_count()
dist.init_process_group(backend=args.backend, init_method="env://")
# Hyperparameters
hparams = {
"n_cnn_layers": 3,
"n_rnn_layers": 5,
"rnn_dim": 512,
"n_class": 29,
"n_feats": 128,
"stride": 2,
"dropout": 0.1,
"learning_rate": 5e-4,
"batch_size": 10,
"epochs": 20
}
# Load train and test data
train_dataset = torchaudio.datasets.LIBRISPEECH("./data",
url=args.train_url,
download=False)
train_sampler = DistributedSampler(dataset=train_dataset)
train_loader = data.DataLoader(
dataset=train_dataset,
batch_size=hparams['batch_size'],
sampler=train_sampler,
collate_fn=lambda x: data_processing(x, 'train'),
num_workers=num_gpus,
pin_memory=True)
test_dataset = torchaudio.datasets.LIBRISPEECH("./data",
url="test-clean",
download=False)
test_loader = data.DataLoader(
dataset=test_dataset,
batch_size=hparams['batch_size'],
shuffle=False,
collate_fn=lambda x: data_processing(x, 'valid'),
num_workers=num_gpus,
pin_memory=True)
# Set seed for reproducibility across all GPUs
torch.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.enabled = False
# Instantiate model
if args.model == "baseline":
model = DeepSpeech2(hparams['n_cnn_layers'], hparams['n_rnn_layers'],
hparams['rnn_dim'], hparams['n_class'],
hparams['n_feats'], hparams['stride'],
hparams['dropout'])
elif args.model == "improved":
model = ImprovedDeepSpeech2(hparams['n_cnn_layers'],
hparams['n_rnn_layers'],
hparams['rnn_dim'], hparams['n_class'],
hparams['n_feats'], hparams['stride'],
hparams['dropout'])
device = torch.device('cuda:{}'.format(args.local_rank))
model.to(device)
model = DistributedDataParallel(model,
device_ids=[args.local_rank],
output_device=args.local_rank)
optimizer = optim.AdamW(model.parameters(), hparams['learning_rate'])
criterion = nn.CTCLoss(blank=28)
scheduler = optim.lr_scheduler.OneCycleLR(optimizer,
max_lr=hparams['learning_rate'],
steps_per_epoch=int(
len(train_loader)),
epochs=hparams['epochs'],
anneal_strategy='linear')
results = []
start = time.perf_counter()
for epoch in range(1, hparams['epochs'] + 1):
print(f"Local Rank: {args.local_rank}, Epoch: {epoch}, Training...")
train(model, device, train_loader, criterion, optimizer, scheduler,
epoch)
if args.local_rank == 0:
avg_cer, avg_wer = test(model, device, test_loader, criterion,
epoch)
print(
f"Epoch: {epoch:2} Average CER: {avg_cer:.4f}, Average WER: {avg_wer:.4f}\n"
)
results.append([avg_cer, avg_wer])
end = time.perf_counter()
print(f"Execution time: {end - start:0.2f} seconds")
df = pd.DataFrame(results, columns=['avg_cer', 'avg_wer'])
df.to_csv(args.model + '_' + args.train_url + '.csv', index=False)
n_in = [28*28]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels)
# Load model
if bayesian_approximation = 'dropout':
model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay)
else bayesian_approximation = "variational":
model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size))
pd = pd.DataFrame()
# mini-batch training
epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold)
# mini-batch testing
acc, bayes_acc = training.test(model, X_test, y_test, batch_size)
df.set_value(experiment_name, 'test_acc', acc)
df.set_value(experiment_name, 'bayes_test_acc', bayes_acc)
# uncertainty prediction
test_mean_std_bayesian = {x:[] for x in range(10)}
test_mean_std_deterministic = {x:[] for x in range(10)}
test_entropy_bayesian = {x:[] for x in range(10)}
test_entropy_deterministic = {x:[] for x in range(10)}
for i in range(len(X_test_all)):
bayesian_probs = model.probabilities(np.title(X_test_all[i], batch_size).reshape([-1]+n_in))
bayesian_entropy = model.entropy_bayesian(np.title(X_terst_all[i], batch_size).reshape([-1]+n_in))
predictve_mean = np.mean(bayesian_probs, axis=0)
predictve_std = np.std(bayesian_probs, axis=0)
hidden_size=50,
embedding_dim=300,
dropout=0)
net.to(device)
if checkpoint:
print('==> Resuming from checkpoint..')
print(checkpoint)
checkpoint = torch.load(checkpoint)
net.load_state_dict(checkpoint['net_state_dicts'])
torch_rng_state = checkpoint['torch_rng_state']
torch.set_rng_state(torch_rng_state)
numpy_rng_state = checkpoint['numpy_rng_state']
np.random.set_state(numpy_rng_state)
eval_iter = enumerate(eval_loader)
for i in range(100):
net.eval()
print(training.test(net, eval_iter, device))
batch_size = 100
eval_data = util.SICKData('validation')
eval_loader = DataLoader(eval_data,
batch_size=batch_size,
shuffle=True,
drop_last=True)
eval_iter = enumerate(eval_loader)
step, (cla_label, reg_label, senA, senB) = next(eval_iter)
net.eval()
output_1hot = net.forward(senA, senB)
output = util.onehot2class(output_1hot)
(output == cla_label.to(device)).sum().item() / 100
positive_weights=5.0,
version_tag="FULL_RUN_ON_OTHER_DATASETS",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='wmf_vae',
data_subdir=data_subdir,
n_epochs_pred_only=100,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
max_kl=0.05,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=0.0001,
evaluation_metric="NDCG",
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
positive_weights=5.0,
version_tag="FULL_RUN_ON_OTHER_DATASETS",
)
print("On to round 2! Now we'll do the critic.")
train(
model_class='wmf_vae',
data_subdir=data_subdir,
n_epochs_pred_only=0,
n_epochs_ac_only=50,
print("On to testing.")
test(
# model_class="wmf",
# model_class='multi_vae',
model_class='warp_encoder',
n_epochs_pred_only=0,
n_epochs_ac_only=10,
n_epochs_pred_and_ac=10,
epochs_to_anneal_over=100,
# min_kl=0.0001,
max_kl=0.0,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=1e-5,
# positive_weights=5,
# evaluation_metric='AP',
evaluation_metric="NDCG",
# logging_frequency=25,
# logging_frequency=50,
# logging_frequency=50,
batch_size=500,
# batch_size=25,
break_early=False,
verbose=False,
# path_to_save_actor="best_ndcg_trained_150_epochs",
# path_to_save_last_actor="last_actor_after_150_trained_epochs",
version_tag="WARP_WITH_CRITIC",
# path_to_save_actor="BEST_WARP_RUN_15_EPOCHS_TRUTHFUL_LOSS",
restore_trained_actor_path="BEST_WARP_RUN_15_EPOCHS_TRUTHFUL_LOSS")
exit()
from tensorflow import keras
import data_preparation as dp
import training as tr
if __name__ == '__main__':
model = keras.models.load_model('models/updated_model')
df = dp.make_dataframe_and_add_y()
print(df.head())
print(df.shape)
print("data frame is ready!")
scaled_df, scaler = dp.scale_data_for_lstm(df)
X_train, y_train, X_test, y_test = dp.create_train_test_sets(scaled_df, 100, 0.8)
tr.test(model, scaler, X_test, y_test)
def run_experiment(_exp_name, _epochs, _train_manifest, _test_manifest,
_labels, _use_mfcc_in, _use_ivectors_in, _use_embeddings_in,
_use_transcripts_out, _use_accents_out, _batch_size,
_num_workers, _mfcc_size, _ivector_size, _embedding_size,
_rnn_type, _rnn_hidden_size, _nb_head_layers,
_nb_speech_layers, _nb_accents_layers, _bidirectional,
_losses_mix, _learning_rate, _lm_path, _decoder_alpha,
_decoder_beta, _decoder_cutoff_top_n, _decoder_beam_width,
_cuda, _tensorboard_path, _saved_models_path,
_bottleneck_size, _accent_loss):
print(f'\n##### Running experiment {_exp_name} #####')
# Tools to log values
results_dict = {}
results_dict['train_loss'] = []
results_dict['train_loss_text'] = []
results_dict['train_loss_accent'] = []
results_dict['test_loss'] = []
results_dict['test_loss_text'] = []
results_dict['test_loss_accent'] = []
results_dict['test_wer'] = []
results_dict['test_accent_acc'] = []
tb_path = Path(_tensorboard_path) / _exp_name
makedirs(tb_path, exist_ok=True)
tb_writer = SummaryWriter(tb_path)
### DATA LOADING
# Training set
train_dataset = MultiDataset(_train_manifest,
_labels,
use_mfcc_in=_use_mfcc_in,
use_ivectors_in=_use_ivectors_in,
use_embeddings_in=_use_embeddings_in,
embedding_size=_embedding_size,
use_transcripts_out=_use_transcripts_out,
use_accents_out=_use_accents_out)
train_loader = MultiDataLoader(train_dataset,
batch_size=_batch_size,
shuffle=True,
num_workers=_num_workers)
# Testing set
test_dataset = MultiDataset(_test_manifest,
_labels,
use_mfcc_in=_use_mfcc_in,
use_ivectors_in=_use_ivectors_in,
use_embeddings_in=_use_embeddings_in,
embedding_size=_embedding_size,
use_transcripts_out=_use_transcripts_out,
use_accents_out=_use_accents_out)
test_loader = MultiDataLoader(test_dataset,
batch_size=_batch_size,
shuffle=True,
num_workers=_num_workers)
### CREATE MODEL
model = MultiTask(use_mfcc_in=_use_mfcc_in,
use_ivectors_in=_use_ivectors_in,
use_embeddings_in=_use_embeddings_in,
use_transcripts_out=_use_transcripts_out,
use_accents_out=_use_accents_out,
mfcc_size=_mfcc_size,
ivector_size=_ivector_size,
embedding_size=_embedding_size,
rnn_type=_rnn_type,
labels=_labels,
accents_dict=train_dataset.accent_dict,
rnn_hidden_size=_rnn_hidden_size,
nb_head_layers=_nb_head_layers,
nb_speech_layers=_nb_speech_layers,
nb_accents_layers=_nb_accents_layers,
bidirectional=_bidirectional,
bottleneck_size=_bottleneck_size,
DEBUG=False)
if _cuda:
model = model.cuda()
print(model, '\n')
print('Model parameters counts:', MultiTask.get_param_size(model), '\n')
### OPTIMIZER, CRITERION, DECODER
# Optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=_learning_rate)
# Criterion
if _use_accents_out:
if _accent_loss == 'focal':
AccLoss = FocalLoss()
elif _accent_loss == 'CE':
AccLoss = nn.CrossEntropyLoss()
else:
raise ValueError(
f'Loss {_accent_loss} for accent_loss is unknown. Please use either "focal" or "CE".'
)
if not _use_transcripts_out: # only accent classification
criterion = AccLoss
elif not _use_accents_out: # only text recognition
criterion = nn.CTCLoss()
else: # both tasks
criterion = (nn.CTCLoss(), FocalLoss())
# Decoder
if _use_transcripts_out:
decoder = BeamCTCDecoder(_labels,
lm_path=_lm_path,
alpha=_decoder_alpha,
beta=_decoder_beta,
cutoff_top_n=_decoder_cutoff_top_n,
cutoff_prob=_decoder_cutoff_top_n,
beam_width=_decoder_beam_width,
num_processes=_num_workers)
target_decoder = GreedyDecoder(_labels)
else:
decoder, target_decoder = None, None
### EPOCHS
best_wer = math.inf
best_acc = 0
for epoch in range(1, _epochs + 1):
### TRAIN
print(f'Epoch {epoch} training: {exp_name}')
train_results = train(model,
train_loader,
criterion,
optimizer,
losses_mix=_losses_mix)
train_loss, train_loss_text, train_loss_accent = train_results
results_dict['train_loss'].append(train_loss)
results_dict['train_loss_text'].append(train_loss_text)
results_dict['train_loss_accent'].append(train_loss_accent)
print(f'Epoch {epoch} training loss: {train_loss}')
### TEST
print(f'Epoch {epoch} testing')
test_results = test(model,
test_loader,
criterion,
decoder,
target_decoder,
losses_mix=_losses_mix)
test_loss, test_loss_text, test_loss_accent, test_wer, test_accent_acc = test_results
results_dict['test_loss'].append(test_loss)
results_dict['test_loss_text'].append(test_loss_text)
results_dict['test_loss_accent'].append(test_loss_accent)
results_dict['test_wer'].append(test_wer)
results_dict['test_accent_acc'].append(test_accent_acc)
print(f'Epoch {epoch} testing loss: {test_loss}')
# Add values to tensorboard
for key, results in results_dict.items():
tb_writer.add_scalar(key, results[-1], epoch)
#Save model if it is best
save_new = False
if _use_transcripts_out:
if test_wer < best_wer:
save_new = True
best_wer = test_wer
else:
if test_accent_acc > best_acc:
save_new = True
best_acc = test_accent_acc
if save_new:
MultiTask.serialize(
model,
Path(_saved_models_path) / _exp_name,
save=True,
exp_name=_exp_name,
optimizer=optimizer,
epoch=epoch,
train_losses=results_dict['train_loss'],
test_losses=results_dict['test_loss'],
text_train_losses=results_dict['train_loss_text'],
text_test_losses=results_dict['test_loss_text'],
text_wers=results_dict['test_wer'],
accent_train_losses=results_dict['train_loss_accent'],
accent_test_losses=results_dict['test_loss_accent'],
accent_accuracies=results_dict['test_accent_acc'])
del model
gc.collect()
torch.cuda.empty_cache()
order = parse_order(args.order)
if depth and rgb:
model = RGBDNet.double_combined18(classes_list.values(), index, order, args.pretrained)
else:
model = cbnet.combined_net18(classes_list.values(), pre_imagenet=True, pretrained=args.pretrained,
order=order)
model.set_index(index)
else:
raise(Exception("Error in parameters. Network should be one between " + str(network_list)))
if not TEST:
accuracy = training.train(args.network, model, train_loader, test_loader, prefix=prefix, visdom_env=visdom,
step=step, batch=batch, epochs=epochs, lr=lr, decay=decay, adamlr=adamlr, momentum=0.9,
freeze=args.frozen, cost_function=cost_function, metric=metric)
else:
accuracy = training.test(model, test_loader, cost_function, metric)
print("Saving results in RESULTS.txt")
out = open("RESULTS.txt", "a")
output = {
"date" : str(date.today()),
"name" : visdom,
"task" : args.task,
"net" : args.network,
"rgb" : rgb,
"depth" : depth,
"epochs" : epochs,
"adamlr" : adamlr,
"lr" : lr,
"max_acc": "{:.2f}".format(accuracy[0]),
def anomaly(experiment_name,
dataset="mnist",
bayesian_approximation="dropout",
inside_labels=[0, 1],
num_epochs=50,
batch_size=128,
acc_threshold=0.6,
weight_decay=1e-5,
dropout_p=0.5,
fc_layers=[512, 512],
plot=True):
"""
This methods trains a neural network classifier on a subset of classes.
After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies.
The anomalous classes are the ones that are not part of the training subset.
dataset = "mnist" or "cifar"
For MNIST we use a fully-connected MLP.
For CIFAR10 we use a convolutional net (similar to LeNet)
bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST
bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST.
inside_labels are the subset of trained classes, the other classes are only used for testing.
"""
n_out = len(inside_labels)
# Prepare Theano variables for inputs and targets
# Load the dataset
print("Loading data...")
if dataset == "mnist":
input_var = T.matrix('inputs')
target_var = T.ivector('targets')
n_in = [28 * 28]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(
inside_labels)
if bayesian_approximation == "dropout":
model = models.mlp_dropout(input_var, target_var, n_in, n_out,
fc_layers, dropout_p, weight_decay)
elif bayesian_approximation == "variational":
model = models.mlp_variational(input_var, target_var, n_in, n_out,
fc_layers, batch_size,
len(X_train) / float(batch_size))
elif dataset == "cifar":
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
n_in = [3, 32, 32]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10(
inside_labels)
model = models.convnet_dropout(input_var, target_var, n_in, n_out,
dropout_p, weight_decay)
df = pd.DataFrame()
# Mini-batch training with ADAM
epochs = training.train(model, X_train, y_train, X_test, y_test,
batch_size, num_epochs, acc_threshold)
# Mini-batch testing
acc, bayes_acc = training.test(model, X_test, y_test, batch_size)
df.set_value(experiment_name, "test_acc", acc)
df.set_value(experiment_name, "bayes_test_acc", bayes_acc)
# Uncertainty prediction
test_mean_std_bayesian = {x: [] for x in range(10)}
test_mean_std_deterministic = {x: [] for x in range(10)}
test_entropy_bayesian = {x: [] for x in range(10)}
test_entropy_deterministic = {x: [] for x in range(10)}
for i in range(len(X_test_all)):
bayesian_probs = model.probabilities(
np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
bayesian_entropy = model.entropy_bayesian(
np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
classical_probs = model.probabilities_deterministic(
X_test_all[i][np.newaxis, :])[0]
classical_entropy = model.entropy_deterministic(
X_test_all[i][np.newaxis, :])
predictive_mean = np.mean(bayesian_probs, axis=0)
predictive_std = np.std(bayesian_probs, axis=0)
test_mean_std_bayesian[y_test_all[i]].append(
np.concatenate((predictive_mean, predictive_std)))
test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy)
test_entropy_deterministic[y_test_all[i]].append(classical_entropy)
test_mean_std_deterministic[y_test_all[i]].append(classical_probs)
# Plotting
if plot:
for k in sorted(test_mean_std_bayesian.keys()):
sns.plt.figure()
#sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k))
sns.plt.hist(test_entropy_bayesian[k],
label="Bayesian Entropy v1 for " + str(k))
#sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k))
#sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k))
sns.plt.legend()
sns.plt.show()
# Anomaly detection using simple threshold
def anomaly_detection_old(anomaly_score_dict, name, df):
threshold = np.logspace(-30, 1.0, 1000)
acc = {}
for t in threshold:
tp = 0.0
tn = 0.0
for l in anomaly_score_dict:
if l in inside_labels:
tp += (np.array(anomaly_score_dict[l]) < t).mean()
else:
tn += (np.array(anomaly_score_dict[l]) >= t).mean()
tp /= len(inside_labels)
tn /= 10.0 - len(inside_labels)
bal_acc = (tp + tn) / 2.0
f1_score = 2.0 * tp / (2.0 + tp - tn)
acc[t] = [bal_acc, f1_score, tp, tn]
print("{}\tscore\tthreshold\tTP\tTN".format(name))
sorted_acc = sorted(acc.items(), key=lambda x: x[1][0], reverse=True)
df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0])
df.set_value(experiment_name, name + ' bal_acc_threshold',
sorted_acc[0][0])
print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(
sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2],
sorted_acc[0][1][3]))
sorted_acc = sorted(acc.items(), key=lambda x: x[1][1], reverse=True)
df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1])
df.set_value(experiment_name, name + ' f1_score_threshold',
sorted_acc[0][0])
print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(
sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2],
sorted_acc[0][1][3]))
return df
# Anomaly detection using logistic regression
def anomaly_detection(anomaly_score_dict, name, df):
X = []
y = []
for l in anomaly_score_dict:
X += anomaly_score_dict[l]
if l in inside_labels:
y += [0] * len(anomaly_score_dict[l])
else:
y += [1] * len(anomaly_score_dict[l])
X = np.array(X)
y = np.array(y)
X, y = utils.shuffle(X, y, random_state=0)
X_train = X[:len(X) / 2]
X_test = X[len(X) / 2:]
y_train = y[:len(y) / 2]
y_test = y[len(y) / 2:]
clf = linear_model.LogisticRegression(C=1.0)
clf.fit(X_train, y_train)
auc = metrics.roc_auc_score(np.array(y_test),
clf.predict_proba(np.array(X_test))[:, 1])
print("AUC", auc)
df.set_value(experiment_name, name + ' AUC', auc)
if plot: # Plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test),
clf.predict_proba(
np.array(X_test))[:,
1],
pos_label=1)
sns.plt.figure()
sns.plt.plot(fpr, tpr, label='ROC curve')
sns.plt.plot([0, 1], [0, 1], 'k--')
sns.plt.xlim([0.0, 1.0])
sns.plt.ylim([0.0, 1.05])
sns.plt.xlabel('False Positive Rate')
sns.plt.ylabel('True Positive Rate')
sns.plt.title('Receiver operating characteristic example')
sns.plt.legend(loc="lower right")
sns.plt.show()
return df
df.set_value(experiment_name, 'dataset', dataset)
df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation)
df.set_value(experiment_name, 'inside_labels', str(inside_labels))
df.set_value(experiment_name, 'epochs', epochs)
df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df)
df = anomaly_detection(test_mean_std_deterministic, "Classical prediction",
df)
df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df)
df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df)
return df
# Finally, we do the remainder of phase 1.
THIRD_TRAIN_KWARGS = dict(DEFAULT_KWARGS)
# THIRD_TRAIN_KWARGS['n_epochs_pred_only']=1
THIRD_TRAIN_KWARGS['n_epochs_pred_only'] = 50
THIRD_TRAIN_KWARGS['restore_trained_actor_path'] = FIRST_TRAIN_KWARGS[
'path_to_save_actor']
THIRD_TRAIN_KWARGS[
'path_to_save_actor'] = 'BEST_ACTOR_FROM_WARP_CRITIC_AFTER_ADDITIONAL_PHASE_1'
THIRD_TEST_KWARGS = dict(THIRD_TRAIN_KWARGS)
del THIRD_TEST_KWARGS['logging_frequency']
del THIRD_TEST_KWARGS['path_to_save_actor']
train(**FIRST_TRAIN_KWARGS)
tf.reset_default_graph()
test(**FIRST_TEST_KWARGS)
tf.reset_default_graph()
train(**SECOND_TRAIN_KWARGS)
tf.reset_default_graph()
test(**SECOND_TEST_KWARGS)
tf.reset_default_graph()
train(**THIRD_TRAIN_KWARGS)
tf.reset_default_graph()
test(**THIRD_TEST_KWARGS)
tf.reset_default_graph()
exit()
# print('\n\n\nchange from multivae!!!!!!\n\n\n')
print("First, train just the actor")
train(
if args.net == 'resnet':
model = net.wide_resnet(d_names.values(), args.pretrained, args.frozen,
args.old)
elif args.net == 'piggyback':
model = net.piggyback_net(d_names.values(), args.pretrained, args.old)
else:
raise ValueError(
'The network you specified is not included in our model.'
'Please specify one of the following: resnet, piggyback or quantized.'
)
accuracy = 0
if args.dataset in d_names.keys():
if not args.test:
accuracy = training.train(model,
args.dataset,
args.prefix,
mirror=args.mirror,
bn=args.bn,
scaling=args.scaling,
epochs=args.epochs,
visdom_env=args.visdom)
else:
accuracy = training.test(model, args.dataset)
if args.output:
file = open(args.output, "a")
file.write("Dataset: " + args.dataset + " " + "Top1= " +
str(accuracy) + "\n")
file.close()
