def train(traindatafile, no_line_types = False):
"""Takes in file with training data, uses that for training purposes."""
print("Training...")
# the actual training happens here
training.train(dictionary.read_input(traindatafile),
monograms, bigrams, digrams, line_types)
print("Storing...")
# stores data to pickle files
if not no_line_types:
linetypes_file = open("line_types.p", "wb")
pickle.dump(line_types, linetypes_file)
linetypes_file.close()
(monograms_file, bigrams_file, digrams_file) = (open("monograms.p", "wb"),
open("bigrams.p", "wb"), open("digrams.p", "wb"))
pickle.dump(monograms, monograms_file)
pickle.dump(bigrams, bigrams_file)
pickle.dump(digrams, digrams_file)
monograms_file.close()
bigrams_file.close()
digrams_file.close()
print("Done!")
def main_test(model_file, fasta_file, args = None):
print( " *** Training model ***" )
model = dotio.load( model_file )
print(type(model))
training.train(fasta_file, model, [], 0.05)
#model.debug_print()
dotio.WriteDOT(model, "{}_train_vs_effectors.faa.dot".format(model_file) )
dotio.store(model, "{}_train_vs_effectors.faa.pickle".format(model_file) )
def main_test(fasta_file, repeats, args = None):
perturb_amount = 0.8
rnd = _rng()
for i in repeats:
model = hmmdsl_py.Model()
InitializePrototype(model)
#perturb = hmmdsl_py.Perturb(model)
#perturb.perturb_emissions(perturb_amount)
#for k in range(2,14+2):
# model.SetMu (k, rnd()*3.0 + 1.0 )
dotio.WriteDOT(model, "hsmm_local_prototype_04.dot" )
dotio.store(model, "hsmm_local_prototype_04.pickle" )
backgroundmodel = hmmdsl_py.Model()
InitializePrototype(backgroundmodel, True)
dotio.WriteDOT(backgroundmodel, "hsmm_local_prototype_04_background.dot" )
dotio.store(backgroundmodel, "hsmm_local_prototype_04_background.pickle" )
return
#model.split_state( 4, 0.7 )
#dotio.WriteDOT(model, "model_split.dot" )
#dotio.store(model, "model_split.pickle" )
training.train(fasta_file, model, [])
#model.debug_print()
dotio.WriteDOT(model, "hsmm14_perturb{}_vs_effectros.faa.dot".format(i) )
dotio.store(model, "hsmm14_perturb{}_vs_effectors.faa.pickle".format(i) )
def main():
# spawn theano vars
xl = T.matrix('xl')
xr = T.matrix('xr')
y = T.ivector('y')
learning_rate = T.scalar('learning_rate')
trng = RandomStreams(4321)
# use test values
"""
import numpy as np
batch_size = 10
theano.config.compute_test_value = 'raise'
xl.tag.test_value = np.random.randn(batch_size, 392).astype(floatX)
xr.tag.test_value = np.random.randn(batch_size, 392).astype(floatX)
y.tag.test_value = np.random.randint(8, size=batch_size).astype(np.int32)
learning_rate.tag.test_value = 0.5
"""
# build cgs
model_l, model_r, model_b = build_model(
xl, xr, y, learning_rate, trng=trng,
**options)
# compile
opt = get_optimizer(options['optimizer'])
f_train_l = opt(learning_rate, model_l, [xl, y])
f_train_r = opt(learning_rate, model_r, [xr, y])
f_train_b = opt(learning_rate, model_b, [xl, xr, y], return_alpha=True)
# compile validation/test functions
f_valid_l = theano.function([xl, y], [model_l.cost, model_l.acc])
f_valid_r = theano.function([xr, y], [model_r.cost, model_r.acc])
f_valid_b = theano.function([xl, xr, y], [model_b.cost, model_b.acc])
# training loop
train(f_train_l, f_train_r, f_train_b, f_valid_l, f_valid_r, f_valid_b,
xl, xr, y, **options)
def hyperparameter_search(number_of_combinations):
embedding_dimension = [2, 16, 64, 128]
number_of_hidden_layers = [
2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16
]
hidden_layer_dimension = [2, 16, 64, 128]
activation_function = [nn.ReLU, nn.Sigmoid, nn.Tanh]
number_of_training_epochs = [5, 10, 20]
loss_function_choice = [
nn.NLLLoss, nn.L1Loss, nn.CrossEntropyLoss, nn.MSELoss
]
optimizer_choice = [optim.SGD, optim.Adam]
learning_rate = [0.001, 0.01, 0.1, 1]
params_results = []
#TODO - Memorize each choice of params, so no combination is selected more than once
for i in range(number_of_combinations):
params = {
"embedding_dimension": choice(embedding_dimension),
"number_of_hidden_layers": choice(number_of_hidden_layers),
"hidden_layer_dimension": choice(hidden_layer_dimension),
"activation_function": choice(activation_function),
"number_of_training_epochs": choice(number_of_training_epochs),
"loss_function_choice": choice(loss_function_choice),
"optimizer_choice": choice(optimizer_choice),
"learning_rate": choice(learning_rate)
}
print("Searching with following params:s")
print(params)
validation_accuracy = train(**params)
print("")
params_results.append((params, validation_accuracy))
params_results.sort(key=lambda x: x[1])
print("Best found params:")
print(params_results[0][0])
print("")
print("Accuracy on best found params")
print(params_results[0][1])
print("")
return params_results
def objective(options=None):
# Initialize the best validation loss, which is the value to be minimized by the network
best_val_loss = float("Inf")
# Define hyperparameters
lr = 0.001
dropout = 0.3
batch_size = 128
print(f"Learning rate: {lr}")
print(f"Dropout: {dropout}")
print(f"Batch size: {batch_size}")
# Use CUDA if GPU is available, else CPU
use_cuda = options["use_cuda"] and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
print(f"Using device {device}")
# Obtain the MNIST train and validation loaders using a helper function
train_loader, val_loader = get_mnist_dataloaders(options["data_path"], batch_size)
# Initialize network
model = Net(dropout=dropout).to(device)
# Learning rate optimizer
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = StepLR(optimizer, step_size=1, gamma=0.7)
# Network training & validation loop
for epoch in range(0, options["epochs"]):
avg_train_loss = train(
options, model, device, train_loader, optimizer, epoch
)
avg_val_loss = validate(model, device, val_loader)
if avg_val_loss <= best_val_loss:
best_val_loss = avg_val_loss
# Print intermediate validation & training loss
print(f"Epoch {epoch + 1} of {options['epochs']} --- average train loss: {avg_train_loss} --- average validation loss: {avg_val_loss}")
scheduler.step()
# Return the best validation loss
return best_val_loss
def solver_to_results(solver_constructor: Callable[[], Solver],
bandit: NArmedBandit,
num_trainings) -> List[TrainResult]:
"""
Takes a solver constructor and trains it NUM_TRAININGS times on the given bandit.
:param solver_constructor: A callable that creates a solver
:param bandit: The bandit to train on
:param num_trainings: Specifies how often the solver is trained on the bandit
:return: The results of the training
"""
results = []
solver_name = str(solver_constructor())
for _ in tqdm(range(num_trainings),
desc=f'training solver {solver_name}: '):
solver = solver_constructor()
result = train(bandit, solver, NUM_PLAYS_PER_TRAINING)
results.append(result)
return results
def node(test_node, test_leaf):
node_dataset, leaf_dataset, test_dataset = prepare_dataset.prepare_dataset(
[[1], [7]])
PATHS = ['./node_net.pth', './left_net.pth', './right_net.pth']
if test_node:
for h in range(1):
training.train(leaf_dataset, PATHS[1])
training.train(leaf_dataset, PATHS[2])
for i in range(1, 3, 1):
print(f'Number of epochs: {i}')
node = Node.Node(node_dataset, PATHS[1], PATHS[2], PATHS[0])
node.train(i)
testing_node.test(test_dataset, PATHS)
#print('NEW TRY')
if test_leaf:
training.train(leaf_dataset, PATHS[1])
training.train(leaf_dataset, PATHS[2])
testing_leaf.test(test_dataset, PATHS[1])
testing_leaf.test(test_dataset, PATHS[2])
def main():
global training_path, query_path, result_path, training_data_path, csv_path
cur_path = os.path.dirname(os.path.abspath(__file__))
data_path = os.path.join(cur_path, "data")
training_path = os.path.join(data_path, training_file)
query_path = os.path.join(data_path, query_file)
result_path = os.path.join(data_path, result_file)
training_data_path = os.path.join(data_path, training_data_file)
csv_path = os.path.join(data_path, csv_file)
start_time = timeit.default_timer()
args, predict_fun, feature_fun = handle_args()
load_flag = args.load
feature_num = args.num_feature
compare_flag = args.compare
if compare_flag:
compare()
else:
dic = None
if load_flag:
with open(training_data_path, "rb") as f:
dic = pickle.load(f)
else:
dic = training.train(training_path)
with open(training_data_path, "wb") as f:
pickle.dump(dic, f)
if feature_fun != None:
feature_selection.select_feature(dic, feature_num, feature_fun)
else:
print("-"*60)
print("No feature selection!")
classifier.nbc(query_path, result_path, dic, predict_fun)
end_time = timeit.default_timer()
print("Total time: {:.3} sec".format(end_time - start_time))
result = accuracy(query_path, result_path)
def modelSvm(trainingImagePath, k):
t1 = cv2.getTickCount()
trainingImage = cv2.imread(trainingImagePath)
trainingImage = cv2.cvtColor(trainingImage, cv2.COLOR_BGR2LAB)
m, n, _ = trainingImage.shape
print "Color Quantization : "
#Preprocessing variable from image
l = trainingImage[:, :, 0]
a = trainingImage[:, :, 1]
b = trainingImage[:, :, 2]
scaler = preprocessing.MinMaxScaler()
pca = PCA(32)
qab, centroid = quantization(a, b, k)
print centroid
# with open('./../temp/'+testCaseNumber+'/centroids', 'w') as csvfile:
# writer = csv.writer(csvfile)
# [writer.writerow(r) for r in centroid]
t2 = cv2.getTickCount()
t = (t2 - t1) / cv2.getTickFrequency()
print "Time for quantization : ", t, " seconds"
print "Feature extraction : "
feat, classes = getKeyPointFeatures(l, qab)
print "Length of feature descriptor before PCA : ", len(feat[0])
feat = scaler.fit_transform(feat)
feat = pca.fit_transform(feat)
print "Length of feature descriptor after PCA : ", len(feat[0])
t3 = cv2.getTickCount()
t = (t3 - t2) / cv2.getTickFrequency()
print "Time for feature extraction : ", t, " seconds"
print "Training : "
svm_classifier = train(feat, classes, k)
t4 = cv2.getTickCount()
t = (t4 - t3) / cv2.getTickFrequency()
print "Time for training: ", t, " seconds"
return (centroid, svm_classifier, scaler, pca)
def go(batch_size, epochs, dataset, input_model):
num_classes = 10
x_train, y_train, x_test, y_test, input_shape = get_data(dataset, num_classes)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
model = train(get_model(input_model),
x_train, y_train, batch_size, epochs, x_test, y_test)
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
keras_file = "mnist_pruned.h5"
print('Saving model to: ', keras_file)
tf.keras.models.save_model(model, keras_file, include_optimizer=False)
def main(fragment_file, lead_file):
fragment_mols = read_file(fragment_file)
lead_mols = read_file(lead_file)
fragment_mols += lead_mols
fragments, used_mols = get_fragments(fragment_mols)
encodings, decodings = get_encodings(fragments)
save_decodings(decodings)
lead_mols = np.asarray(fragment_mols[-len(lead_mols):])[used_mols[-len(lead_mols):]]
X = encode_list(lead_mols, encodings)
actor, critic = build_models(X.shape[1:])
X = clean_good(X, decodings)
history = train(X, actor, critic, decodings)
np.save("History/history.npy", history)
def mlp_predictions(randomsearch=False):
if randomsearch:
layersizes = random_search.architecture_search() # 7,32,32,16,1
hparams = random_search.hparams_search(layersizes) # 0.005, 16
else:
layersizes = [7, 32, 32, 16, 1]
hparams = [0.005, 16]
hparams_setting = {
"epochs": 1000,
"batchsize": hparams[1],
"learningrate": hparams[0],
"history": 1
}
model_design = {"layer_sizes": layersizes}
X, Y, Y_Preles = preprocessing.preprocessing()
arr = []
years = [
2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011,
2012
]
for year in years:
x, y, y_nn = preprocessing.split_by_year(X, Y, Y_Preles, years=[year])
#x = x.drop(["year"], axis=1)
running_losses = training.train(hparams_setting, model_design,
x.to_numpy(), y.to_numpy(), "D1")
preds, mae, nse = prediction.predict(hparams_setting, model_design,
x.to_numpy(), y.to_numpy(), "D1")
arr.append(preds)
arr = np.concatenate(arr, axis=1)
return arr
def objective(trial):
relevant_metric = 'val_loss'
args = get_parser().parse_args()
args.experiment_name = trial.study.study_name
# hyperparameter can be modified here!
args.learning_rate = trial.suggest_categorical('learning_rate',
[1e-2, 5e-3, 1e-3])
args.lr_warmup_steps = trial.suggest_categorical('lr_warmup_steps',
[1000, 2000, 4000])
args.accumulate_grad_batches = trial.suggest_categorical(
'accumulate_grad_batches', [32, 64, 128])
args.positional_encoding_dropout = trial.suggest_uniform(
'positional_encoding_dropout', 0.1, 0.3)
args.transformer_dropout = trial.suggest_uniform('transformer_dropout',
0.1, 0.3)
args.version = (f'lr{args.learning_rate}'
f'_ws{args.lr_warmup_steps}'
f'_agb{args.accumulate_grad_batches}'
f'_pd{args.positional_encoding_dropout}'
f'_td{args.transformer_dropout}')
trainer = train(
args,
custom_callbacks=[
PyTorchLightningPruningCallback(trial, monitor=relevant_metric),
EarlyStopping(
monitor=relevant_metric,
min_delta=0.5,
patience=3,
verbose=False,
mode='min',
),
],
)
return find_best_checkpoint(trainer.logger.log_dir)
def main():
try:
### 학습모델 불러오기
model = tf.keras.models.load_model('my_model.h5')
except (FileNotFoundError, OSError) as e:
model = training.train()
result = []
ocrs = imgs.ReadImg(sys.argv[1])
trainImg = []
for e in ocrs:
trainImg.append(e.data)
result.append(model.predict(e.data).argmax())
print('predict: ', result)
print('Is predict right? (1.right, 2.wrong)')
right = input()
if int(right) != 1:
print('Input Answer: ')
ans = input()
darr = digit(ans)
for idx, e in enumerate(trainImg):
model.fit(e, [darr[idx]])
model.save('my_model.h5')
from training import pretrain, train_discriminator, train
from data import sample_groups
import torch
n_target_samples = 7
plot_accuracy = True
if __name__ == '__main__':
cuda = torch.cuda.is_available()
groups, data = sample_groups(n_target_samples=n_target_samples)
encoder, classifier = pretrain(data, cuda=cuda, epochs=20)
discriminator = train_discriminator(encoder, groups, n_target_samples=n_target_samples, epochs=50, cuda=cuda)
train(encoder, discriminator, classifier, data, groups, n_target_samples=n_target_samples, cuda=cuda, epochs=150, plot_accuracy=plot_accuracy)
pickle_in = open("../../Q1_data/data.pkl", "rb")
rawDict = pickle.load(pickle_in)
xDict , yDict = np.hsplit(rawDict, 2)
xTrain, xTest, yTrain, yTest = train_test_split(xDict, yDict, test_size=0.1, random_state=57)
temp = np.concatenate((xTrain, yTrain), axis =1)
np.random.shuffle(temp)
xTrain, yTrain = np.hsplit(temp, 2)
dx , dy = distribute(xTrain, yTrain)
bias, variance = train(dx, dy, xTest, yTest)
table = np.vstack(([i for i in range(1, 100)], bias, variance)).T
print(tabulate(table, headers=["Degree", "Bias", "Variance"], tablefmt='psql'))
plt.plot(bias, color = 'red',label='bias')
plt.plot(variance, color = 'blue', label='variance')
plt.legend()
plt.show()
# now that we have the bias and the variance ,we can plot this data or whatever
if not os.path.exists(os.path.join(args.model_dir, args.model)):
os.makedirs(os.path.join(args.model_dir, args.model))
transformer = transform_train if (args.mode == "train") else transform_val
data_loader = dataloader(
transform=transformer,
mode=args.mode,
batch_size=args.batch_size,
vocab_threshold=args.vocab_threshold,
from_vocab_file=args.from_vocab_file,
vocab_file=args.vocab_file,
data_path=args.data_dir,
image_data_unavailable=args.data_unavailable,
model=args.model,
)
vocab_size = len(data_loader.dataset.vocab)
print("vocabulary size: {}".format(vocab_size))
if args.model == "lstm":
encoder, decoder = lstm(vocab_size)
elif args.model == "attention":
encoder, decoder = attention_lstm(vocab_size)
if args.mode == "train":
train(encoder, decoder, data_loader, vocab_size, args)
elif args.mode == "val":
validation(encoder, decoder, data_loader, vocab_size, args)
if opt.conv_encoder:
model = meta_modules.ConvolutionalNeuralProcessImplicit2DHypernet(
in_features=img_dataset.img_channels,
out_features=img_dataset.img_channels,
image_resolution=image_resolution)
else:
model = meta_modules.NeuralProcessImplicit2DHypernet(
in_features=img_dataset.img_channels + 2,
out_features=img_dataset.img_channels,
image_resolution=image_resolution)
model.cuda()
# Define the loss
loss_fn = partial(loss_functions.image_hypernetwork_loss, None, opt.kl_weight,
opt.fw_weight)
summary_fn = partial(utils.write_image_summary_small, image_resolution, None)
root_path = os.path.join(opt.logging_root, opt.experiment_name)
training.train(model=model,
train_dataloader=dataloader,
epochs=opt.num_epochs,
lr=opt.lr,
steps_til_summary=opt.steps_til_summary,
epochs_til_checkpoint=opt.epochs_til_ckpt,
model_dir=root_path,
loss_fn=loss_fn,
summary_fn=summary_fn,
clip_grad=True)
def main(modeltype, batch_size, iterations, learning_rate, segment_len, segment_step, train_save_params, dataset=["dataset"], outputdir="output", validation=None, validation_generate_ct=1, resume=None, resume_auto=False, check_nan=False, generate=False, generate_over=None, auto_connectome_keys=None, **model_kwargs):
generate = generate or (generate_over is not None)
should_setup = not generate
unroll_batch_num = None if generate else training.BATCH_SIZE
for dataset_dir in dataset:
if os.path.samefile(dataset_dir,outputdir):
print("WARNING: Directory {} passed as both dataset and output directory!".format(outputdir))
print("This may cause problems by adding generated samples to the dataset directory.")
while True:
result = input("Continue anyway? (y/n)")
if result == "y":
break
elif result == "n":
sys.exit(0)
else:
print("Please type y or n")
if generate_over is None:
training.set_params(batch_size, segment_step, segment_len)
leadsheets = [training.filter_leadsheets(training.find_leadsheets(d)) for d in dataset]
else:
# Don't bother loading leadsheets, we don't need them
leadsheets = []
if validation is not None:
validation_leadsheets = training.filter_leadsheets(training.find_leadsheets(validation))
else:
validation_leadsheets = None
m = builders[modeltype].build(should_setup, check_nan, unroll_batch_num, **model_kwargs)
m.set_learning_rate(learning_rate)
if resume_auto:
paramfile = os.path.join(outputdir,'final_params.p')
if os.path.isfile(paramfile):
with open(os.path.join(outputdir,'data.csv'), 'r') as f:
for line in f:
pass
lastline = line
start_idx = lastline.split(',')[0]
print("Automatically resuming from {} after iteration {}.".format(paramfile, start_idx))
resume = (start_idx, paramfile)
else:
print("Didn't find anything to resume. Starting from the beginning...")
if resume is not None:
start_idx, paramfile = resume
start_idx = int(start_idx)
m.params = pickle.load( open(paramfile, "rb" ) )
else:
start_idx = 0
if not os.path.exists(outputdir):
os.makedirs(outputdir)
if generate:
print("Setting up generation")
m.setup_produce()
print("Starting to generate")
start_time = time.process_time()
if generate_over is not None:
source, divwidth = generate_over
if divwidth == 'full':
divwidth = 0
elif divwidth == 'debug_firststep':
divwidth = -1
elif len(divwidth)>3 and divwidth[-3:] == 'bar':
divwidth = int(divwidth[:-3])*(constants.WHOLE//constants.RESOLUTION_SCALAR)
else:
divwidth = int(divwidth)
ch,mel = leadsheet.parse_leadsheet(source)
lslen = leadsheet.get_leadsheet_length(ch,mel)
if divwidth == 0:
batch = ([ch],[mel]), [source]
elif divwidth == -1:
slices = [leadsheet.slice_leadsheet(ch,mel,0,1)]
batch = list(zip(*slices)), [source]
else:
slices = [leadsheet.slice_leadsheet(ch,mel,s,s+divwidth) for s in range(0,lslen,divwidth)]
batch = list(zip(*slices)), [source]
training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True, batch=batch)
else:
training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True)
end_time = time.process_time()
print("Generation took {} seconds.".format(end_time-start_time))
else:
training.train(m, leadsheets, iterations, outputdir, start_idx, train_save_params, validation_leadsheets=validation_leadsheets, validation_generate_ct=validation_generate_ct, auto_connectome_keys=auto_connectome_keys)
pickle.dump( m.params, open( os.path.join(outputdir, "final_params.p"), "wb" ) )
#module 2
from prep_cifar10 import data_to_categorical
y_train, y_test = data_to_categorical(y_train, y_test, num_classes)
#module 3
from shape_model import add_model
model = add_model(num_classes, x_train)
#module 3
from shape_model import opt
opt = opt()
#module 4
#Processing / Training
from training import train
x_train, x_test, model, opt = train(model, opt, x_train, x_test)
#module 4
from training import augment
x_train, y_train, x_test, y_test = augment(data_augmentation, x_train, y_train,
batch_size, epochs, x_test, y_test,
model)
#module 5
#OUTPUT:
# Save trained model
from output import save_cnn
model, model_path = save_cnn(model, save_dir, model_name)
from output import cnn_stats
model = cnn_stats(model, model_path, x_test, y_test)
def main():
# Initialization - argument parsing, logging and stage preparations
logging.basicConfig(format='%(asctime)s %(message)s',
filename='logs.log',
level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
args = parse_args()
out_folder = args.out_folder if args.out_folder.endswith(
'/') else args.out_folder + '/'
if not os.path.exists(out_folder):
os.makedirs(out_folder)
# Training
logging.info("Preparing images for training...")
_, gray_img, _, samples, labels = prepare_for_training(
args.training_image, kernels)
logging.info("Starting training process...")
weights = train(samples,
labels,
num_of_epochs=args.num_of_epochs,
batch_size=args.batch_size)
logging.info("Training done.")
# Training results
results = np.round(weights[-1, :], 2)
logging.info("Results: ")
for result, kernel, name in zip(results, kernels, kernel_names):
logging.info(f"Original {name} filter: \n{kernel}")
logging.info(f"Trained {name} filter: \n{result}")
out_image = out_folder + f"{name}/training_image_compare.jpg"
logging.info(f"Saving training comparition to {out_image}\n")
compare_images(convolution(gray_img, result),
convolution(gray_img, kernel),
out_image=out_image)
# Results on validation image
_, _, validation_image = load_normalize_and_grayscale(
args.validation_image)
logging.info(
f"Acceptance error is set to {args.acceptance_error}% of distance between highest and lowest possible value"
)
for result, kernel, name in zip(results, kernels, kernel_names):
out_image = out_folder + f"{name}/validation_image_compare.jpg"
logging.info(f"Saving validation comparition to {out_image}")
perc_of_similarity = compare_images(
convolution(validation_image, result),
convolution(validation_image, kernel),
out_image=out_image,
acceptance_error=args.acceptance_error)
logging.info(
f"Similarity between validation image filtered with trained and original {name} kernel is {perc_of_similarity}%"
)
# Animated matrix
if args.create_gifs:
for i, name in zip(range(weights.shape[1]), kernel_names):
out_image = out_folder + f"{name}/animated_training_matrix.gif"
logging.info(f"Creating matrix gif for {name}...")
create_matrix_gif(weights[:, i],
save_folder=out_folder + "gif_images/",
out_image=out_folder +
f"{name}/animated_training_matrix.gif")
else:
logging.info("Skipping gif creation.")
parms['flow_layers'] = c[3]
parms['flow_units'] = c[4]
parms['flow_type'] = c[5]
parms['autoregressive'] = c[6]
parms['n_pixelcnn_layers'] = c[7]
parms['anneal'] = c[8]
parms['n_z'] = c[9]
parms['n_mixtures'] = c[10]
parms['learning_rate'] = c[11]
for name, model in models.items():
name = experiment_name + '_nz_' + str(parms['n_z']) + '_nmix_' + str(parms['n_mixtures']) \
+ '_lr_' + str(parms['learning_rate'])
if parms['flow']:
name += "_flow_" + str(parms['flow_layers']) + "_" + str(parms['flow_units']) + "_" + parms['flow_type']
if parms['autoregressive']:
name += "_ar_" + str(parms['n_pixelcnn_layers'])
if parms['anneal'] < 0:
name += "_anneal_" + str(parms['anneal'])
train(name=name, model=model, parameters=parms, data=data, tracker=tracker)
print("Finished :)", flush=True)
BREAK_EARLY = False
BATCH_SIZE = 500
for data_subdir in ['ml-20m', 'netflix-prize', 'msd']:
actor_path = "WMFVAE_ACTOR_TRAIN_{}".format(data_subdir)
train(
model_class='wmf_vae',
data_subdir=data_subdir,
n_epochs_pred_only=100,
n_epochs_ac_only=0,
n_epochs_pred_and_ac=0,
max_kl=0.05,
ac_reg_loss_scaler=0.0,
actor_reg_loss_scaler=0.0001,
evaluation_metric="NDCG",
logging_frequency=50,
batch_size=BATCH_SIZE,
break_early=BREAK_EARLY,
verbose=False,
positive_weights=5.0,
version_tag="FULL_RUN_ON_OTHER_DATASETS",
path_to_save_actor=actor_path,
log_critic_training_error=False,
)
print("Now, hopefully on to testing...")
test(
model_class='wmf_vae',
data_subdir=data_subdir,
return train_iterator, valid_iterator, test_iterator, text_field.vocab
# load data
print("\nLoading data...")
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
text_field = data.Field(tokenize='spacy', lower=True, include_lengths=True, batch_first=True)
label_field = data.Field(sequential=False, use_vocab=False, batch_first=True, dtype=torch.float)
docid_field = data.RawField()
train_iter, dev_iter, test_iter, text_voca = fall_data(docid_field, text_field, label_field, device=-1, repeat=False)
# train or eval
if args.test:
best_model = model.LSTM().to(device)
optimizer = optim.Adam(best_model.parameters(), lr=args.lr)
t.load_checkpoint(destination_folder + '/model.pt', best_model, optimizer)
t.evaluate(best_model, test_iter)
else:
print('start training')
wandb.init()
wandb.watch(model)
model = model.LSTM(text_voca).to(device)
optimizer = optim.Adam(model.parameters(), lr=args.lr)
eval_every = len(train_iter) // 2
t.train(model=model, optimizer=optimizer, train_loader=train_iter, valid_loader=dev_iter, num_epochs=args.epochs, eval_every = eval_every, file_path= destination_folder, device=device)
# -*- coding: utf-8 -*- """ Spyder Editor This is a temporary script file. """ from model import create_model from DataGeneration import dataGeneration from Checkpoint import createCheckpoint from Performance import plot, calculate from training import train from Predict import predict loadPreviouslySavedWeights = 1 network = create_model() dataSet = dataGeneration() cp_callback = createCheckpoint() train(network, dataSet, cp_callback, loadPreviouslySavedWeights) calculate(network, dataSet) predict(network, dataSet)
import normalization
import training
import testing
import time
start = time.time()
common_words = normalization.normalize_text("test/sport.txt")
print("text normalize")
''' for training
training.train(common_words, "mathematics")
print(time.time() - start)
'''
''' for testing '''
testing.classify(common_words)
print(time.time() - start)
def random_forest(path_train,
path_test,
name_identifiers,
name_targets,
features,
delimiter,
num_cores=1):
'''
this method performs the training,testing and evaluation of the
random forest algorithm.
@type path_train: str
@param path_train: full path to csv (first line contains headers).
delimiter should be specified with param delimiter
@type path_test: str
@param path_test: full path to csv (first line contains headers).
delimiter should be specified with param delimiter
@type name_identifiers: str
@param name_identifiers: name of column containing identifiers
@type name_targets: str
@param name_targets: name of column containing targets
@type features: str
@param features: list of features to be used
@type delimiter: str
@param delimiter: delimiter used in csv. tested with "," and "\t"
@type num_cores: int
@param num_cores: [optional]: num of cores you want to use in training
@rtype: tuple
@return: (output_classifier,evaluation). both are dicts. output_classifier
maps identifier -> output_classifier. evaluation maps all kinds of evaluation metrics
to floats.
'''
#call train using class
identifiers_training,target_training,rf = train(path_train,
name_identifiers,
name_targets,
features,
output_path_model=None,
cores=num_cores,
the_delimiter=delimiter)
#call test using class
identifiers_test,target,prediction = test(path_test,
name_identifiers,
name_targets,
features,
loaded_rf_model=rf,
path_rf_model=None,
the_delimiter=delimiter)
#evaluate
classifier_output,evaluation = evaluate(target, prediction, identifiers_test)
return classifier_output,evaluation
'epochs': 50,
'batch_size': 128,
'lr': 1e-3,
'decay': 1e-2,
'gamma': 0.8,
'step_size': 5,
'num_bins': 50,
'prune_milestones': [1000],
'prune_sigmas': [1],
'prune_gamma': 10,
'ignore_below': True,
'p_i': 5e-1,
'p_f': 0.99,
'T': 100
}
model = LeNet_KG(in_chan=1, out_chan=10, imsize=28, kernel_size=5)
# model = CNN()
model, df = train(model, trainset, valset, param_dict)
model.eval()
with torch.no_grad():
alltrainloader = torch.utils.data.DataLoader(trainset,
batch_size=len(trainset),
shuffle=False)
for b, (x, y) in enumerate(alltrainloader):
v, s = model(x)
c = v.argmax(dim=1)
std = torch.empty((s.size()[0]), requires_grad=False)
for i in range(s.size()[0]):
std[i] = torch.sqrt(F.softplus(s[i, c[i]]))
def run():
USE_CUDA = torch.cuda.is_available()
FLAGS.config = 'example_seq2seq.json'
config_path = os.path.join("experiments", FLAGS.config)
print(FLAGS.config)
if not os.path.exists(config_path):
raise FileNotFoundError
with open(config_path, "r") as f:
config = json.load(f)
config["gpu"] = torch.cuda.is_available()
writer = SummaryWriter('experiments/finally')
# dataset = ToyDataset(5, 15)
# eval_dataset = ToyDataset(5, 15, type='eval')
dataset = Toy_Numbers(10)
eval_dataset = Toy_Numbers(10, train=False)
BATCHSIZE = 32
train_loader = data.DataLoader(dataset,
batch_size=BATCHSIZE,
shuffle=False,
collate_fn=pad_collate,
drop_last=True)
eval_loader = data.DataLoader(eval_dataset,
batch_size=BATCHSIZE,
shuffle=False,
collate_fn=pad_collate,
drop_last=True)
config["batch_size"] = BATCHSIZE
# Models
model = Seq2Seq(config)
model = model.float()
# dataiter = iter(train_loader)
# sample_input= dataiter.next()
# writer.add_graph(model, sample_input)
# writer.close()
if USE_CUDA:
model = model.cuda()
# Optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=config.get("learning_rate", .001))
print("=" * 60)
print(model)
print("=" * 60)
for k, v in sorted(config.items(), key=lambda i: i[0]):
print(" (" + k + ") : " + str(v))
print()
print("=" * 60)
print("\nInitializing weights...")
for name, param in model.named_parameters():
if 'bias' in name:
torch.nn.init.constant_(param, 0.0)
elif 'weight' in name:
torch.nn.init.xavier_normal_(param)
for epoch in range(FLAGS.epochs):
run_state = (epoch, FLAGS.epochs, FLAGS.train_size)
# Train needs to return model and optimizer, otherwise the model keeps restarting from zero at every epoch
model, optimizer = train(model, optimizer, train_loader, run_state,
writer)
# print("losses", l_list)
# for i in l_list:
# # print(i)
# writer.add_scalar('Loss/train',i)
evaluate(model, eval_loader, writer)
def run(env_id, render, render_eval, seed, noise_type, layer_norm, evaluation,
**kwargs):
# Configure things.
rank = MPI.COMM_WORLD.Get_rank()
if rank != 0:
logger.set_level(logger.DISABLED)
# Create envs.
env = RobotPath.env(render)
env = bench.Monitor(env,
logger.get_dir()
and os.path.join(logger.get_dir(), str(rank)),
allow_early_resets=True)
if evaluation and rank == 0:
eval_env = RobotPath.env(render_eval)
eval_env = bench.Monitor(eval_env,
os.path.join(logger.get_dir(), 'gym_eval'),
allow_early_resets=True)
env = bench.Monitor(env, None, allow_early_resets=True)
else:
eval_env = None
# Parse noise_type
action_noise = None
param_noise = None
nb_actions = env.action_space.shape[-1]
for current_noise_type in noise_type.split(','):
current_noise_type = current_noise_type.strip()
if current_noise_type == 'none':
pass
elif 'adaptive-param' in current_noise_type:
_, stddev = current_noise_type.split('_')
param_noise = AdaptiveParamNoiseSpec(
initial_stddev=float(stddev),
desired_action_stddev=float(stddev))
elif 'normal' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = NormalActionNoise(mu=np.zeros(nb_actions),
sigma=float(stddev) *
np.ones(nb_actions))
elif 'ou' in current_noise_type:
_, stddev = current_noise_type.split('_')
action_noise = OrnsteinUhlenbeckActionNoise(
mu=np.zeros(nb_actions),
sigma=float(stddev) * np.ones(nb_actions))
else:
raise RuntimeError(
'unknown noise type "{}"'.format(current_noise_type))
# Configure components.
#print("----------------")
#print(env.observation_space)
env = gym.wrappers.FlattenDictWrapper(
env, ['desired_goal', 'achieved_goal', 'observation'])
#print(env.observation_space)
# print(env.observation_space['start_point'].shape)
#print("----------------")
memory = Memory(limit=int(1e6),
action_shape=env.action_space.shape,
observation_shape=env.observation_space.shape)
critic = Critic(layer_norm=layer_norm)
actor = Actor(nb_actions, layer_norm=layer_norm)
# Seed everything to make things reproducible.
seed = seed + 1000000 * rank
logger.info('rank {}: seed={}, logdir={}'.format(rank, seed,
logger.get_dir()))
tf.reset_default_graph()
set_global_seeds(seed)
env.seed(seed)
if eval_env is not None:
eval_env.seed(seed)
# Disable logging for rank != 0 to avoid noise.
if rank == 0:
start_time = time.time()
training.train(env=env,
eval_env=eval_env,
param_noise=param_noise,
action_noise=action_noise,
actor=actor,
critic=critic,
memory=memory,
**kwargs)
env.close()
if eval_env is not None:
eval_env.close()
if rank == 0:
logger.info('total runtime: {}s'.format(time.time() - start_time))
'prepare'
x_train, y_train = train_dfs[cv_idx][features], train_dfs[cv_idx][target]
x_valid, y_valid = valid_dfs[cv_idx][features], valid_dfs[cv_idx][target]
x_test, y_test = test_dfs[cv_idx][features], test_dfs[cv_idx][target]
# add feature from baseline
x_train, x_valid, x_test, x_sub = add_group_feature(x_train, x_valid, x_test, test_df[features], y_train)
lgb_train, lgb_valid = lgb.Dataset(x_train, y_train, categorical_feature=[3, 4, 5], free_raw_data=False), lgb.Dataset(x_valid, y_valid, categorical_feature=[3, 4, 5], free_raw_data=False)
'train'
if tune:
params = training.tuning(lgb_train, lgb_valid, 100)
pd.to_pickle(params, 'params/{0}_cv{1}.pkl'.format(name, cv_idx))
model = training.train(lgb_train, lgb_valid, params)
score = training.evaluation(model, x_test, y_test)
scores.append(score)
model.save_model('model/{0}_cv{1}.txt'.format(name, cv_idx), num_iteration=model.best_iteration)
'predict'
pred = model.predict(x_sub)
pred = np.round(pred)
sub_df.iloc[:, 1] += pred
sub_df.iloc[:, 1] /= cv
sub_df.iloc[:, 1] = np.round(sub_df.iloc[:, 1]).astype(np.int8)
print(scores)
print(np.mean(scores))
#
if __name__ == '__main__':
model = 'VGG'
dataset = 'CIFAR-100'
pretrained = True
vggnet = torchvision.models.vgg11_bn(pretrained=pretrained, progress=True)
vggnet.classifier = torch.nn.Sequential(
torch.nn.Linear(in_features=25088, out_features=1024, bias=True),
torch.nn.ReLU(inplace=True), torch.nn.Dropout(p=0.5, inplace=False),
torch.nn.Linear(1024, 100))
train_dl, val_dl, test_dl = load_dataloaders_from_dataset(dataset)
optimizer = torch.optim.SGD(vggnet.parameters(),
lr=0.01,
momentum=0.9,
weight_decay=1e-3)
criterion = torch.nn.CrossEntropyLoss()
writer = SummaryWriter('runs/model={}_pretrained={}_dset={}'.format(
model, pretrained, dataset))
train(vggnet,
train_dl,
val_dl,
test_loader=test_dl,
optimizer=optimizer,
criterion=criterion,
device=device,
epochs=5000,
early_stopping=True,
writer=writer)
for i in range(0, 10):
#Train with the BW always to 0
model = Net()
model.type(torch.cuda.FloatTensor)
train_loader, valid_loader, test_loader = pixar_dataset(
0.8, 'addit', test=True
) #both work also, because of test=true, the mode option is useless here
use_gpu = True
n_epoch = 100
learning_rate = 0.002
history = train(model,
train_loader,
valid_loader,
n_epoch,
learning_rate,
use_gpu=use_gpu)
history.display()
test_acc, test_loss = test(model, test_loader, use_gpu=use_gpu)
print('Test:\n\tLoss: {}\n\tAccuracy: {}'.format(test_loss, test_acc))
#Train with Input Dropout in addit mode
model = Net()
model.type(torch.cuda.FloatTensor)
train_loader, valid_loader, test_loader = pixar_dataset(0.8,
'addit',
test=False)
use_gpu = True
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--output_dir",
default=None,
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written."
)
parser.add_argument(
"--model_name_or_path",
default="gpt2-medium",
type=str,
help="The model checkpoint for weights initialization.")
parser.add_argument(
"--load_from_multiple_models",
action='store_true',
help="Whether to load from multiple models with postfix -0 to -4.")
parser.add_argument(
"--block_size",
default=150,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens)."
)
parser.add_argument("--start_checkpoint",
default=-1,
type=int,
help="Start from checkpoint X.")
parser.add_argument("--num_sampled_sessions",
default=40,
type=int,
help="Sample X sessions in a total of 40.")
parser.add_argument("--per_gpu_train_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument(
'--gradient_accumulation_steps',
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass."
)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight deay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs."
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument('--save_steps',
type=int,
default=50,
help="Save checkpoint every X updates steps.")
parser.add_argument("--no_cuda",
action='store_true',
help="Avoid using CUDA when available")
parser.add_argument('--overwrite_output_dir',
action='store_true',
help="Overwrite the content of the output directory")
parser.add_argument('--seed',
type=int,
default=42,
help="random seed for initialization")
args = parser.parse_args()
args.simplifier = True # Train the simplifier
if os.path.exists(args.output_dir) and os.listdir(
args.output_dir) and not args.overwrite_output_dir:
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
device = torch.device(
"cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = torch.cuda.device_count()
args.device = device
# Setup logging
logging.basicConfig(
format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
datefmt='%m/%d/%Y %H:%M:%S',
level=logging.INFO)
logger.warning("device: %s, n_gpu: %s", device, args.n_gpu)
# Training
for i in range(0, len(utili.segments_name)):
# Set seed
utili.set_seed(args)
config_class, model_class, tokenizer_class = GPT2Config, GPT2LMHeadModel, GPT2Tokenizer
postfix = ('-' + str(i)) if args.load_from_multiple_models else ''
config = config_class.from_pretrained(args.model_name_or_path +
postfix) # gpt2 size
tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path +
postfix)
tokenizer.add_special_tokens(utili.special_tokens_dict)
if args.block_size <= 0:
args.block_size = tokenizer.max_len_single_sentence # Our input block size will be the max possible for the model
args.block_size = min(args.block_size,
tokenizer.max_len_single_sentence)
postfix += ('/checkpoint-' + str(args.start_checkpoint)) if (
args.start_checkpoint != -1) else ''
model = model_class.from_pretrained(args.model_name_or_path + postfix)
model.resize_token_embeddings(len(tokenizer)) # resize
assert tokenizer.sep_token == '<SEP>'
assert tokenizer.pad_token == '<PAD>'
assert tokenizer.bos_token == '<BOS>'
assert tokenizer.eos_token == "<EOS>"
logger.info(
"Added sep_token (id: %s), pad_token (id: %s), bos_token (id: %s) and eos_token (id: %s)",
tokenizer.sep_token_id, tokenizer.pad_token_id,
tokenizer.bos_token_id, tokenizer.eos_token_id)
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
train_segments = [utili.segments_name[x] for x in range(i)] + [
utili.segments_name[x]
for x in range(i + 1, len(utili.segments_name))
]
print("train_segments: {}".format(train_segments))
train_dataset = QueryRewriteDataset(train_segments, tokenizer, args,
args.num_sampled_sesions)
global_step, tr_loss = train(args,
train_dataset,
model,
tokenizer,
logger,
cross_validate_id=i)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
# Create output directory if needed
segment_output_dir = args.output_dir + '-' + str(i)
if not os.path.exists(segment_output_dir):
os.makedirs(segment_output_dir)
logger.info("Saving model checkpoint to %s", segment_output_dir)
model_to_save = model.module if hasattr(model, 'module') else model
model_to_save.save_pretrained(segment_output_dir)
tokenizer.save_pretrained(segment_output_dir)
torch.save(args, os.path.join(segment_output_dir, 'training_args.bin'))
del model
torch.cuda.empty_cache()
def anomaly(experiment_name,
dataset = "mnist",
bayesian_approximation = "dropout",
inside_labels = [0, 1],
num_epochs = 50,
batch_size = 128,
acc_threshold = 0.6,
weight_decay = 1e-5,
dropout_p = 0.5,
fc_layers = [512, 512],
plot = True):
"""
This methods trains a neural network classifier on a subset of classes.
After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies.
The anomalous classes are the ones that are not part of the training subset.
dataset = "mnist" or "cifar"
For MNIST we use a fully-connected MLP.
For CIFAR10 we use a convolutional net (similar to LeNet)
bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST
bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST.
inside_labels are the subset of trained classes, the other classes are only used for testing.
"""
n_out = len(inside_labels)
# Prepare Theano variables for inputs and targets
# Load the dataset
print("Loading data...")
if dataset == "mnist":
input_var = T.matrix('inputs')
target_var = T.ivector('targets')
n_in = [28*28]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels)
if bayesian_approximation == "dropout":
model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay)
elif bayesian_approximation == "variational":
model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size))
elif dataset == "cifar":
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
n_in = [3, 32, 32]
X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10(inside_labels)
model = models.convnet_dropout(input_var, target_var, n_in, n_out, dropout_p, weight_decay)
df = pd.DataFrame()
# Mini-batch training with ADAM
epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold)
# Mini-batch testing
acc, bayes_acc = training.test(model, X_test, y_test, batch_size)
df.set_value(experiment_name, "test_acc", acc)
df.set_value(experiment_name, "bayes_test_acc", bayes_acc)
# Uncertainty prediction
test_mean_std_bayesian = {x:[] for x in range(10)}
test_mean_std_deterministic = {x:[] for x in range(10)}
test_entropy_bayesian = {x:[] for x in range(10)}
test_entropy_deterministic = {x:[] for x in range(10)}
for i in range(len(X_test_all)):
bayesian_probs = model.probabilities(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
bayesian_entropy = model.entropy_bayesian(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in))
classical_probs = model.probabilities_deterministic(X_test_all[i][np.newaxis,:])[0]
classical_entropy = model.entropy_deterministic(X_test_all[i][np.newaxis,:])
predictive_mean = np.mean(bayesian_probs, axis=0)
predictive_std = np.std(bayesian_probs, axis=0)
test_mean_std_bayesian[y_test_all[i]].append(np.concatenate((predictive_mean, predictive_std)))
test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy)
test_entropy_deterministic[y_test_all[i]].append(classical_entropy)
test_mean_std_deterministic[y_test_all[i]].append(classical_probs)
# Plotting
if plot:
for k in sorted(test_mean_std_bayesian.keys()):
sns.plt.figure()
#sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k))
sns.plt.hist(test_entropy_bayesian[k], label = "Bayesian Entropy v1 for " + str(k))
#sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k))
#sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k))
sns.plt.legend()
sns.plt.show()
# Anomaly detection using simple threshold
def anomaly_detection_old(anomaly_score_dict, name, df):
threshold = np.logspace(-30, 1.0, 1000)
acc = {}
for t in threshold:
tp = 0.0
tn = 0.0
for l in anomaly_score_dict:
if l in inside_labels:
tp += (np.array(anomaly_score_dict[l]) < t).mean()
else:
tn += (np.array(anomaly_score_dict[l]) >= t).mean()
tp /= len(inside_labels)
tn /= 10.0 - len(inside_labels)
bal_acc = (tp + tn)/2.0
f1_score = 2.0*tp/(2.0 + tp - tn)
acc[t] = [bal_acc, f1_score, tp, tn]
print("{}\tscore\tthreshold\tTP\tTN".format(name))
sorted_acc = sorted(acc.items(), key= lambda x : x[1][0], reverse = True)
df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0])
df.set_value(experiment_name, name + ' bal_acc_threshold', sorted_acc[0][0])
print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3]))
sorted_acc = sorted(acc.items(), key= lambda x : x[1][1], reverse = True)
df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1])
df.set_value(experiment_name, name + ' f1_score_threshold', sorted_acc[0][0])
print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3]))
return df
# Anomaly detection using logistic regression
def anomaly_detection(anomaly_score_dict, name, df):
X = []
y = []
for l in anomaly_score_dict:
X += anomaly_score_dict[l]
if l in inside_labels:
y += [0]*len(anomaly_score_dict[l])
else:
y += [1]*len(anomaly_score_dict[l])
X = np.array(X)
y = np.array(y)
X, y = utils.shuffle(X, y, random_state=0)
X_train = X[:len(X)/2]
X_test = X[len(X)/2:]
y_train = y[:len(y)/2]
y_test = y[len(y)/2:]
clf = linear_model.LogisticRegression(C=1.0)
clf.fit(X_train, y_train)
auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1])
print("AUC", auc)
df.set_value(experiment_name, name + ' AUC', auc)
if plot: # Plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1], pos_label=1)
sns.plt.figure()
sns.plt.plot(fpr, tpr, label='ROC curve')
sns.plt.plot([0, 1], [0, 1], 'k--')
sns.plt.xlim([0.0, 1.0])
sns.plt.ylim([0.0, 1.05])
sns.plt.xlabel('False Positive Rate')
sns.plt.ylabel('True Positive Rate')
sns.plt.title('Receiver operating characteristic example')
sns.plt.legend(loc="lower right")
sns.plt.show()
return df
df.set_value(experiment_name, 'dataset', dataset)
df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation)
df.set_value(experiment_name, 'inside_labels', str(inside_labels))
df.set_value(experiment_name, 'epochs', epochs)
df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df)
df = anomaly_detection(test_mean_std_deterministic, "Classical prediction", df)
df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df)
df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df)
return df
def test_shared_variables(self):
vocab, data, options = self._get_vocab_data_options(True, True)
options['n_epochs'] = 1
train(options, data, 2, self.tmp_dir, self.tmp_dir)
self.assertEqual(len(tf.global_variables()), 64)
def main():
train_adversarial = 1
use_cuda = True
epochs = 200
lr = 0.005
batch_size = 100
test_batch_size = 100
torch.manual_seed(7347)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print('using device {0}'.format(device))
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
])),
batch_size=batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
transform=transforms.Compose([
transforms.ToTensor(),
])),
batch_size=test_batch_size,
shuffle=True,
**kwargs)
model = Net(activation=nn.LeakyReLU()).to(device)
PATH = 'mnist_cnn100.pt.bak'
# model.load_state_dict(torch.load(PATH, map_location=device), strict=False)
reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr)
discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1)
encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1)
if train_adversarial:
compute_loss = compute_loss_adversarial_enc
optimizer = {
'rec': reconstruction_optimizer,
'dis': discriminative_optimizer,
'enc': encoder_optimizer
}
tmp = [
reconstruction_optimizer, discriminative_optimizer,
encoder_optimizer
]
schedulers = [StepLR(x, step_size=5, gamma=0.9) for x in tmp]
else:
compute_loss = compute_loss_autoenc
optimizer = {'rec': reconstruction_optimizer}
schedulers = [StepLR(reconstruction_optimizer, step_size=5, gamma=0.9)]
for epoch in range(1, epochs + 1):
if epoch % 5 == 0:
test(model, compute_loss, device, test_loader)
train(model, compute_loss, device, train_loader, optimizer, epoch)
for scheduler in schedulers:
scheduler.step()
if epoch % 10 == 0 and epoch:
torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch))
print('learning rate: {0}'.format(scheduler.get_lr()))
