def train(traindatafile, no_line_types = False): """Takes in file with training data, uses that for training purposes.""" print("Training...") # the actual training happens here training.train(dictionary.read_input(traindatafile), monograms, bigrams, digrams, line_types) print("Storing...") # stores data to pickle files if not no_line_types: linetypes_file = open("line_types.p", "wb") pickle.dump(line_types, linetypes_file) linetypes_file.close() (monograms_file, bigrams_file, digrams_file) = (open("monograms.p", "wb"), open("bigrams.p", "wb"), open("digrams.p", "wb")) pickle.dump(monograms, monograms_file) pickle.dump(bigrams, bigrams_file) pickle.dump(digrams, digrams_file) monograms_file.close() bigrams_file.close() digrams_file.close() print("Done!")
def main_test(model_file, fasta_file, args = None): print( " *** Training model ***" ) model = dotio.load( model_file ) print(type(model)) training.train(fasta_file, model, [], 0.05) #model.debug_print() dotio.WriteDOT(model, "{}_train_vs_effectors.faa.dot".format(model_file) ) dotio.store(model, "{}_train_vs_effectors.faa.pickle".format(model_file) )
def main_test(fasta_file, repeats, args = None): perturb_amount = 0.8 rnd = _rng() for i in repeats: model = hmmdsl_py.Model() InitializePrototype(model) #perturb = hmmdsl_py.Perturb(model) #perturb.perturb_emissions(perturb_amount) #for k in range(2,14+2): # model.SetMu (k, rnd()*3.0 + 1.0 ) dotio.WriteDOT(model, "hsmm_local_prototype_04.dot" ) dotio.store(model, "hsmm_local_prototype_04.pickle" ) backgroundmodel = hmmdsl_py.Model() InitializePrototype(backgroundmodel, True) dotio.WriteDOT(backgroundmodel, "hsmm_local_prototype_04_background.dot" ) dotio.store(backgroundmodel, "hsmm_local_prototype_04_background.pickle" ) return #model.split_state( 4, 0.7 ) #dotio.WriteDOT(model, "model_split.dot" ) #dotio.store(model, "model_split.pickle" ) training.train(fasta_file, model, []) #model.debug_print() dotio.WriteDOT(model, "hsmm14_perturb{}_vs_effectros.faa.dot".format(i) ) dotio.store(model, "hsmm14_perturb{}_vs_effectors.faa.pickle".format(i) )
def main(): # spawn theano vars xl = T.matrix('xl') xr = T.matrix('xr') y = T.ivector('y') learning_rate = T.scalar('learning_rate') trng = RandomStreams(4321) # use test values """ import numpy as np batch_size = 10 theano.config.compute_test_value = 'raise' xl.tag.test_value = np.random.randn(batch_size, 392).astype(floatX) xr.tag.test_value = np.random.randn(batch_size, 392).astype(floatX) y.tag.test_value = np.random.randint(8, size=batch_size).astype(np.int32) learning_rate.tag.test_value = 0.5 """ # build cgs model_l, model_r, model_b = build_model( xl, xr, y, learning_rate, trng=trng, **options) # compile opt = get_optimizer(options['optimizer']) f_train_l = opt(learning_rate, model_l, [xl, y]) f_train_r = opt(learning_rate, model_r, [xr, y]) f_train_b = opt(learning_rate, model_b, [xl, xr, y], return_alpha=True) # compile validation/test functions f_valid_l = theano.function([xl, y], [model_l.cost, model_l.acc]) f_valid_r = theano.function([xr, y], [model_r.cost, model_r.acc]) f_valid_b = theano.function([xl, xr, y], [model_b.cost, model_b.acc]) # training loop train(f_train_l, f_train_r, f_train_b, f_valid_l, f_valid_r, f_valid_b, xl, xr, y, **options)
def hyperparameter_search(number_of_combinations): embedding_dimension = [2, 16, 64, 128] number_of_hidden_layers = [ 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ] hidden_layer_dimension = [2, 16, 64, 128] activation_function = [nn.ReLU, nn.Sigmoid, nn.Tanh] number_of_training_epochs = [5, 10, 20] loss_function_choice = [ nn.NLLLoss, nn.L1Loss, nn.CrossEntropyLoss, nn.MSELoss ] optimizer_choice = [optim.SGD, optim.Adam] learning_rate = [0.001, 0.01, 0.1, 1] params_results = [] #TODO - Memorize each choice of params, so no combination is selected more than once for i in range(number_of_combinations): params = { "embedding_dimension": choice(embedding_dimension), "number_of_hidden_layers": choice(number_of_hidden_layers), "hidden_layer_dimension": choice(hidden_layer_dimension), "activation_function": choice(activation_function), "number_of_training_epochs": choice(number_of_training_epochs), "loss_function_choice": choice(loss_function_choice), "optimizer_choice": choice(optimizer_choice), "learning_rate": choice(learning_rate) } print("Searching with following params:s") print(params) validation_accuracy = train(**params) print("") params_results.append((params, validation_accuracy)) params_results.sort(key=lambda x: x[1]) print("Best found params:") print(params_results[0][0]) print("") print("Accuracy on best found params") print(params_results[0][1]) print("") return params_results
def objective(options=None): # Initialize the best validation loss, which is the value to be minimized by the network best_val_loss = float("Inf") # Define hyperparameters lr = 0.001 dropout = 0.3 batch_size = 128 print(f"Learning rate: {lr}") print(f"Dropout: {dropout}") print(f"Batch size: {batch_size}") # Use CUDA if GPU is available, else CPU use_cuda = options["use_cuda"] and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") print(f"Using device {device}") # Obtain the MNIST train and validation loaders using a helper function train_loader, val_loader = get_mnist_dataloaders(options["data_path"], batch_size) # Initialize network model = Net(dropout=dropout).to(device) # Learning rate optimizer optimizer = optim.Adam(model.parameters(), lr=lr) scheduler = StepLR(optimizer, step_size=1, gamma=0.7) # Network training & validation loop for epoch in range(0, options["epochs"]): avg_train_loss = train( options, model, device, train_loader, optimizer, epoch ) avg_val_loss = validate(model, device, val_loader) if avg_val_loss <= best_val_loss: best_val_loss = avg_val_loss # Print intermediate validation & training loss print(f"Epoch {epoch + 1} of {options['epochs']} --- average train loss: {avg_train_loss} --- average validation loss: {avg_val_loss}") scheduler.step() # Return the best validation loss return best_val_loss
def solver_to_results(solver_constructor: Callable[[], Solver], bandit: NArmedBandit, num_trainings) -> List[TrainResult]: """ Takes a solver constructor and trains it NUM_TRAININGS times on the given bandit. :param solver_constructor: A callable that creates a solver :param bandit: The bandit to train on :param num_trainings: Specifies how often the solver is trained on the bandit :return: The results of the training """ results = [] solver_name = str(solver_constructor()) for _ in tqdm(range(num_trainings), desc=f'training solver {solver_name}: '): solver = solver_constructor() result = train(bandit, solver, NUM_PLAYS_PER_TRAINING) results.append(result) return results
def node(test_node, test_leaf): node_dataset, leaf_dataset, test_dataset = prepare_dataset.prepare_dataset( [[1], [7]]) PATHS = ['./node_net.pth', './left_net.pth', './right_net.pth'] if test_node: for h in range(1): training.train(leaf_dataset, PATHS[1]) training.train(leaf_dataset, PATHS[2]) for i in range(1, 3, 1): print(f'Number of epochs: {i}') node = Node.Node(node_dataset, PATHS[1], PATHS[2], PATHS[0]) node.train(i) testing_node.test(test_dataset, PATHS) #print('NEW TRY') if test_leaf: training.train(leaf_dataset, PATHS[1]) training.train(leaf_dataset, PATHS[2]) testing_leaf.test(test_dataset, PATHS[1]) testing_leaf.test(test_dataset, PATHS[2])
def main(): global training_path, query_path, result_path, training_data_path, csv_path cur_path = os.path.dirname(os.path.abspath(__file__)) data_path = os.path.join(cur_path, "data") training_path = os.path.join(data_path, training_file) query_path = os.path.join(data_path, query_file) result_path = os.path.join(data_path, result_file) training_data_path = os.path.join(data_path, training_data_file) csv_path = os.path.join(data_path, csv_file) start_time = timeit.default_timer() args, predict_fun, feature_fun = handle_args() load_flag = args.load feature_num = args.num_feature compare_flag = args.compare if compare_flag: compare() else: dic = None if load_flag: with open(training_data_path, "rb") as f: dic = pickle.load(f) else: dic = training.train(training_path) with open(training_data_path, "wb") as f: pickle.dump(dic, f) if feature_fun != None: feature_selection.select_feature(dic, feature_num, feature_fun) else: print("-"*60) print("No feature selection!") classifier.nbc(query_path, result_path, dic, predict_fun) end_time = timeit.default_timer() print("Total time: {:.3} sec".format(end_time - start_time)) result = accuracy(query_path, result_path)
def modelSvm(trainingImagePath, k): t1 = cv2.getTickCount() trainingImage = cv2.imread(trainingImagePath) trainingImage = cv2.cvtColor(trainingImage, cv2.COLOR_BGR2LAB) m, n, _ = trainingImage.shape print "Color Quantization : " #Preprocessing variable from image l = trainingImage[:, :, 0] a = trainingImage[:, :, 1] b = trainingImage[:, :, 2] scaler = preprocessing.MinMaxScaler() pca = PCA(32) qab, centroid = quantization(a, b, k) print centroid # with open('./../temp/'+testCaseNumber+'/centroids', 'w') as csvfile: # writer = csv.writer(csvfile) # [writer.writerow(r) for r in centroid] t2 = cv2.getTickCount() t = (t2 - t1) / cv2.getTickFrequency() print "Time for quantization : ", t, " seconds" print "Feature extraction : " feat, classes = getKeyPointFeatures(l, qab) print "Length of feature descriptor before PCA : ", len(feat[0]) feat = scaler.fit_transform(feat) feat = pca.fit_transform(feat) print "Length of feature descriptor after PCA : ", len(feat[0]) t3 = cv2.getTickCount() t = (t3 - t2) / cv2.getTickFrequency() print "Time for feature extraction : ", t, " seconds" print "Training : " svm_classifier = train(feat, classes, k) t4 = cv2.getTickCount() t = (t4 - t3) / cv2.getTickFrequency() print "Time for training: ", t, " seconds" return (centroid, svm_classifier, scaler, pca)
def go(batch_size, epochs, dataset, input_model): num_classes = 10 x_train, y_train, x_test, y_test, input_shape = get_data(dataset, num_classes) print('x_train shape:', x_train.shape) print(x_train.shape[0], 'train samples') print(x_test.shape[0], 'test samples') model = train(get_model(input_model), x_train, y_train, batch_size, epochs, x_test, y_test) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) keras_file = "mnist_pruned.h5" print('Saving model to: ', keras_file) tf.keras.models.save_model(model, keras_file, include_optimizer=False)
def main(fragment_file, lead_file): fragment_mols = read_file(fragment_file) lead_mols = read_file(lead_file) fragment_mols += lead_mols fragments, used_mols = get_fragments(fragment_mols) encodings, decodings = get_encodings(fragments) save_decodings(decodings) lead_mols = np.asarray(fragment_mols[-len(lead_mols):])[used_mols[-len(lead_mols):]] X = encode_list(lead_mols, encodings) actor, critic = build_models(X.shape[1:]) X = clean_good(X, decodings) history = train(X, actor, critic, decodings) np.save("History/history.npy", history)
def mlp_predictions(randomsearch=False): if randomsearch: layersizes = random_search.architecture_search() # 7,32,32,16,1 hparams = random_search.hparams_search(layersizes) # 0.005, 16 else: layersizes = [7, 32, 32, 16, 1] hparams = [0.005, 16] hparams_setting = { "epochs": 1000, "batchsize": hparams[1], "learningrate": hparams[0], "history": 1 } model_design = {"layer_sizes": layersizes} X, Y, Y_Preles = preprocessing.preprocessing() arr = [] years = [ 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 ] for year in years: x, y, y_nn = preprocessing.split_by_year(X, Y, Y_Preles, years=[year]) #x = x.drop(["year"], axis=1) running_losses = training.train(hparams_setting, model_design, x.to_numpy(), y.to_numpy(), "D1") preds, mae, nse = prediction.predict(hparams_setting, model_design, x.to_numpy(), y.to_numpy(), "D1") arr.append(preds) arr = np.concatenate(arr, axis=1) return arr
def objective(trial): relevant_metric = 'val_loss' args = get_parser().parse_args() args.experiment_name = trial.study.study_name # hyperparameter can be modified here! args.learning_rate = trial.suggest_categorical('learning_rate', [1e-2, 5e-3, 1e-3]) args.lr_warmup_steps = trial.suggest_categorical('lr_warmup_steps', [1000, 2000, 4000]) args.accumulate_grad_batches = trial.suggest_categorical( 'accumulate_grad_batches', [32, 64, 128]) args.positional_encoding_dropout = trial.suggest_uniform( 'positional_encoding_dropout', 0.1, 0.3) args.transformer_dropout = trial.suggest_uniform('transformer_dropout', 0.1, 0.3) args.version = (f'lr{args.learning_rate}' f'_ws{args.lr_warmup_steps}' f'_agb{args.accumulate_grad_batches}' f'_pd{args.positional_encoding_dropout}' f'_td{args.transformer_dropout}') trainer = train( args, custom_callbacks=[ PyTorchLightningPruningCallback(trial, monitor=relevant_metric), EarlyStopping( monitor=relevant_metric, min_delta=0.5, patience=3, verbose=False, mode='min', ), ], ) return find_best_checkpoint(trainer.logger.log_dir)
def main(): try: ### 학습모델 불러오기 model = tf.keras.models.load_model('my_model.h5') except (FileNotFoundError, OSError) as e: model = training.train() result = [] ocrs = imgs.ReadImg(sys.argv[1]) trainImg = [] for e in ocrs: trainImg.append(e.data) result.append(model.predict(e.data).argmax()) print('predict: ', result) print('Is predict right? (1.right, 2.wrong)') right = input() if int(right) != 1: print('Input Answer: ') ans = input() darr = digit(ans) for idx, e in enumerate(trainImg): model.fit(e, [darr[idx]]) model.save('my_model.h5')
from training import pretrain, train_discriminator, train from data import sample_groups import torch n_target_samples = 7 plot_accuracy = True if __name__ == '__main__': cuda = torch.cuda.is_available() groups, data = sample_groups(n_target_samples=n_target_samples) encoder, classifier = pretrain(data, cuda=cuda, epochs=20) discriminator = train_discriminator(encoder, groups, n_target_samples=n_target_samples, epochs=50, cuda=cuda) train(encoder, discriminator, classifier, data, groups, n_target_samples=n_target_samples, cuda=cuda, epochs=150, plot_accuracy=plot_accuracy)
pickle_in = open("../../Q1_data/data.pkl", "rb") rawDict = pickle.load(pickle_in) xDict , yDict = np.hsplit(rawDict, 2) xTrain, xTest, yTrain, yTest = train_test_split(xDict, yDict, test_size=0.1, random_state=57) temp = np.concatenate((xTrain, yTrain), axis =1) np.random.shuffle(temp) xTrain, yTrain = np.hsplit(temp, 2) dx , dy = distribute(xTrain, yTrain) bias, variance = train(dx, dy, xTest, yTest) table = np.vstack(([i for i in range(1, 100)], bias, variance)).T print(tabulate(table, headers=["Degree", "Bias", "Variance"], tablefmt='psql')) plt.plot(bias, color = 'red',label='bias') plt.plot(variance, color = 'blue', label='variance') plt.legend() plt.show() # now that we have the bias and the variance ,we can plot this data or whatever
if not os.path.exists(os.path.join(args.model_dir, args.model)): os.makedirs(os.path.join(args.model_dir, args.model)) transformer = transform_train if (args.mode == "train") else transform_val data_loader = dataloader( transform=transformer, mode=args.mode, batch_size=args.batch_size, vocab_threshold=args.vocab_threshold, from_vocab_file=args.from_vocab_file, vocab_file=args.vocab_file, data_path=args.data_dir, image_data_unavailable=args.data_unavailable, model=args.model, ) vocab_size = len(data_loader.dataset.vocab) print("vocabulary size: {}".format(vocab_size)) if args.model == "lstm": encoder, decoder = lstm(vocab_size) elif args.model == "attention": encoder, decoder = attention_lstm(vocab_size) if args.mode == "train": train(encoder, decoder, data_loader, vocab_size, args) elif args.mode == "val": validation(encoder, decoder, data_loader, vocab_size, args)
if opt.conv_encoder: model = meta_modules.ConvolutionalNeuralProcessImplicit2DHypernet( in_features=img_dataset.img_channels, out_features=img_dataset.img_channels, image_resolution=image_resolution) else: model = meta_modules.NeuralProcessImplicit2DHypernet( in_features=img_dataset.img_channels + 2, out_features=img_dataset.img_channels, image_resolution=image_resolution) model.cuda() # Define the loss loss_fn = partial(loss_functions.image_hypernetwork_loss, None, opt.kl_weight, opt.fw_weight) summary_fn = partial(utils.write_image_summary_small, image_resolution, None) root_path = os.path.join(opt.logging_root, opt.experiment_name) training.train(model=model, train_dataloader=dataloader, epochs=opt.num_epochs, lr=opt.lr, steps_til_summary=opt.steps_til_summary, epochs_til_checkpoint=opt.epochs_til_ckpt, model_dir=root_path, loss_fn=loss_fn, summary_fn=summary_fn, clip_grad=True)
def main(modeltype, batch_size, iterations, learning_rate, segment_len, segment_step, train_save_params, dataset=["dataset"], outputdir="output", validation=None, validation_generate_ct=1, resume=None, resume_auto=False, check_nan=False, generate=False, generate_over=None, auto_connectome_keys=None, **model_kwargs): generate = generate or (generate_over is not None) should_setup = not generate unroll_batch_num = None if generate else training.BATCH_SIZE for dataset_dir in dataset: if os.path.samefile(dataset_dir,outputdir): print("WARNING: Directory {} passed as both dataset and output directory!".format(outputdir)) print("This may cause problems by adding generated samples to the dataset directory.") while True: result = input("Continue anyway? (y/n)") if result == "y": break elif result == "n": sys.exit(0) else: print("Please type y or n") if generate_over is None: training.set_params(batch_size, segment_step, segment_len) leadsheets = [training.filter_leadsheets(training.find_leadsheets(d)) for d in dataset] else: # Don't bother loading leadsheets, we don't need them leadsheets = [] if validation is not None: validation_leadsheets = training.filter_leadsheets(training.find_leadsheets(validation)) else: validation_leadsheets = None m = builders[modeltype].build(should_setup, check_nan, unroll_batch_num, **model_kwargs) m.set_learning_rate(learning_rate) if resume_auto: paramfile = os.path.join(outputdir,'final_params.p') if os.path.isfile(paramfile): with open(os.path.join(outputdir,'data.csv'), 'r') as f: for line in f: pass lastline = line start_idx = lastline.split(',')[0] print("Automatically resuming from {} after iteration {}.".format(paramfile, start_idx)) resume = (start_idx, paramfile) else: print("Didn't find anything to resume. Starting from the beginning...") if resume is not None: start_idx, paramfile = resume start_idx = int(start_idx) m.params = pickle.load( open(paramfile, "rb" ) ) else: start_idx = 0 if not os.path.exists(outputdir): os.makedirs(outputdir) if generate: print("Setting up generation") m.setup_produce() print("Starting to generate") start_time = time.process_time() if generate_over is not None: source, divwidth = generate_over if divwidth == 'full': divwidth = 0 elif divwidth == 'debug_firststep': divwidth = -1 elif len(divwidth)>3 and divwidth[-3:] == 'bar': divwidth = int(divwidth[:-3])*(constants.WHOLE//constants.RESOLUTION_SCALAR) else: divwidth = int(divwidth) ch,mel = leadsheet.parse_leadsheet(source) lslen = leadsheet.get_leadsheet_length(ch,mel) if divwidth == 0: batch = ([ch],[mel]), [source] elif divwidth == -1: slices = [leadsheet.slice_leadsheet(ch,mel,0,1)] batch = list(zip(*slices)), [source] else: slices = [leadsheet.slice_leadsheet(ch,mel,s,s+divwidth) for s in range(0,lslen,divwidth)] batch = list(zip(*slices)), [source] training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True, batch=batch) else: training.generate(m, leadsheets, os.path.join(outputdir, "generated"), with_vis=True) end_time = time.process_time() print("Generation took {} seconds.".format(end_time-start_time)) else: training.train(m, leadsheets, iterations, outputdir, start_idx, train_save_params, validation_leadsheets=validation_leadsheets, validation_generate_ct=validation_generate_ct, auto_connectome_keys=auto_connectome_keys) pickle.dump( m.params, open( os.path.join(outputdir, "final_params.p"), "wb" ) )
#module 2 from prep_cifar10 import data_to_categorical y_train, y_test = data_to_categorical(y_train, y_test, num_classes) #module 3 from shape_model import add_model model = add_model(num_classes, x_train) #module 3 from shape_model import opt opt = opt() #module 4 #Processing / Training from training import train x_train, x_test, model, opt = train(model, opt, x_train, x_test) #module 4 from training import augment x_train, y_train, x_test, y_test = augment(data_augmentation, x_train, y_train, batch_size, epochs, x_test, y_test, model) #module 5 #OUTPUT: # Save trained model from output import save_cnn model, model_path = save_cnn(model, save_dir, model_name) from output import cnn_stats model = cnn_stats(model, model_path, x_test, y_test)
def main(): # Initialization - argument parsing, logging and stage preparations logging.basicConfig(format='%(asctime)s %(message)s', filename='logs.log', level=logging.INFO) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) args = parse_args() out_folder = args.out_folder if args.out_folder.endswith( '/') else args.out_folder + '/' if not os.path.exists(out_folder): os.makedirs(out_folder) # Training logging.info("Preparing images for training...") _, gray_img, _, samples, labels = prepare_for_training( args.training_image, kernels) logging.info("Starting training process...") weights = train(samples, labels, num_of_epochs=args.num_of_epochs, batch_size=args.batch_size) logging.info("Training done.") # Training results results = np.round(weights[-1, :], 2) logging.info("Results: ") for result, kernel, name in zip(results, kernels, kernel_names): logging.info(f"Original {name} filter: \n{kernel}") logging.info(f"Trained {name} filter: \n{result}") out_image = out_folder + f"{name}/training_image_compare.jpg" logging.info(f"Saving training comparition to {out_image}\n") compare_images(convolution(gray_img, result), convolution(gray_img, kernel), out_image=out_image) # Results on validation image _, _, validation_image = load_normalize_and_grayscale( args.validation_image) logging.info( f"Acceptance error is set to {args.acceptance_error}% of distance between highest and lowest possible value" ) for result, kernel, name in zip(results, kernels, kernel_names): out_image = out_folder + f"{name}/validation_image_compare.jpg" logging.info(f"Saving validation comparition to {out_image}") perc_of_similarity = compare_images( convolution(validation_image, result), convolution(validation_image, kernel), out_image=out_image, acceptance_error=args.acceptance_error) logging.info( f"Similarity between validation image filtered with trained and original {name} kernel is {perc_of_similarity}%" ) # Animated matrix if args.create_gifs: for i, name in zip(range(weights.shape[1]), kernel_names): out_image = out_folder + f"{name}/animated_training_matrix.gif" logging.info(f"Creating matrix gif for {name}...") create_matrix_gif(weights[:, i], save_folder=out_folder + "gif_images/", out_image=out_folder + f"{name}/animated_training_matrix.gif") else: logging.info("Skipping gif creation.")
parms['flow_layers'] = c[3] parms['flow_units'] = c[4] parms['flow_type'] = c[5] parms['autoregressive'] = c[6] parms['n_pixelcnn_layers'] = c[7] parms['anneal'] = c[8] parms['n_z'] = c[9] parms['n_mixtures'] = c[10] parms['learning_rate'] = c[11] for name, model in models.items(): name = experiment_name + '_nz_' + str(parms['n_z']) + '_nmix_' + str(parms['n_mixtures']) \ + '_lr_' + str(parms['learning_rate']) if parms['flow']: name += "_flow_" + str(parms['flow_layers']) + "_" + str(parms['flow_units']) + "_" + parms['flow_type'] if parms['autoregressive']: name += "_ar_" + str(parms['n_pixelcnn_layers']) if parms['anneal'] < 0: name += "_anneal_" + str(parms['anneal']) train(name=name, model=model, parameters=parms, data=data, tracker=tracker) print("Finished :)", flush=True)
BREAK_EARLY = False BATCH_SIZE = 500 for data_subdir in ['ml-20m', 'netflix-prize', 'msd']: actor_path = "WMFVAE_ACTOR_TRAIN_{}".format(data_subdir) train( model_class='wmf_vae', data_subdir=data_subdir, n_epochs_pred_only=100, n_epochs_ac_only=0, n_epochs_pred_and_ac=0, max_kl=0.05, ac_reg_loss_scaler=0.0, actor_reg_loss_scaler=0.0001, evaluation_metric="NDCG", logging_frequency=50, batch_size=BATCH_SIZE, break_early=BREAK_EARLY, verbose=False, positive_weights=5.0, version_tag="FULL_RUN_ON_OTHER_DATASETS", path_to_save_actor=actor_path, log_critic_training_error=False, ) print("Now, hopefully on to testing...") test( model_class='wmf_vae', data_subdir=data_subdir,
return train_iterator, valid_iterator, test_iterator, text_field.vocab # load data print("\nLoading data...") device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') text_field = data.Field(tokenize='spacy', lower=True, include_lengths=True, batch_first=True) label_field = data.Field(sequential=False, use_vocab=False, batch_first=True, dtype=torch.float) docid_field = data.RawField() train_iter, dev_iter, test_iter, text_voca = fall_data(docid_field, text_field, label_field, device=-1, repeat=False) # train or eval if args.test: best_model = model.LSTM().to(device) optimizer = optim.Adam(best_model.parameters(), lr=args.lr) t.load_checkpoint(destination_folder + '/model.pt', best_model, optimizer) t.evaluate(best_model, test_iter) else: print('start training') wandb.init() wandb.watch(model) model = model.LSTM(text_voca).to(device) optimizer = optim.Adam(model.parameters(), lr=args.lr) eval_every = len(train_iter) // 2 t.train(model=model, optimizer=optimizer, train_loader=train_iter, valid_loader=dev_iter, num_epochs=args.epochs, eval_every = eval_every, file_path= destination_folder, device=device)
# -*- coding: utf-8 -*- """ Spyder Editor This is a temporary script file. """ from model import create_model from DataGeneration import dataGeneration from Checkpoint import createCheckpoint from Performance import plot, calculate from training import train from Predict import predict loadPreviouslySavedWeights = 1 network = create_model() dataSet = dataGeneration() cp_callback = createCheckpoint() train(network, dataSet, cp_callback, loadPreviouslySavedWeights) calculate(network, dataSet) predict(network, dataSet)
import normalization import training import testing import time start = time.time() common_words = normalization.normalize_text("test/sport.txt") print("text normalize") ''' for training training.train(common_words, "mathematics") print(time.time() - start) ''' ''' for testing ''' testing.classify(common_words) print(time.time() - start)
def random_forest(path_train, path_test, name_identifiers, name_targets, features, delimiter, num_cores=1): ''' this method performs the training,testing and evaluation of the random forest algorithm. @type path_train: str @param path_train: full path to csv (first line contains headers). delimiter should be specified with param delimiter @type path_test: str @param path_test: full path to csv (first line contains headers). delimiter should be specified with param delimiter @type name_identifiers: str @param name_identifiers: name of column containing identifiers @type name_targets: str @param name_targets: name of column containing targets @type features: str @param features: list of features to be used @type delimiter: str @param delimiter: delimiter used in csv. tested with "," and "\t" @type num_cores: int @param num_cores: [optional]: num of cores you want to use in training @rtype: tuple @return: (output_classifier,evaluation). both are dicts. output_classifier maps identifier -> output_classifier. evaluation maps all kinds of evaluation metrics to floats. ''' #call train using class identifiers_training,target_training,rf = train(path_train, name_identifiers, name_targets, features, output_path_model=None, cores=num_cores, the_delimiter=delimiter) #call test using class identifiers_test,target,prediction = test(path_test, name_identifiers, name_targets, features, loaded_rf_model=rf, path_rf_model=None, the_delimiter=delimiter) #evaluate classifier_output,evaluation = evaluate(target, prediction, identifiers_test) return classifier_output,evaluation
'epochs': 50, 'batch_size': 128, 'lr': 1e-3, 'decay': 1e-2, 'gamma': 0.8, 'step_size': 5, 'num_bins': 50, 'prune_milestones': [1000], 'prune_sigmas': [1], 'prune_gamma': 10, 'ignore_below': True, 'p_i': 5e-1, 'p_f': 0.99, 'T': 100 } model = LeNet_KG(in_chan=1, out_chan=10, imsize=28, kernel_size=5) # model = CNN() model, df = train(model, trainset, valset, param_dict) model.eval() with torch.no_grad(): alltrainloader = torch.utils.data.DataLoader(trainset, batch_size=len(trainset), shuffle=False) for b, (x, y) in enumerate(alltrainloader): v, s = model(x) c = v.argmax(dim=1) std = torch.empty((s.size()[0]), requires_grad=False) for i in range(s.size()[0]): std[i] = torch.sqrt(F.softplus(s[i, c[i]]))
def run(): USE_CUDA = torch.cuda.is_available() FLAGS.config = 'example_seq2seq.json' config_path = os.path.join("experiments", FLAGS.config) print(FLAGS.config) if not os.path.exists(config_path): raise FileNotFoundError with open(config_path, "r") as f: config = json.load(f) config["gpu"] = torch.cuda.is_available() writer = SummaryWriter('experiments/finally') # dataset = ToyDataset(5, 15) # eval_dataset = ToyDataset(5, 15, type='eval') dataset = Toy_Numbers(10) eval_dataset = Toy_Numbers(10, train=False) BATCHSIZE = 32 train_loader = data.DataLoader(dataset, batch_size=BATCHSIZE, shuffle=False, collate_fn=pad_collate, drop_last=True) eval_loader = data.DataLoader(eval_dataset, batch_size=BATCHSIZE, shuffle=False, collate_fn=pad_collate, drop_last=True) config["batch_size"] = BATCHSIZE # Models model = Seq2Seq(config) model = model.float() # dataiter = iter(train_loader) # sample_input= dataiter.next() # writer.add_graph(model, sample_input) # writer.close() if USE_CUDA: model = model.cuda() # Optimizer optimizer = torch.optim.Adam(model.parameters(), lr=config.get("learning_rate", .001)) print("=" * 60) print(model) print("=" * 60) for k, v in sorted(config.items(), key=lambda i: i[0]): print(" (" + k + ") : " + str(v)) print() print("=" * 60) print("\nInitializing weights...") for name, param in model.named_parameters(): if 'bias' in name: torch.nn.init.constant_(param, 0.0) elif 'weight' in name: torch.nn.init.xavier_normal_(param) for epoch in range(FLAGS.epochs): run_state = (epoch, FLAGS.epochs, FLAGS.train_size) # Train needs to return model and optimizer, otherwise the model keeps restarting from zero at every epoch model, optimizer = train(model, optimizer, train_loader, run_state, writer) # print("losses", l_list) # for i in l_list: # # print(i) # writer.add_scalar('Loss/train',i) evaluate(model, eval_loader, writer)
def run(env_id, render, render_eval, seed, noise_type, layer_norm, evaluation, **kwargs): # Configure things. rank = MPI.COMM_WORLD.Get_rank() if rank != 0: logger.set_level(logger.DISABLED) # Create envs. env = RobotPath.env(render) env = bench.Monitor(env, logger.get_dir() and os.path.join(logger.get_dir(), str(rank)), allow_early_resets=True) if evaluation and rank == 0: eval_env = RobotPath.env(render_eval) eval_env = bench.Monitor(eval_env, os.path.join(logger.get_dir(), 'gym_eval'), allow_early_resets=True) env = bench.Monitor(env, None, allow_early_resets=True) else: eval_env = None # Parse noise_type action_noise = None param_noise = None nb_actions = env.action_space.shape[-1] for current_noise_type in noise_type.split(','): current_noise_type = current_noise_type.strip() if current_noise_type == 'none': pass elif 'adaptive-param' in current_noise_type: _, stddev = current_noise_type.split('_') param_noise = AdaptiveParamNoiseSpec( initial_stddev=float(stddev), desired_action_stddev=float(stddev)) elif 'normal' in current_noise_type: _, stddev = current_noise_type.split('_') action_noise = NormalActionNoise(mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions)) elif 'ou' in current_noise_type: _, stddev = current_noise_type.split('_') action_noise = OrnsteinUhlenbeckActionNoise( mu=np.zeros(nb_actions), sigma=float(stddev) * np.ones(nb_actions)) else: raise RuntimeError( 'unknown noise type "{}"'.format(current_noise_type)) # Configure components. #print("----------------") #print(env.observation_space) env = gym.wrappers.FlattenDictWrapper( env, ['desired_goal', 'achieved_goal', 'observation']) #print(env.observation_space) # print(env.observation_space['start_point'].shape) #print("----------------") memory = Memory(limit=int(1e6), action_shape=env.action_space.shape, observation_shape=env.observation_space.shape) critic = Critic(layer_norm=layer_norm) actor = Actor(nb_actions, layer_norm=layer_norm) # Seed everything to make things reproducible. seed = seed + 1000000 * rank logger.info('rank {}: seed={}, logdir={}'.format(rank, seed, logger.get_dir())) tf.reset_default_graph() set_global_seeds(seed) env.seed(seed) if eval_env is not None: eval_env.seed(seed) # Disable logging for rank != 0 to avoid noise. if rank == 0: start_time = time.time() training.train(env=env, eval_env=eval_env, param_noise=param_noise, action_noise=action_noise, actor=actor, critic=critic, memory=memory, **kwargs) env.close() if eval_env is not None: eval_env.close() if rank == 0: logger.info('total runtime: {}s'.format(time.time() - start_time))
'prepare' x_train, y_train = train_dfs[cv_idx][features], train_dfs[cv_idx][target] x_valid, y_valid = valid_dfs[cv_idx][features], valid_dfs[cv_idx][target] x_test, y_test = test_dfs[cv_idx][features], test_dfs[cv_idx][target] # add feature from baseline x_train, x_valid, x_test, x_sub = add_group_feature(x_train, x_valid, x_test, test_df[features], y_train) lgb_train, lgb_valid = lgb.Dataset(x_train, y_train, categorical_feature=[3, 4, 5], free_raw_data=False), lgb.Dataset(x_valid, y_valid, categorical_feature=[3, 4, 5], free_raw_data=False) 'train' if tune: params = training.tuning(lgb_train, lgb_valid, 100) pd.to_pickle(params, 'params/{0}_cv{1}.pkl'.format(name, cv_idx)) model = training.train(lgb_train, lgb_valid, params) score = training.evaluation(model, x_test, y_test) scores.append(score) model.save_model('model/{0}_cv{1}.txt'.format(name, cv_idx), num_iteration=model.best_iteration) 'predict' pred = model.predict(x_sub) pred = np.round(pred) sub_df.iloc[:, 1] += pred sub_df.iloc[:, 1] /= cv sub_df.iloc[:, 1] = np.round(sub_df.iloc[:, 1]).astype(np.int8) print(scores) print(np.mean(scores)) #
if __name__ == '__main__': model = 'VGG' dataset = 'CIFAR-100' pretrained = True vggnet = torchvision.models.vgg11_bn(pretrained=pretrained, progress=True) vggnet.classifier = torch.nn.Sequential( torch.nn.Linear(in_features=25088, out_features=1024, bias=True), torch.nn.ReLU(inplace=True), torch.nn.Dropout(p=0.5, inplace=False), torch.nn.Linear(1024, 100)) train_dl, val_dl, test_dl = load_dataloaders_from_dataset(dataset) optimizer = torch.optim.SGD(vggnet.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-3) criterion = torch.nn.CrossEntropyLoss() writer = SummaryWriter('runs/model={}_pretrained={}_dset={}'.format( model, pretrained, dataset)) train(vggnet, train_dl, val_dl, test_loader=test_dl, optimizer=optimizer, criterion=criterion, device=device, epochs=5000, early_stopping=True, writer=writer)
for i in range(0, 10): #Train with the BW always to 0 model = Net() model.type(torch.cuda.FloatTensor) train_loader, valid_loader, test_loader = pixar_dataset( 0.8, 'addit', test=True ) #both work also, because of test=true, the mode option is useless here use_gpu = True n_epoch = 100 learning_rate = 0.002 history = train(model, train_loader, valid_loader, n_epoch, learning_rate, use_gpu=use_gpu) history.display() test_acc, test_loss = test(model, test_loader, use_gpu=use_gpu) print('Test:\n\tLoss: {}\n\tAccuracy: {}'.format(test_loss, test_acc)) #Train with Input Dropout in addit mode model = Net() model.type(torch.cuda.FloatTensor) train_loader, valid_loader, test_loader = pixar_dataset(0.8, 'addit', test=False) use_gpu = True
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--output_dir", default=None, type=str, required=True, help= "The output directory where the model predictions and checkpoints will be written." ) parser.add_argument( "--model_name_or_path", default="gpt2-medium", type=str, help="The model checkpoint for weights initialization.") parser.add_argument( "--load_from_multiple_models", action='store_true', help="Whether to load from multiple models with postfix -0 to -4.") parser.add_argument( "--block_size", default=150, type=int, help="Optional input sequence length after tokenization." "The training dataset will be truncated in block of this size for training." "Default to the model max input length for single sentence inputs (take into account special tokens)." ) parser.add_argument("--start_checkpoint", default=-1, type=int, help="Start from checkpoint X.") parser.add_argument("--num_sampled_sessions", default=40, type=int, help="Sample X sessions in a total of 40.") parser.add_argument("--per_gpu_train_batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.") parser.add_argument( '--gradient_accumulation_steps', type=int, default=1, help= "Number of updates steps to accumulate before performing a backward/update pass." ) parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight deay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument("--num_train_epochs", default=1.0, type=float, help="Total number of training epochs to perform.") parser.add_argument( "--max_steps", default=-1, type=int, help= "If > 0: set total number of training steps to perform. Override num_train_epochs." ) parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.") parser.add_argument('--save_steps', type=int, default=50, help="Save checkpoint every X updates steps.") parser.add_argument("--no_cuda", action='store_true', help="Avoid using CUDA when available") parser.add_argument('--overwrite_output_dir', action='store_true', help="Overwrite the content of the output directory") parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") args = parser.parse_args() args.simplifier = True # Train the simplifier if os.path.exists(args.output_dir) and os.listdir( args.output_dir) and not args.overwrite_output_dir: raise ValueError( "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome." .format(args.output_dir)) device = torch.device( "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() args.device = device # Setup logging logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO) logger.warning("device: %s, n_gpu: %s", device, args.n_gpu) # Training for i in range(0, len(utili.segments_name)): # Set seed utili.set_seed(args) config_class, model_class, tokenizer_class = GPT2Config, GPT2LMHeadModel, GPT2Tokenizer postfix = ('-' + str(i)) if args.load_from_multiple_models else '' config = config_class.from_pretrained(args.model_name_or_path + postfix) # gpt2 size tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path + postfix) tokenizer.add_special_tokens(utili.special_tokens_dict) if args.block_size <= 0: args.block_size = tokenizer.max_len_single_sentence # Our input block size will be the max possible for the model args.block_size = min(args.block_size, tokenizer.max_len_single_sentence) postfix += ('/checkpoint-' + str(args.start_checkpoint)) if ( args.start_checkpoint != -1) else '' model = model_class.from_pretrained(args.model_name_or_path + postfix) model.resize_token_embeddings(len(tokenizer)) # resize assert tokenizer.sep_token == '<SEP>' assert tokenizer.pad_token == '<PAD>' assert tokenizer.bos_token == '<BOS>' assert tokenizer.eos_token == "<EOS>" logger.info( "Added sep_token (id: %s), pad_token (id: %s), bos_token (id: %s) and eos_token (id: %s)", tokenizer.sep_token_id, tokenizer.pad_token_id, tokenizer.bos_token_id, tokenizer.eos_token_id) model.to(args.device) logger.info("Training/evaluation parameters %s", args) train_segments = [utili.segments_name[x] for x in range(i)] + [ utili.segments_name[x] for x in range(i + 1, len(utili.segments_name)) ] print("train_segments: {}".format(train_segments)) train_dataset = QueryRewriteDataset(train_segments, tokenizer, args, args.num_sampled_sesions) global_step, tr_loss = train(args, train_dataset, model, tokenizer, logger, cross_validate_id=i) logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) # Create output directory if needed segment_output_dir = args.output_dir + '-' + str(i) if not os.path.exists(segment_output_dir): os.makedirs(segment_output_dir) logger.info("Saving model checkpoint to %s", segment_output_dir) model_to_save = model.module if hasattr(model, 'module') else model model_to_save.save_pretrained(segment_output_dir) tokenizer.save_pretrained(segment_output_dir) torch.save(args, os.path.join(segment_output_dir, 'training_args.bin')) del model torch.cuda.empty_cache()
def anomaly(experiment_name, dataset = "mnist", bayesian_approximation = "dropout", inside_labels = [0, 1], num_epochs = 50, batch_size = 128, acc_threshold = 0.6, weight_decay = 1e-5, dropout_p = 0.5, fc_layers = [512, 512], plot = True): """ This methods trains a neural network classifier on a subset of classes. After the training, it uses uncertainty measures (e.g. entropy) to detect anomalies. The anomalous classes are the ones that are not part of the training subset. dataset = "mnist" or "cifar" For MNIST we use a fully-connected MLP. For CIFAR10 we use a convolutional net (similar to LeNet) bayesian_approximation = "dropout" for Yarin Gal's method - work either with MNIST bayesian_approximation = "variational" for fully-factorized Gaussian variational approximation - only work with MNIST. inside_labels are the subset of trained classes, the other classes are only used for testing. """ n_out = len(inside_labels) # Prepare Theano variables for inputs and targets # Load the dataset print("Loading data...") if dataset == "mnist": input_var = T.matrix('inputs') target_var = T.ivector('targets') n_in = [28*28] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_MNIST(inside_labels) if bayesian_approximation == "dropout": model = models.mlp_dropout(input_var, target_var, n_in, n_out, fc_layers, dropout_p, weight_decay) elif bayesian_approximation == "variational": model = models.mlp_variational(input_var, target_var, n_in, n_out, fc_layers, batch_size, len(X_train)/float(batch_size)) elif dataset == "cifar": input_var = T.tensor4('inputs') target_var = T.ivector('targets') n_in = [3, 32, 32] X_train, y_train, X_test, y_test, X_test_all, y_test_all = datasets.load_CIFAR10(inside_labels) model = models.convnet_dropout(input_var, target_var, n_in, n_out, dropout_p, weight_decay) df = pd.DataFrame() # Mini-batch training with ADAM epochs = training.train(model, X_train, y_train, X_test, y_test, batch_size, num_epochs, acc_threshold) # Mini-batch testing acc, bayes_acc = training.test(model, X_test, y_test, batch_size) df.set_value(experiment_name, "test_acc", acc) df.set_value(experiment_name, "bayes_test_acc", bayes_acc) # Uncertainty prediction test_mean_std_bayesian = {x:[] for x in range(10)} test_mean_std_deterministic = {x:[] for x in range(10)} test_entropy_bayesian = {x:[] for x in range(10)} test_entropy_deterministic = {x:[] for x in range(10)} for i in range(len(X_test_all)): bayesian_probs = model.probabilities(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) bayesian_entropy = model.entropy_bayesian(np.tile(X_test_all[i], batch_size).reshape([-1] + n_in)) classical_probs = model.probabilities_deterministic(X_test_all[i][np.newaxis,:])[0] classical_entropy = model.entropy_deterministic(X_test_all[i][np.newaxis,:]) predictive_mean = np.mean(bayesian_probs, axis=0) predictive_std = np.std(bayesian_probs, axis=0) test_mean_std_bayesian[y_test_all[i]].append(np.concatenate((predictive_mean, predictive_std))) test_entropy_bayesian[y_test_all[i]].append(bayesian_entropy) test_entropy_deterministic[y_test_all[i]].append(classical_entropy) test_mean_std_deterministic[y_test_all[i]].append(classical_probs) # Plotting if plot: for k in sorted(test_mean_std_bayesian.keys()): sns.plt.figure() #sns.plt.hist(test_pred_mean[k], label = "Prediction mean for " + str(k)) sns.plt.hist(test_entropy_bayesian[k], label = "Bayesian Entropy v1 for " + str(k)) #sns.plt.hist(test_pred_std[k], label = "Prediction std for " + str(k)) #sns.plt.hist(test_entropy_deterministic[k], label = "Classical entropy for " + str(k)) sns.plt.legend() sns.plt.show() # Anomaly detection using simple threshold def anomaly_detection_old(anomaly_score_dict, name, df): threshold = np.logspace(-30, 1.0, 1000) acc = {} for t in threshold: tp = 0.0 tn = 0.0 for l in anomaly_score_dict: if l in inside_labels: tp += (np.array(anomaly_score_dict[l]) < t).mean() else: tn += (np.array(anomaly_score_dict[l]) >= t).mean() tp /= len(inside_labels) tn /= 10.0 - len(inside_labels) bal_acc = (tp + tn)/2.0 f1_score = 2.0*tp/(2.0 + tp - tn) acc[t] = [bal_acc, f1_score, tp, tn] print("{}\tscore\tthreshold\tTP\tTN".format(name)) sorted_acc = sorted(acc.items(), key= lambda x : x[1][0], reverse = True) df.set_value(experiment_name, name + ' bal_acc', sorted_acc[0][1][0]) df.set_value(experiment_name, name + ' bal_acc_threshold', sorted_acc[0][0]) print("\tbalanced acc\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][0], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) sorted_acc = sorted(acc.items(), key= lambda x : x[1][1], reverse = True) df.set_value(experiment_name, name + ' f1_score', sorted_acc[0][1][1]) df.set_value(experiment_name, name + ' f1_score_threshold', sorted_acc[0][0]) print("\tf1 score\t{:.3f}\t{:.6f}\t\t{:.3f}\t{:.3f}".format(sorted_acc[0][1][1], sorted_acc[0][0], sorted_acc[0][1][2], sorted_acc[0][1][3])) return df # Anomaly detection using logistic regression def anomaly_detection(anomaly_score_dict, name, df): X = [] y = [] for l in anomaly_score_dict: X += anomaly_score_dict[l] if l in inside_labels: y += [0]*len(anomaly_score_dict[l]) else: y += [1]*len(anomaly_score_dict[l]) X = np.array(X) y = np.array(y) X, y = utils.shuffle(X, y, random_state=0) X_train = X[:len(X)/2] X_test = X[len(X)/2:] y_train = y[:len(y)/2] y_test = y[len(y)/2:] clf = linear_model.LogisticRegression(C=1.0) clf.fit(X_train, y_train) auc = metrics.roc_auc_score(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1]) print("AUC", auc) df.set_value(experiment_name, name + ' AUC', auc) if plot: # Plot ROC curve fpr, tpr, thresholds = metrics.roc_curve(np.array(y_test), clf.predict_proba(np.array(X_test))[:,1], pos_label=1) sns.plt.figure() sns.plt.plot(fpr, tpr, label='ROC curve') sns.plt.plot([0, 1], [0, 1], 'k--') sns.plt.xlim([0.0, 1.0]) sns.plt.ylim([0.0, 1.05]) sns.plt.xlabel('False Positive Rate') sns.plt.ylabel('True Positive Rate') sns.plt.title('Receiver operating characteristic example') sns.plt.legend(loc="lower right") sns.plt.show() return df df.set_value(experiment_name, 'dataset', dataset) df.set_value(experiment_name, 'bayesian_approx', bayesian_approximation) df.set_value(experiment_name, 'inside_labels', str(inside_labels)) df.set_value(experiment_name, 'epochs', epochs) df = anomaly_detection(test_entropy_deterministic, "Classical entropy", df) df = anomaly_detection(test_mean_std_deterministic, "Classical prediction", df) df = anomaly_detection(test_entropy_bayesian, "Bayesian entropy", df) df = anomaly_detection(test_mean_std_bayesian, "Bayesian prediction", df) return df
def test_shared_variables(self): vocab, data, options = self._get_vocab_data_options(True, True) options['n_epochs'] = 1 train(options, data, 2, self.tmp_dir, self.tmp_dir) self.assertEqual(len(tf.global_variables()), 64)
def main(): train_adversarial = 1 use_cuda = True epochs = 200 lr = 0.005 batch_size = 100 test_batch_size = 100 torch.manual_seed(7347) device = 'cuda' if torch.cuda.is_available() else 'cpu' print('using device {0}'.format(device)) kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=False, download=True, transform=transforms.Compose([ transforms.ToTensor(), ])), batch_size=batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, transform=transforms.Compose([ transforms.ToTensor(), ])), batch_size=test_batch_size, shuffle=True, **kwargs) model = Net(activation=nn.LeakyReLU()).to(device) PATH = 'mnist_cnn100.pt.bak' # model.load_state_dict(torch.load(PATH, map_location=device), strict=False) reconstruction_optimizer = optim.AdamW(model.autoenc_params(), lr=lr) discriminative_optimizer = optim.AdamW(model.disc_params(), lr=lr * 0.1) encoder_optimizer = optim.AdamW(model.enc_params(), lr=lr * 0.1) if train_adversarial: compute_loss = compute_loss_adversarial_enc optimizer = { 'rec': reconstruction_optimizer, 'dis': discriminative_optimizer, 'enc': encoder_optimizer } tmp = [ reconstruction_optimizer, discriminative_optimizer, encoder_optimizer ] schedulers = [StepLR(x, step_size=5, gamma=0.9) for x in tmp] else: compute_loss = compute_loss_autoenc optimizer = {'rec': reconstruction_optimizer} schedulers = [StepLR(reconstruction_optimizer, step_size=5, gamma=0.9)] for epoch in range(1, epochs + 1): if epoch % 5 == 0: test(model, compute_loss, device, test_loader) train(model, compute_loss, device, train_loader, optimizer, epoch) for scheduler in schedulers: scheduler.step() if epoch % 10 == 0 and epoch: torch.save(model.state_dict(), "mnist_cnn{0}.pt".format(epoch)) print('learning rate: {0}'.format(scheduler.get_lr()))