def train_and_plot(model, X, y, sorted_index, num_eps=10, num_plots=5, generating_func=None, silent=False): error_list = [] for i in range(num_eps): errors = model.train_and_evaluate(X, y) error_list += errors #gauss_lr.train(X,y) #vanilla.train(X,y) if i % (num_eps / num_plots) == 0: #make sure X is new data! gauss_preds = model.predict(X) gauss_var = model.predict_var(X) #3lr_preds = gauss_lr.predict(X) #lr_var = gauss_lr.predict_var(X) #vanilla_preds = vanilla.predict(X) if not silent: plot_prediction(X, gauss_preds, sorted_index, gauss_var, generating_func=generating_func) plt.plot(X, y, 'x') plt.show() evaluate_model(X, y, gauss_preds, var=gauss_var) #plt.plot(np.squeeze(gauss_lr_error_list)) return error_list
def train_new_model_cli(device): print("\n") print("Camull-Net can be trained for two seperate tasks.") print( "1. NC v AD : Distinguishing between subjects from the normal cohort group and subjects with Alzheimers Disease." ) print( "2. sMCI v pMCI : Distinguishing between subjects with static mild cognitive impairement and progressive mild cognitive impairement." ) print("\n") choice = input("Please enter your choice [(1),2]:") if choice == "": choice = 1 else: choice = int(choice) if choice == 1: task = Task.NC_v_AD else: task = Task.sMCI_v_pMCI ld_helper = LoaderHelper(task) print("\n") epochs = input("How many epochs would you like to do? (default: 40): ") print("\n") uuid = "" if epochs == "": uuid = start(device, ld_helper, 40) else: try: uuid = start(device, ld_helper, int(epochs)) except: print("Number of epochs must be a valid number.") print("\n") print("A new {} model has been trained under the tag: {}".format( str(task), uuid)) print("\n") print( "Would you like to evaluate it? You must do so for it to be saved to the database." ) print("\n") choice = input("Enter your choice [Y/n]: ") if choice == 'Y' or 'y' or '': choice = 1 else: choice = 0 if (int(choice) == 1): print("\n") print("There are 5 folds to evaluate") print( "Input a fold number to evaluate or input 6 to evaluate all folds." ) print("\n") choice = int(input("Enter your choice [1,6]: ")) if (choice != 6): evaluate_fold(device, uuid, ld_helper, choice, cur) else: evaluate_model(device, uuid, ld_helper, cur) else: basic_run(device) #loop back round to the start of the program.
def main_eval(args): assert args.load_from is not None, '--load_from required in eval mode' logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) dataset_train, dataset_test, scaler = get_data(args) logging.info(f'evaluation mode. Level: {args.level}') device = torch.device( 'cuda:0') if torch.cuda.is_available() else torch.device('cpu') n_features = dataset_train.items.shape[1] generator, discriminator = get_models(args, n_features, device) experiment = Experiment(args.comet_api_key, project_name=args.comet_project_name, workspace=args.comet_workspace) experiment.log_parameters(vars(args)) load_model(Path(args.load_from), generator, discriminator, None, None, device) n_events = len(dataset_test) steps = (args.gan_test_ratio * n_events) // args.eval_batch_size evaluate_model(generator, experiment, dataset_test, args.eval_batch_size, steps, args, device, scaler, 0)
def main(): args = get_arguments() utils.print_args(args) dataset = data_loading.get_dataset(args.dataset, args.normalize_raw, args.normalize_reps, args.cifar_path) print("Collected arguments and raw dataset.") if args.network_type == 'simple': input_shape = args.dim_red if args.dim_red is not None else dataset.get_raw_input_shape( ) rounds.add_network_to_vector_rounds(args.rounds, dataset, input_shape, args.neurons, args.max_iter_optimization, args.alpha_optimization, args.n_train, args.dim_red, network_type='simple') elif args.network_type == 'conv': input_shape = dataset.get_raw_input_shape(True) rounds.add_network_to_vector_rounds(args.rounds, dataset, input_shape, args.neurons, args.max_iter_optimization, args.alpha_optimization, args.n_train, None, network_type='conv') else: raise ValueError("Network type {} not supported".format( args.network_type)) print("Finished getting Representations") print("Getting represented dataset:") print("Getting training examples...") x, y = dataset.get_training_examples(args.n_train, dim_reduction=args.dim_red, print_progress=True) print("Getting test examples...") x_test, y_test = dataset.get_test_examples(args.n_test, dim_reduction=args.dim_red, print_progress=True) print("Getting final linear separator") w = svm.get_linear_separator(x, y, type_of_classifier='sdca', verbose=2, alpha=args.alpha_evaluation, max_iter=args.max_iter_evaluation) performance = evaluation.evaluate_model(w, x, y) print("train performance is {}".format(performance)) performance = evaluation.evaluate_model(w, x_test, y_test) print("test performance is {}".format(performance))
def single_run(args, dataset): model = NeuMF(args, dataset.num_users, dataset.num_items) model.build_model() sess = tf.Session() init = tf.global_variables_initializer() sess.run(init) t1 = time() ahit, andcg = evaluate_model(sess, model, dataset, args.topk) best_hr, best_ndcg, best_iter = ahit, andcg, -1 print('Init: HR = %.4f, NDCG = %.4f\t [%.1f s]' % (ahit, andcg, time() - t1)) tf.summary.scalar('loss', best_hr) summaryMerged = tf.summary.merge_all() writer = tf.summary.FileWriter('./graph_tensorboard', sess.graph) for epoch in range(args.epochs): t1 = time() train_users, train_items, train_labels, num_inst = \ dataset.make_training_instances(args.num_neg_inst) # print(train_labels[0:20]) loss_per_epoch, error_per_epoch = 0, 0 for ite in range((num_inst - 1) // args.batch_size + 1): start_idx = ite * args.batch_size end_idx = min((ite + 1) * args.batch_size, num_inst) cur_user_indices = train_users[start_idx: end_idx] cur_item_indices = train_items[start_idx: end_idx] cur_label = train_labels[start_idx: end_idx] _, loss, error = sess.run( [model.train_step, model.loss, model.raw_error], {model.user_indices: cur_user_indices, model.item_indices: cur_item_indices, model.ratings: cur_label}) loss_per_epoch += loss error_per_epoch += error summary = sess.run(summaryMerged) writer.add_summary(summary, epoch) error_per_epoch /= num_inst t2 = time() if epoch % args.verbose == 0: ahit, andcg = evaluate_model(sess, model, dataset, args.topk) print('epoch %d\t[%.1f s]: HR= %.4f\tNDCG= %.4f\tloss= %.4f\terror' '= %.4f\t[%.1f s]' % ( epoch, t2 - t1, ahit, andcg, loss_per_epoch, error_per_epoch, time() - t2)) if ahit > best_hr: best_hr = ahit best_iter = epoch if andcg > best_ndcg: best_ndcg = andcg print("End. Best Epoch %d: HR = %.4f, NDCG = %.4f. " % ( best_iter, best_hr, best_ndcg))
def main(): starting_time = time.time() train_sents, test_sents = open_data() X_train, y_train, X_test, y_test = define_train_test(train_sents, test_sents) crf = train_model(X_train, y_train) evaluation.evaluate_model(crf, X_test, y_test) evaluation.label_transitions(Counter(crf.transition_features_).most_common()) evaluation.feature_transitions(Counter(crf.state_features_).most_common()) #best_crf = evaluation.hyperparameter_optimization(X_train, y_train, list(crf.classes_)) #evaluation.evaluate_model(best_crf, X_test, y_test) save_model(crf) print('Execution time:', round(time.time() - starting_time, 3), 'seconds')
def val_model(model, epoch_i): if config["kde_val"]: # Eval KDE ev.evaluate_model(model, data=val_data, config=config, epoch_i=epoch_i, kde=True, make_plots=False) # Eval true return ev.evaluate_model(model, data=val_data, config=config, epoch_i=epoch_i)
def main(): #CN v AD ld_helper = loader_helper(task=Task.CN_v_AD) #uuid = train_camull(ld_helper, epochs=40) #evaluate_model(device, "c51bf83c4455416e8bc8b1ebbc8b75ca", ld_helper) #transfer learning for pMCI v sMCI ld_helper.change_task(Task.sMCI_v_pMCI) model = load_model( "camull", "../weights/CN_v_AD/c51bf83c4455416e8bc8b1ebbc8b75ca/fold_5_weights-2020-04-29_13:17:33" ) uuid = train_camull(ld_helper, model=model, epochs=40) evaluate_model(device, uuid, ld_helper)
def handle_annotated_english_tweets(config, folder): tweets_train_X, tweets_train_y, tweets_eval_X, tweets_eval_y, tweets_test_X, tweets_test_y = get_english_annotated_tweets_sets( config, folder) softmax_values, predicted_classes = evaluate_model(config, folder, tweets_test_X, tweets_test_y, None, None, True)
def test(self, topK, epoch_num=0): total_ahit, total_andcg = [], [] for i in xrange(self.testset.epoch): # # user_list,pos_items, neg_items, mask_frame, mask, feats_idx= self.trainset.get_batch(i) user_list, mask_frame, mask, feats_idx, pos_items = self.testset.get_batch( i) feat = self.video_features.take(feats_idx, axis=0) [user_vector, V_matrix, time_attent, img_attent] = self.test_model(user_list, mask_frame, mask, feat) #V_value = np.asarray(V_matrix.eval()) score_maxtrix = np.dot(user_vector, V_matrix.T) # index_top_K = score_maxtrix.argsort()[:,-topK:][:,::-1] ahit, andcg = evaluation.evaluate_model(score_maxtrix, self.testDataset2, topK, user_list, pos_items) total_ahit.append(ahit) total_andcg.append(andcg) #save_attent(time_attent, img_attent, i) print(i) #print type(time_attent) #print type(img_attent) #print img_attent.shape img_attent = lower_dim(img_attent) feats_idx = lower_dim(feats_idx) try: time_attent0 = np.loadtxt("Output/epoch_" + str(epoch_num) + "_time_attent.csv", delimiter=",") user_list0 = np.loadtxt("Output/epoch_" + str(epoch_num) + "_user_list.csv", delimiter=",") img_attent0 = np.loadtxt("Output/epoch_" + str(epoch_num) + "_img_attent.csv", delimiter=",") feats_idx0 = np.loadtxt("Output/epoch_" + str(epoch_num) + "_feats_idx.csv", delimiter=",") #print time_attent0 time_attent = np.concatenate([time_attent0, time_attent]) user_list = np.concatenate([user_list0, user_list]) img_attent = np.concatenate([img_attent0, img_attent]) feats_idx = np.concatenate([feats_idx0, feats_idx]) except Exception, e: #print 'time_attent.csv does note exist, creating' print Exception, ":", e np.savetxt("Output/epoch_" + str(epoch_num) + "_time_attent.csv", time_attent, delimiter=",") np.savetxt("Output/epoch_" + str(epoch_num) + "_user_list.csv", user_list, delimiter=",") np.savetxt("Output/epoch_" + str(epoch_num) + "_img_attent.csv", img_attent, delimiter=",") np.savetxt("Output/epoch_" + str(epoch_num) + "_feats_idx.csv", feats_idx, delimiter=",")
def demographic_parity_train(model, dr, vdr, vvector, args): feat, rel = dr.data N = len(rel) from utils import get_optimizer optimizer = get_optimizer(model.parameters(), args.lr[0], args.optimizer, weight_decay=args.weight_decay[0]) for epoch in range(args.epochs[0]): for i in range(N): feat, rel = shuffle_combined(feat, rel) optimizer.zero_grad() curr_feats = feat[i] scores = model(torchify(curr_feats)).squeeze() probs = torch.nn.Softmax(dim=0)(scores) if np.sum(rel[i]) == 0: continue normalized_rels = rel[i] # / np.sum(rel[i]) # np.random.shuffle(normalized_rels) ranking_loss = -torch.sum( torch.FloatTensor(normalized_rels) * torch.log(probs)) # print(scores, probs, # torch.log(probs), normalized_rels, # torch.log(probs) * torch.FloatTensor(normalized_rels), # ranking_loss) exposures = vvector[0] * probs groups = curr_feats[:, args.group_feat_id] if np.all(groups == 0) or np.all(groups == 1): fairness_loss = 0.0 else: avg_exposure_0 = torch.sum( torch.FloatTensor(1 - groups) * exposures) / torch.sum(1 - torch.FloatTensor(groups)) avg_exposure_1 = torch.sum( torch.FloatTensor(groups) * exposures) / torch.sum( torch.FloatTensor(groups)) # print(avg_exposure_0, avg_exposure_1) fairness_loss = torch.pow( torch.clamp(avg_exposure_1 - avg_exposure_0, min=0), 2) loss = args.lambda_reward * ranking_loss + args.lambda_group_fairness * fairness_loss loss.backward() optimizer.step() # break if i % args.evaluate_interval == 0 and i != 0: results = evaluate_model(model, vdr, fairness_evaluation=False, group_fairness_evaluation=True, deterministic=True, args=args, num_sample_per_query=100) print(results) return model
def eval_params(w, bias, dr, D, det=False, args=None, intercept=True): # Given the model weights, this function evaluates the model model = LinearModel(D=D) model.w.weight.data = torch.FloatTensor([w]) if intercept: model.w.bias.data = torch.FloatTensor([bias]) return evaluate_model(model, dr, deterministic=det, group_fairness_evaluation=True, args=args, fairness_evaluation=True)
def experiment_rbm_learning(): """Train and test an RBM with normal toy data""" trainingset = generate_dataset(N_train) testset = generate_dataset(N_test) model = StackedRBMs(trainingset.layer_sizes) # Training train_unsupervised(model, trainingset) # Testing results = evaluate_model(model, testset) return results
def experiment_mlp_learning(): """Test if an MLP learns the desired features""" trainingset = generate_dataset(N_train) testset = generate_dataset(N_test) model = MLP(trainingset.layer_sizes) # Training train_supervised(model, trainingset) # Testing results = evaluate_model(model, testset) return results
def experiment_rbm_power(): """Test if an RBM has the representational power to learn the desired features""" trainingset = generate_dataset(N_train) testset = generate_dataset(N_test) model = StackedRBMs(trainingset.layer_sizes) # Train the model using also the (not so) hidden labels train_with_hidden_labels(model, trainingset) # Testing results = evaluate_model(model, testset) return results
def train_and_evaluate(): # reload weights from restore_file if specified if args.restore_file is not None: restore_path = os.path.join( args.tensorboard_dir, args.restore_file + '.pth.tar') if os.path.exists(restore_path): logging.info("Restoring parameters from {}".format(restore_path)) utils.load_checkpoint(restore_path, model.module, optimizer) best_val_f1 = 0.0 for epoch in range(1, model_params.num_epochs+1): # Run one epoch logging.info("Epoch {}/{}".format(epoch, model_params.num_epochs)) # compute number of batches in one epoch (one full pass over the training set) train(epoch) # Evaluate for one epoch on validation set val_metrics = evaluate_model(model.module, val_dataset, report_dir=args.tensorboard_dir) logging.info(val_metrics) val_f1 = val_metrics['Weighted Macro F1'] is_best = val_f1 >= best_val_f1 # Save weights utils.save_checkpoint({'epoch': epoch, 'state_dict': model.module.state_dict(), 'optim_dict': optimizer.state_dict()}, is_best=is_best, checkpoint=args.tensorboard_dir) # If best_eval, best_save_path if is_best: logging.info("- Found new best macro f1") best_val_f1 = val_f1 # Save best val metrics in a json file in the model directory best_json_path = os.path.join( args.tensorboard_dir, "metrics_val_best_weights.json") utils.save_dict_to_json(val_metrics, best_json_path) last_report_path = os.path.join(args.tensorboard_dir, 'report.txt') best_report_path = os.path.join(args.tensorboard_dir, 'best_report.txt') if os.path.exists(last_report_path): shutil.copy(last_report_path, best_report_path) # Save latest val metrics in a json file in the model directory last_json_path = os.path.join( args.tensorboard_dir, "metrics_val_last_weights.json") utils.save_dict_to_json(val_metrics, last_json_path)
def main(): args, params = read_params_and_args() params = adjust_params(args, params) # prepare data (download and extract) if args.prep_data: data_handler.prepare_data(params) # evaluation elif args.eval: evaluation.evaluate_model(args, params) elif args.eval_bbox: evaluation.evaluate_iobb(args, params, args.bbox_image, args.bbox_disease) else: model = networks.init_unified_net(args.model, params) optimizer = torch.optim.Adam(params=model.parameters(), lr=params['lr']) tracker = helper.init_comet(args, params) if args.use_comet else None trainer.train(args, params, model, optimizer, tracker)
def experiment_mlp_power(): """Try to find 'ideal' parameters to test if the MLP has the representational power to required to learn the desired features. """ trainingset = generate_dataset(N_train) testset = generate_dataset(N_test) model = MLP(trainingset.layer_sizes) # Train the model using also the (not so) hidden labels train_with_hidden_labels(model, trainingset) # Testing results = evaluate_model(model, testset) return results
def evaluate(model, shortname, fullname, with_aux): shortname = shortname + ('_aux' if with_aux else '') s = '' if with_aux else 'out' print( f'Training "{fullname} ({shortname})" model with{s} auxiliary training loss' ) torch.manual_seed(args.seed) loss, acc = evaluate_model(model, num_epochs=args.epochs, num_rounds=args.rounds, with_aux_classes=with_aux, aux_loss_factor=args.aux_loss_factor) losses.append(loss) accuracies.append(acc) model_names.append(shortname)
def experiment_mlp_power_uncorrelated_features(): """A variation of experiment_mlp_power. Train the hidden layer with an 'uncorrelated' dataset, so it can not cheat and learn to detect features by looking for commonly co-occurring features. """ uncorrelated_trainingset = generate_uncorrelated_dataset(N_train) trainingset = generate_dataset(N_train) testset = generate_dataset(N_test) model = MLP(trainingset.layer_sizes) model.train_bottom(*uncorrelated_trainingset.get_layers()[:-1]) model.train_top(*trainingset.get_layers()[1:]) results = evaluate_model(model, testset) return results
def experiment_rbm_power_stability(): """Test if 'ideal' parameters for us also form an optimum for the RBMs We first do the test for representational power, and then continue with unsupervised training to see whether the parameters stay around the same values. """ # First, train with hidden labels to test representational power results0 = experiment_rbm_power() model = results0['model'] testset = results0['testset'] # Then, unsupervised training to see if parameters stay or change trainingset1 = generate_dataset(N_train) train_unsupervised(model, trainingset1) results1 = evaluate_model(model, testset) return results0, results1
def configure(self, resolution, min_fps): """ Evaluate model performance and select model with desired fps Parameters: fps (int): Minimum frames per second for executing model """ model_found = False for args in MODEL_RANKING: perf = evaluate_model(args, resolution) if perf.fps < min_fps: self.upsampler = Upsampler(args) model_found = True break self.configured = False return model_found
def evaluate_and_store_results(y_test, y_pred, model, example_based_results, label_based_results, print_results=True): # example based evaluation accuracy_score, precision_score, \ recall_score, f1_score, hamming_loss, classfication_report = evaluate_model(y_test, y_pred, print_results=print_results) # label based evaluation accuracy_per_label, precision_per_label, \ recall_per_label, f1_per_label = evaluate_per_label(y_test, y_pred, print_results=True) example_based_results = example_based_results.append( { 'representation': representation, 'estimator': str(estimator), 'model': model, 'accuracy': accuracy_score, 'precision': precision_score, 'recall': recall_score, 'f1': f1_score, 'hamming_loss': hamming_loss }, ignore_index=True) label_based_results = label_based_results.append( { 'representation': representation, 'estimator': str(estimator), 'model': model, 'accuracy': accuracy_per_label, 'precision': precision_per_label, 'recall': recall_per_label, 'f1': f1_per_label }, ignore_index=True) return example_based_results, label_based_results
def evaluate(self, mat, masked_mat, top_n): R_hat = self.user_vectors @ self.item_vectors.T MAP, rec_at_k, mpr_all, mpr_mask = evaluate_model( R_hat, mat, masked_mat, top_n) return MAP, rec_at_k, mpr_all, mpr_mask
from keras.callbacks import EarlyStopping # create the models model = Sequential() model.add(Embedding(len(word_index) + 1, 128, input_shape=(280, ))) model.add(Conv1D(128, 16, activation='relu')) model.add(MaxPooling1D(pool_size=8)) model.add(Flatten()) model.add(Dropout(0.5)) model.add(Dense(100, input_shape=(X_train.shape[1], ))) model.add(Activation('sigmoid')) model.add(Dropout(0.1)) model.add(Dense(y_train.shape[1], activation="softmax")) model.compile(loss="categorical_crossentropy", optimizer=Adam(), metrics=["accuracy"]) model.summary() history = model.fit(X_train, y_train, batch_size=32, epochs=5, shuffle=True, verbose=1) # evaluation evaluate_model(model, X_test, y_test)
def main(): parser = create_parser() args = parser.parse_args() if args.setup: create_directories() if args.debug: dataset = DATASETS['debug'] args.dataset = "debug" features, _, labels, _ = preprocess_data(args.patch_size, args.distribution, dataset=dataset) #print(features, 'debug') #print("length of features: ",type(features), len(features),'element.shape: ',features[0][0]) features_train, features_test = features[:100], features[100:120] labels_train, labels_test = labels[:100], labels[100:120] elif args.train_model or args.evaluate_model or args.preprocess_data: dataset = DATASETS[args.dataset] #print(dataset.values()) load_from_cache = not args.preprocess_data try: features_train, features_test, labels_train, labels_test = preprocess_data( args.patch_size, args.distribution, dataset=dataset, only_cache=load_from_cache) #print(features_train, 'train_model or evaluate_model or preprocess_data') print("Length of features_train: ", len(features_train)) except IOError: print("Cache file does not exist. Please run again with -p flag.") sys.exit(1) if args.visualise: visualise_labels(labels_train, args.patch_size, LABELS_DIR) visualise_labels(labels_test, args.patch_size, LABELS_DIR) if not args.model_id: timestamp = time.strftime("%d_%m_%Y_%H%M") model_id = "{}_{}_{}".format(timestamp, args.dataset, args.architecture) else: model_id = args.model_id if args.init_model or args.train_model or args.evaluate_model: model_dir = os.path.join(OUTPUT_DIR, model_id) save_makedirs(model_dir) # Hyperparameters for the model. Since there are so many of them it is # more convenient to set them in the source code as opposed to passing # them as arguments to the Command Line Interface. We use a list of tuples instead of a # dict since we want to print the hyperparameters and for that purpose # keep them in the predefined order. hyperparameters = [ ("architecture", args.architecture), # Hyperparameters for the first convolutional layer. ("nb_filters_1", 64), ("filter_size_1", 9), ("stride_1", (2, 2)), # Hyperparameter for the first pooling layer. ("pool_size_1", (2, 2)), # Hyperparameter for the second convolutional layer (when # two layer architecture is used). ("nb_filters_2", 128), ("filter_size_2", 5), ("stride_2", (1, 1)), # Hyperparameters for Stochastic Gradient Descent. ("learning_rate", 0.05), ("momentum", 0.9), ("decay", 0.0) ] hyperparameters_mnih = [ ("architecture", args.architecture), # Hyperparameters for the first convolutional layer. ("nb_filters_1", 64), ("filter_size_1", 16), ("stride_1", (4, 4)), # Hyperparameter for the first pooling layer. ("pool_size_1", (2, 2)), ("pool_stride", 1), # Hyperparameter for the second convolutional layer). ("nb_filters_2", 112), ("filter_size_2", 4), ("stride_2", (1, 1)), # Hyperparameter for the third convolutional layer). ("nb_filters_3", 80), ("filter_size_3", 3), ("stride_3", (1, 1)), # Hyperparameters for Stochastic Gradient Descent. ("learning_rate", 0.05), ("momentum", 0.9), ("decay", 0.0) ] if args.init_model: model = init_model(args.patch_size, model_id, **dict(hyperparameters_mnih)) save_model_summary(hyperparameters_mnih, model, model_dir) elif args.train_model or args.evaluate_model: hyperparameters = dict(hyperparameters_mnih) model = load_model(model_id) model = compile_model(model, hyperparameters["learning_rate"], hyperparameters['momentum'], hyperparameters["decay"]) if args.train_model: model = train_model(model, features_train, labels_train, args.patch_size, model_id, model_dir, nb_epoch=args.epochs, checkpoints=args.checkpoints, tensorboard=args.tensorboard, earlystop=args.earlystop) if args.evaluate_model: evaluate_model(model, features_test, labels_test, args.patch_size, model_dir, out_format=args.out_format)
def main(): #CN v AD ld_helper = loader_helper(task=Task.CN_v_AD) uuid = train_camull(ld_helper, epochs=40) evaluate_model(device, uuid, ld_helper)
def evaluate_a_model(device): print("Which task would you like to evaluate?") print("1) NC v AD") print("2) sMCI v pMCI") choice = int(input("Please enter your input [1..2]: ")) task = Task if choice == 1: task = Task.NC_v_AD else: task = Task.sMCI_v_pMCI print("Here are 10 of your most recent unevaluated {} models.".format( str(task))) print("\n") print(" id | model uuid | Time | model task") model_uuids = fetch_models_from_db(task) target_uuid = "" if not model_uuids == []: valid = False while (not valid): choice = input( "Please enter the model number [1, 10] or the uuid that you would like to choose:" ) try: target_uuid = model_uuids[int(choice) - 1] valid = True except: for i in range(len(model_uuids)): if choice == model_uuids: target_uuid = choice break print("Please enter a valid input.") ld_helper = LoaderHelper(task) print("There are 5 folds to evaluate") print( "Input a fold number to evaluate or input 6 to evaluate all folds." ) print("\n") choice = int(input("Enter your choice [1,6]: ")) if (choice != 6): evaluate_fold(device, target_uuid, ld_helper, choice, commit_to_db=True) remove_from_db(target_uuid) else: evaluate_model(device, target_uuid, ld_helper, cur) remove_from_db(target_uuid) print( "The model has been evaluated. Check the graphs folder and look at the metrics in the database." ) basic_run(device) else: print("\n") print("No models available. Please train a new model.") choice = input("Would you like to train a new model[Y/n]?: ") if choice == 'y' or 'Y' or '': train_new_model_cli else: basic_run
def main(): config = get_config() # Set all random seeds seed_all(config["seed"]) # Figure out what device to use, (GP needs data in cpu-memory) if (not config["cpu"]) and (not config["model"] == "gp") and torch.cuda.is_available(): device = torch.device("cuda") else: device = torch.device("cpu") # Read data dataset = dataset_list.get_dataset_spec(config["dataset"])() train_data, val_data, test_data = dataset.load(device) config["x_dim"] = dataset.x_dim config["y_dim"] = dataset.y_dim # Init wandb if config["name"] == None: config["name"] = "{}-{}-{}".format(config["model"], config["dataset"], time.strftime("%H-%M")) tags = [] if config["test"]: tags.append("test") # Also setting config wandb.init(project=constants.WANDB_PROJECT, config=config, name=config["name"], tags=tags) # Set computational device (should not be synched to wandb) config["device"] = device # Load model model = models[config["model"]](config) # Train model if config["train"]: # Create evaluation function to feed to model for use during training def val_model(model, epoch_i): if config["kde_val"]: # Eval KDE ev.evaluate_model(model, data=val_data, config=config, epoch_i=epoch_i, kde=True, make_plots=False) # Eval true return ev.evaluate_model(model, data=val_data, config=config, epoch_i=epoch_i) model.train(train_data, config, val_func=val_model) # Test model if config["test"]: # determine kernel scaling from evaluation set _, best_ks = utils.kde_eval(model, val_data, config) print("Kernel Scale: {}".format(best_ks)) model.kernel_scale = best_ks # Store best kernel scale in model # Get true (according to model) log-likelihood true_evaluation_vals = ev.evaluate_model( model, test_data, config=config, make_plots=True) # Make plots only this run # Average testing using KDE over multiple random seeds eval_list = [] # list of dicts mapping eval_metric -> value for i in range(config["test_runs"]): seed_all(constants.TEST_SEEDS[i]) evaluation_vals = ev.evaluate_model(model, test_data, config=config, make_plots=False, kde=True) eval_list.append(evaluation_vals) wandb_test_dict = {} for key in eval_list[0].keys(): eval_values = [val_dict[key] for val_dict in eval_list] wandb_test_dict["test_{}_mean".format(constants.EVAL_NAME_MAP[key])] =\ np.mean(eval_values) wandb_test_dict["test_{}_std_dev".format(constants.EVAL_NAME_MAP[key])] =\ np.std(eval_values) if key in true_evaluation_vals: wandb_test_dict["test_{}_true".format(constants.EVAL_NAME_MAP[key])] =\ true_evaluation_vals[key] wandb.log(wandb_test_dict) test_print_string = "\t".join([]) print("Test metrics over {} seeds".format(config["test_runs"])) for key in eval_list[0].keys(): wandb_key = "test_{}_".format(constants.EVAL_NAME_MAP[key]) print("{}: {:.5}±{:.5}".format( key, wandb_test_dict[wandb_key + "mean"], wandb_test_dict[wandb_key + "std_dev"], )) if key in true_evaluation_vals: print("true {}: {:.5}".format( key, wandb_test_dict[wandb_key + "true"])) if config["eval_div"]: divergence = ev.estimate_divergences(model, config, { "train": train_data, "val": val_data, "test": test_data, })
nthread=1, subsample=0.7500000000000001)), Normalizer(norm="max"), StackingEstimator( estimator=LogisticRegression(C=0.5, dual=False, penalty="l1")), StackingEstimator( estimator=LogisticRegression(C=0.5, dual=False, penalty="l1")), Binarizer(threshold=0.25), XGBClassifier(learning_rate=0.001, max_depth=1, min_child_weight=10, n_estimators=100, nthread=1, subsample=1.0)) exported_pipeline.fit(training_features, training_target) y_pred = exported_pipeline.predict(testing_features).astype(np.float64) y_prob = exported_pipeline.predict_proba(testing_features)[:, 1].astype( np.float64) evaluate_model(y_test.astype(np.float64), y_pred, y_prob, model_name='tpot', country=COUNTRY, predict_pov_rate=True, store_model=True, show=False, model=exported_pipeline) print("done")