def run_HAC(FLAGS, env, agent): # Print task summary print_summary(FLAGS, env) # Determine training mode. If not testing and not solely training, interleave training and testing to track progress mix_train_test = False if not FLAGS.test and not FLAGS.train_only: mix_train_test = True # Track total training episodes completed total_episodes = 0 for batch in range(NUM_BATCH): num_episodes = agent.other_params["num_exploration_episodes"] # Evaluate policy every TEST_FREQ batches if interleaving training and testing if mix_train_test and batch % TEST_FREQ == 0: print("\n--- TESTING ---") agent.FLAGS.test = True num_episodes = num_test_episodes # Reset successful episode counter successful_episodes = 0 for episode in range(num_episodes): print("\nBatch %d, Episode %d" % (batch, episode)) # Train for an episode success = agent.train(env, episode, total_episodes) if FLAGS.train_only or (mix_train_test and batch % TEST_FREQ != 0): total_episodes += 1 if success: print("Batch %d, Episode %d End Goal Achieved\n" % (batch, episode)) # Increment successful episode counter if applicable if mix_train_test and batch % TEST_FREQ == 0: successful_episodes += 1 # Save agent agent.save_model(episode) # Finish evaluating policy if tested prior batch if mix_train_test and batch % TEST_FREQ == 0: # Log performance success_rate = successful_episodes / num_test_episodes * 100 print("\nTesting Success Rate %.2f%%" % success_rate) agent.log_performance(success_rate) agent.FLAGS.test = False print("\n--- END TESTING ---\n")
def infinite_fcn(train_embedding, test_embedding, data_set, binary=True): _, _, kernel_fn = stax.serial( stax.Dense(64, 2., 0.05), stax.Relu(), stax.Dense(32, 2., 0.05), stax.Relu(), stax.Dense(4, 2., 0.05), stax.Relu(), ) # 0 for no batching, whole batch kernel_fn = nt.batch(kernel_fn, device_count=0, batch_size=0) start = time.time() # Bayesian and infinite-time gradient descent inference with infinite network. #for i in range(10): predict_fn = \ nt.predict.gradient_descent_mse_ensemble(kernel_fn, train_embedding, data_set['Y_train'], diag_reg_absolute_scale=True, learning_rate=1, diag_reg=1e-3) #1e0 1e-3 nngp_mean, nngp_covariance = predict_fn(x_test=test_embedding, get='nngp', compute_cov=True) #fx_test_nngp.block_until_ready() #fx_test_ntk.block_until_ready() duration = time.time() - start print('Kernel construction and inference done in %s seconds.' % duration) # Print out accuracy and loss for infinite network predictions. loss = lambda fx, y_hat: 0.5 * np.mean((fx - y_hat)**2) utils.print_summary('NNGP test', data_set['Y_test'], nngp_mean, None, loss, nngp_covariance, binary=binary)
print(f"Graph {graph} at ec1 vs {epoch} with feature {feature_group}") X, y, group = filter_dataframe(graph, epoch, feature_group) # Create a pipeline pipe = Pipeline([('imputer', SimpleImputer(missing_values=np.nan, strategy='mean')), ('scaler', StandardScaler()), ('clf', DummyEstimator())]) # Placeholder Estimator # We will use as many processor as possible for the gridsearch gs = GridSearchCV(pipe, search_space, cv=LeaveOneGroupOut(), n_jobs=-1) accuracies, f1s, cms, best_params = classify_loso_model_selection( X, y, group, gs) # Saving the performance metrics clf_data = { 'accuracies': accuracies, 'f1s': f1s, 'cms': cms, 'best_params': best_params, } best_clf_file = open(best_clf_filename, 'ab') pickle.dump(clf_data, best_clf_file) best_clf_file.close() # Print out the summary in the console print_summary(accuracies, group)
def show_reviews_per_person_dist(reviews_counts, idcol): count_table = utils.create_value_counts_table(reviews_counts, 'num_of_reviews', 'num_of_guest_reviews') print(count_table.head(10)) utils.print_summary(reviews_counts['num_of_reviews'], False)
def main(): try: # init spark spark = get_spark(app_name="pyspark-xgb") # get logger logger = get_logger(spark, "app") # load data train = spark.read.csv(DATASET_PATH + "/iris_train.csv", get_mtrain_schema(), header=True) test = spark.read.csv(DATASET_PATH + "/iris_test.csv", get_mtrain_schema(), header=True) # preprocess # get label encode result from csv, since StringIndexer will get different result STR_LABEL = 'class' LABEL = 'label' FEATURES = 'features' N_CLASS = 3 features = [c for c in train.columns if c not in [STR_LABEL, LABEL]] assembler = VectorAssembler(inputCols=features, outputCol=FEATURES) pipeline = Pipeline(stages=[assembler]) preprocess = pipeline.fit(train) train = preprocess.transform(train).select(FEATURES, LABEL) test = preprocess.transform(test).select(FEATURES, LABEL) # set param map xgb_params = { "eta": 0.1, "eval_metric": "mlogloss", "gamma": 0, "max_depth": 5, "min_child_weight": 1.0, "objective": "multi:softprob", "seed": 0, "num_class": N_CLASS, # xgboost4j only "num_round": 100, "num_early_stopping_rounds": 10, "maximize_evaluation_metrics": False, "num_workers": 1, "use_external_memory": False, "missing": np.nan, } scala_map = spark._jvm.PythonUtils.toScalaMap(xgb_params) # set evaluation set eval_set = {'eval': test._jdf} scala_eval_set = spark._jvm.PythonUtils.toScalaMap(eval_set) logger.info('training') j = JavaWrapper._new_java_obj( "ml.dmlc.xgboost4j.scala.spark.XGBoostClassifier", scala_map) \ .setFeaturesCol(FEATURES).setLabelCol(LABEL) \ .setEvalSets(scala_eval_set) jmodel = j.fit(train._jdf) print_summary(jmodel) # get validation metric preds = jmodel.transform(test._jdf) pred = DataFrame(preds, spark) slogloss = pred.withColumn('log_loss', udf_logloss(LABEL, 'probability', N_CLASS)) \ .agg({"log_loss": "mean"}).collect()[0]['avg(log_loss)'] logger.info('[xgboost4j] valid logloss: {}'.format(slogloss)) # save model - using native booster for single node library to read model_path = MODEL_PATH + '/model.bin' logger.info('save model to {}'.format(model_path)) jbooster = jmodel.nativeBooster() jbooster.saveModel(model_path) # get feature score imp_type = "gain" feature_map_path = MODEL_PATH + '/feature.map' create_feature_map(feature_map_path, features) jfeatureMap = jbooster.getScore(feature_map_path, imp_type) f_imp = dict() for feature in features: if not jfeatureMap.get(feature).isEmpty(): f_imp[feature] = jfeatureMap.get(feature).get() feature_imp_path = MODEL_PATH + '/feature.imp' create_feature_imp(feature_imp_path, f_imp) # [Optional] load model training by xgboost, predict and get validation metric local_model_path = LOCAL_MODEL_PATH + '/model.bin' if os.path.exists(local_model_path): logger.info('load model from {}'.format(local_model_path)) scala_xgb = spark.sparkContext._jvm.ml.dmlc.xgboost4j.scala.XGBoost jbooster = scala_xgb.loadModel(local_model_path) # uid, num_class, booster xgb_cls_model = JavaWrapper._new_java_obj( "ml.dmlc.xgboost4j.scala.spark.XGBoostClassificationModel", "xgbc", N_CLASS, jbooster) jpred = xgb_cls_model.transform(test._jdf) pred = DataFrame(jpred, spark) slogloss = pred.withColumn('log_loss', udf_logloss(LABEL, 'probability', N_CLASS)) \ .agg({"log_loss": "mean"}).collect()[0]['avg(log_loss)'] logger.info('[xgboost] valid logloss: {}'.format(slogloss)) else: logger.info( "local model is not exist, call python_xgb/train_multi.py to get the model " "and compare logloss between xgboost and xgboost4j" ) except Exception: logger.error(traceback.print_exc()) finally: # stop spark spark.stop()
def main(argv=None): tstart = time.time() # Arguments parser = argparse.ArgumentParser( description= 'FACIL - Framework for Analysis of Class Incremental Learning') # miscellaneous args parser.add_argument('--gpu', type=int, default=0, help='GPU (default=%(default)s)') parser.add_argument('--results-path', type=str, default='../results', help='Results path (default=%(default)s)') parser.add_argument('--exp-name', default=None, type=str, help='Experiment name (default=%(default)s)') parser.add_argument('--seed', type=int, default=0, help='Random seed (default=%(default)s)') parser.add_argument( '--log', default=['disk'], type=str, choices=['disk', 'tensorboard'], help='Loggers used (disk, tensorboard) (default=%(default)s)', nargs='*', metavar="LOGGER") parser.add_argument('--save-models', action='store_true', help='Save trained models (default=%(default)s)') parser.add_argument('--last-layer-analysis', action='store_true', help='Plot last layer analysis (default=%(default)s)') parser.add_argument( '--no-cudnn-deterministic', action='store_true', help='Disable CUDNN deterministic (default=%(default)s)') # dataset args parser.add_argument('--datasets', default=['cifar100'], type=str, choices=list(dataset_config.keys()), help='Dataset or datasets used (default=%(default)s)', nargs='+', metavar="DATASET") parser.add_argument( '--num-workers', default=4, type=int, required=False, help= 'Number of subprocesses to use for dataloader (default=%(default)s)') parser.add_argument( '--pin-memory', default=False, type=bool, required=False, help= 'Copy Tensors into CUDA pinned memory before returning them (default=%(default)s)' ) parser.add_argument( '--batch-size', default=64, type=int, required=False, help='Number of samples per batch to load (default=%(default)s)') parser.add_argument( '--num-tasks', default=4, type=int, required=False, help='Number of tasks per dataset (default=%(default)s)') parser.add_argument( '--nc-first-task', default=None, type=int, required=False, help='Number of classes of the first task (default=%(default)s)') parser.add_argument( '--use-valid-only', action='store_true', help='Use validation split instead of test (default=%(default)s)') parser.add_argument( '--stop-at-task', default=0, type=int, required=False, help='Stop training after specified task (default=%(default)s)') # model args parser.add_argument('--network', default='resnet32', type=str, choices=allmodels, help='Network architecture used (default=%(default)s)', metavar="NETWORK") parser.add_argument( '--keep-existing-head', action='store_true', help='Disable removing classifier last layer (default=%(default)s)') parser.add_argument('--pretrained', action='store_true', help='Use pretrained backbone (default=%(default)s)') # training args parser.add_argument('--approach', default='finetuning', type=str, choices=approach.__all__, help='Learning approach used (default=%(default)s)', metavar="APPROACH") parser.add_argument( '--nepochs', default=200, type=int, required=False, help='Number of epochs per training session (default=%(default)s)') parser.add_argument('--lr', default=0.1, type=float, required=False, help='Starting learning rate (default=%(default)s)') parser.add_argument('--lr-min', default=1e-4, type=float, required=False, help='Minimum learning rate (default=%(default)s)') parser.add_argument( '--lr-factor', default=3, type=float, required=False, help='Learning rate decreasing factor (default=%(default)s)') parser.add_argument( '--lr-patience', default=5, type=int, required=False, help= 'Maximum patience to wait before decreasing learning rate (default=%(default)s)' ) parser.add_argument('--clipping', default=10000, type=float, required=False, help='Clip gradient norm (default=%(default)s)') parser.add_argument('--momentum', default=0.0, type=float, required=False, help='Momentum factor (default=%(default)s)') parser.add_argument('--weight-decay', default=0.0, type=float, required=False, help='Weight decay (L2 penalty) (default=%(default)s)') parser.add_argument('--warmup-nepochs', default=0, type=int, required=False, help='Number of warm-up epochs (default=%(default)s)') parser.add_argument( '--warmup-lr-factor', default=1.0, type=float, required=False, help='Warm-up learning rate factor (default=%(default)s)') parser.add_argument( '--multi-softmax', action='store_true', help='Apply separate softmax for each task (default=%(default)s)') parser.add_argument( '--fix-bn', action='store_true', help='Fix batch normalization after first task (default=%(default)s)') parser.add_argument( '--eval-on-train', action='store_true', help='Show train loss and accuracy (default=%(default)s)') # gridsearch args parser.add_argument( '--gridsearch-tasks', default=-1, type=int, help= 'Number of tasks to apply GridSearch (-1: all tasks) (default=%(default)s)' ) # Args -- Incremental Learning Framework args, extra_args = parser.parse_known_args(argv) args.results_path = os.path.expanduser(args.results_path) base_kwargs = dict(nepochs=args.nepochs, lr=args.lr, lr_min=args.lr_min, lr_factor=args.lr_factor, lr_patience=args.lr_patience, clipgrad=args.clipping, momentum=args.momentum, wd=args.weight_decay, multi_softmax=args.multi_softmax, wu_nepochs=args.warmup_nepochs, wu_lr_factor=args.warmup_lr_factor, fix_bn=args.fix_bn, eval_on_train=args.eval_on_train) if args.no_cudnn_deterministic: print('WARNING: CUDNN Deterministic will be disabled.') utils.cudnn_deterministic = False utils.seed_everything(seed=args.seed) print('=' * 108) print('Arguments =') for arg in np.sort(list(vars(args).keys())): print('\t' + arg + ':', getattr(args, arg)) print('=' * 108) # Args -- CUDA if torch.cuda.is_available(): torch.cuda.set_device(args.gpu) device = 'cuda' else: print('WARNING: [CUDA unavailable] Using CPU instead!') device = 'cpu' # Multiple gpus # if torch.cuda.device_count() > 1: # self.C = torch.nn.DataParallel(C) # self.C.to(self.device) #################################################################################################################### # Args -- Network from networks.network import LLL_Net if args.network in tvmodels: # torchvision models tvnet = getattr(importlib.import_module(name='torchvision.models'), args.network) if args.network == 'googlenet': init_model = tvnet(pretrained=args.pretrained, aux_logits=False) else: init_model = tvnet(pretrained=args.pretrained) set_tvmodel_head_var(init_model) else: # other models declared in networks package's init net = getattr(importlib.import_module(name='networks'), args.network) # WARNING: fixed to pretrained False for other model (non-torchvision) init_model = net(pretrained=False) # Args -- Continual Learning Approach from approach.incremental_learning import Inc_Learning_Appr Appr = getattr(importlib.import_module(name='approach.' + args.approach), 'Appr') assert issubclass(Appr, Inc_Learning_Appr) appr_args, extra_args = Appr.extra_parser(extra_args) print('Approach arguments =') for arg in np.sort(list(vars(appr_args).keys())): print('\t' + arg + ':', getattr(appr_args, arg)) print('=' * 108) # Args -- Exemplars Management from datasets.exemplars_dataset import ExemplarsDataset Appr_ExemplarsDataset = Appr.exemplars_dataset_class() if Appr_ExemplarsDataset: assert issubclass(Appr_ExemplarsDataset, ExemplarsDataset) appr_exemplars_dataset_args, extra_args = Appr_ExemplarsDataset.extra_parser( extra_args) print('Exemplars dataset arguments =') for arg in np.sort(list(vars(appr_exemplars_dataset_args).keys())): print('\t' + arg + ':', getattr(appr_exemplars_dataset_args, arg)) print('=' * 108) else: appr_exemplars_dataset_args = argparse.Namespace() # Args -- GridSearch if args.gridsearch_tasks > 0: from gridsearch import GridSearch gs_args, extra_args = GridSearch.extra_parser(extra_args) Appr_finetuning = getattr( importlib.import_module(name='approach.finetuning'), 'Appr') assert issubclass(Appr_finetuning, Inc_Learning_Appr) GridSearch_ExemplarsDataset = Appr.exemplars_dataset_class() print('GridSearch arguments =') for arg in np.sort(list(vars(gs_args).keys())): print('\t' + arg + ':', getattr(gs_args, arg)) print('=' * 108) assert len(extra_args) == 0, "Unused args: {}".format(' '.join(extra_args)) #################################################################################################################### # Log all arguments full_exp_name = reduce( (lambda x, y: x[0] + y[0]), args.datasets) if len(args.datasets) > 0 else args.datasets[0] full_exp_name += '_' + args.approach if args.exp_name is not None: full_exp_name += '_' + args.exp_name logger = MultiLogger(args.results_path, full_exp_name, loggers=args.log, save_models=args.save_models) logger.log_args( argparse.Namespace(**args.__dict__, **appr_args.__dict__, **appr_exemplars_dataset_args.__dict__)) # Loaders utils.seed_everything(seed=args.seed) trn_loader, val_loader, tst_loader, taskcla = get_loaders( args.datasets, args.num_tasks, args.nc_first_task, args.batch_size, num_workers=args.num_workers, pin_memory=args.pin_memory) # Apply arguments for loaders if args.use_valid_only: tst_loader = val_loader max_task = len(taskcla) if args.stop_at_task == 0 else args.stop_at_task # Network and Approach instances utils.seed_everything(seed=args.seed) net = LLL_Net(init_model, remove_existing_head=not args.keep_existing_head) utils.seed_everything(seed=args.seed) # taking transformations and class indices from first train dataset first_train_ds = trn_loader[0].dataset transform, class_indices = first_train_ds.transform, first_train_ds.class_indices appr_kwargs = {**base_kwargs, **dict(logger=logger, **appr_args.__dict__)} if Appr_ExemplarsDataset: appr_kwargs['exemplars_dataset'] = Appr_ExemplarsDataset( transform, class_indices, **appr_exemplars_dataset_args.__dict__) utils.seed_everything(seed=args.seed) appr = Appr(net, device, **appr_kwargs) # GridSearch if args.gridsearch_tasks > 0: ft_kwargs = { **base_kwargs, **dict(logger=logger, exemplars_dataset=GridSearch_ExemplarsDataset( transform, class_indices)) } appr_ft = Appr_finetuning(net, device, **ft_kwargs) gridsearch = GridSearch(appr_ft, args.seed, gs_args.gridsearch_config, gs_args.gridsearch_acc_drop_thr, gs_args.gridsearch_hparam_decay, gs_args.gridsearch_max_num_searches) # Loop tasks print(taskcla) acc_taw = np.zeros((max_task, max_task)) acc_tag = np.zeros((max_task, max_task)) forg_taw = np.zeros((max_task, max_task)) forg_tag = np.zeros((max_task, max_task)) for t, (_, ncla) in enumerate(taskcla): # Early stop tasks if flag if t >= max_task: continue print('*' * 108) print('Task {:2d}'.format(t)) print('*' * 108) # Add head for current task net.add_head(taskcla[t][1]) net.to(device) # GridSearch if t < args.gridsearch_tasks: # Search for best finetuning learning rate -- Maximal Plasticity Search print('LR GridSearch') best_ft_acc, best_ft_lr = gridsearch.search_lr( appr.model, t, trn_loader[t], val_loader[t]) # Apply to approach appr.lr = best_ft_lr gen_params = gridsearch.gs_config.get_params('general') for k, v in gen_params.items(): if not isinstance(v, list): setattr(appr, k, v) # Search for best forgetting/intransigence tradeoff -- Stability Decay print('Trade-off GridSearch') best_tradeoff, tradeoff_name = gridsearch.search_tradeoff( args.approach, appr, t, trn_loader[t], val_loader[t], best_ft_acc) # Apply to approach if tradeoff_name is not None: setattr(appr, tradeoff_name, best_tradeoff) print('-' * 108) # Train appr.train(t, trn_loader[t], val_loader[t]) print('-' * 108) # Test for u in range(t + 1): test_loss, acc_taw[t, u], acc_tag[t, u] = appr.eval(u, tst_loader[u]) if u < t: forg_taw[t, u] = acc_taw[:t, u].max(0) - acc_taw[t, u] forg_tag[t, u] = acc_tag[:t, u].max(0) - acc_tag[t, u] print( '>>> Test on task {:2d} : loss={:.3f} | TAw acc={:5.1f}%, forg={:5.1f}%' '| TAg acc={:5.1f}%, forg={:5.1f}% <<<'.format( u, test_loss, 100 * acc_taw[t, u], 100 * forg_taw[t, u], 100 * acc_tag[t, u], 100 * forg_tag[t, u])) logger.log_scalar(task=t, iter=u, name='loss', group='test', value=test_loss) logger.log_scalar(task=t, iter=u, name='acc_taw', group='test', value=100 * acc_taw[t, u]) logger.log_scalar(task=t, iter=u, name='acc_tag', group='test', value=100 * acc_tag[t, u]) logger.log_scalar(task=t, iter=u, name='forg_taw', group='test', value=100 * forg_taw[t, u]) logger.log_scalar(task=t, iter=u, name='forg_tag', group='test', value=100 * forg_tag[t, u]) # Save print('Save at ' + os.path.join(args.results_path, full_exp_name)) logger.log_result(acc_taw, name="acc_taw", step=t) logger.log_result(acc_tag, name="acc_tag", step=t) logger.log_result(forg_taw, name="forg_taw", step=t) logger.log_result(forg_tag, name="forg_tag", step=t) logger.save_model(net.state_dict(), task=t) logger.log_result(acc_taw.sum(1) / np.tril(np.ones(acc_taw.shape[0])).sum(1), name="avg_accs_taw", step=t) logger.log_result(acc_tag.sum(1) / np.tril(np.ones(acc_tag.shape[0])).sum(1), name="avg_accs_tag", step=t) aux = np.tril( np.repeat([[tdata[1] for tdata in taskcla[:max_task]]], max_task, axis=0)) logger.log_result((acc_taw * aux).sum(1) / aux.sum(1), name="wavg_accs_taw", step=t) logger.log_result((acc_tag * aux).sum(1) / aux.sum(1), name="wavg_accs_tag", step=t) # Last layer analysis if args.last_layer_analysis: weights, biases = last_layer_analysis(net.heads, t, taskcla, y_lim=True) logger.log_figure(name='weights', iter=t, figure=weights) logger.log_figure(name='bias', iter=t, figure=biases) # Output sorted weights and biases weights, biases = last_layer_analysis(net.heads, t, taskcla, y_lim=True, sort_weights=True) logger.log_figure(name='weights', iter=t, figure=weights) logger.log_figure(name='bias', iter=t, figure=biases) # Print Summary utils.print_summary(acc_taw, acc_tag, forg_taw, forg_tag) print('[Elapsed time = {:.1f} h]'.format( (time.time() - tstart) / (60 * 60))) print('Done!') return acc_taw, acc_tag, forg_taw, forg_tag, logger.exp_path
parser.model.set_weights(w) del w else: print('Initiate model', time.strftime("%Y-%m-%d %H:%M:%S")) parser = Parser(params) parser.fit(train_data) parser.model.summary() print('Save model', time.strftime("%Y-%m-%d %H:%M:%S")) joblib.dump(parser, params.model_file) if params.evaluate: print('Predict on train', time.strftime("%Y-%m-%d %H:%M:%S")) pred_train = parser.predict(train_data) print_summary(pred_train, train_data) if params.valid is not None: print('Load valid data', time.strftime("%Y-%m-%d %H:%M:%S")) valid_data = loader.load(params.valid) print('Predict on valid', time.strftime("%Y-%m-%d %H:%M:%S")) pred_valid = parser.predict(valid_data) print_summary(pred_valid, valid_data) elif params.mode == 'multitrain': if params.evaluate and params.valid is not None: print('Load valid data', time.strftime("%Y-%m-%d %H:%M:%S")) valid_data = loader.load(params.valid) if params.continue_training:
plt.xlabel("CLASS", fontsize=16) plt.ylabel("COUNT", fontsize=16) x = np.arange(len(count["CLASS"])) y = np.array(list(count["COUNT"])) for a,b in zip(x,y): plt.text(a, b+0.05, '%.0f' % b, ha='center', va= 'bottom',fontsize=16) plt.show() # Baseline baseline_weights, _, baseline_test_logits = train_nn(X_train, y_train, X_test=X_test, y_test=y_test, epoch=2000, batch_size=1000) baseline_accuracy = (baseline_test_logits.argmax(axis=1)==y_test.argmax(axis=1)).mean() _, baseline_names_logits, _ = train_nn(test_sequences_embed, y_train=None, weights=baseline_weights, epoch=0) test_df['baseline_logits'] = baseline_names_logits[:,1] - baseline_names_logits[:,0] print_summary(test_df, 'baseline', baseline_accuracy) # SenSR_0 expert expert_sens_directions = np.copy(test_sequences_embed) sensr0_expert_weights, _, sensr0_expert_test_logits = train_fair_nn(X_train, y_train, expert_sens_directions, X_test = X_test, y_test=y_test) sensr0_expert_accuracy = (sensr0_expert_test_logits.argmax(axis=1)==y_test.argmax(axis=1)).mean() _, sensr0_expert_names_logits, _ = train_nn(test_sequences_embed, y_train=None, weights=sensr0_expert_weights, epoch=0) test_df['sensr0_expert_logits'] = sensr0_expert_names_logits[:,1] - sensr0_expert_names_logits[:,0] print_summary(test_df, 'sensr0_expert', sensr0_expert_accuracy) # Calculate quantile class_1 = test_df.loc[test_df['result'] == 'Class 1'] class_2 = test_df.loc[test_df['result'] == 'Class 2'] Q1 = class_1['baseline_logits'].quantile(0.25) Q2 = class_1['baseline_logits'].quantile(0.5) Q3 = class_1['baseline_logits'].quantile(0.75)
def train_primary(model_name, df_name, max_len, train_ratio, text_feature, embeddings_version, embeddings_path, config_primary, models_path, models_df): device = utils.find_device() loss_function, labels_num, dropout, epochs_num, threshold, \ hid_dim_lstm, lin_output_dim, lr, batch_size, momentum = \ utils.read_config_primary(config_primary, "train_primary") train_df, val_df = utils.split_train_val(df_name, train_ratio) train_dataloader, val_dataloader = utils.create_dataloaders_train( train_df, val_df, text_feature, embeddings_version, max_len, batch_size) model = get_model(model_name, embeddings_path, hid_dim_lstm, loss_function, labels_num, dropout, lin_output_dim) model.to(device) optimizer = AdamW(model.parameters(), lr=2e-5, eps=1e-8) train_len = len(train_dataloader) val_len = len(val_dataloader) train_accuracy_list, train_loss_list, accuracy_list_val, loss_list_val = [], [], [], [] best_val_acc = 0.0 total_steps = len(train_dataloader) * epochs_num scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0, num_training_steps=total_steps) for epoch in range(epochs_num): utils.print_epochs_progress(epoch, epochs_num) start_train = time.clock() acc_num_train = 0.0 loss_scalar = 0.0 model.train() optimizer.zero_grad() t0 = time.time() predictions_dict_train = dict() for batch_idx, batch in enumerate(train_dataloader): if batch_idx % 10 == 0 and not batch_idx == 0: utils.print_batches_progress(t0, batch_idx, train_dataloader) input_ids = batch[0].to(device, dtype=torch.long) masks = batch[1].to(device, dtype=torch.long) labels = batch[2].to(device, dtype=torch.long) hadm_id = batch[3] model.zero_grad() loss, predictions, probabilities = model(input_ids, masks, labels) loss_scalar += loss.item() loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) acc_num_train += utils.add_accuracy(predictions, labels) optimizer.step() scheduler.step() predictions_dict_train = utils.update_predictions_dict( predictions_dict_train, hadm_id, predictions, probabilities, labels) loss_scalar /= train_len end_train = time.clock() total_time = end_train - start_train utils.print_train_epoch(epoch, acc_num_train, train_len, loss_scalar, total_time) train_loss_list.append(round(float(loss_scalar), 3)) train_accuracy_list.append(round(float(acc_num_train / train_len), 3)) utils.print_train_epoch_end(t0) val_acc, val_loss, predictions_dict_val = evaluate_val( model, val_dataloader, device, val_len) accuracy_list_val.append(val_acc) loss_list_val.append(val_loss) if val_acc > best_val_acc: torch.save(model.state_dict(), models_path + 'primary.pkl') best_val_acc = val_acc utils.save_predictions_to_df(predictions_dict_train, models_path, 'train', 'primary') utils.save_predictions_to_df(predictions_dict_val, models_path, 'validation', 'primary') utils.save_model(models_df, 'primary', hid_dim_lstm, labels_num, loss_function, dropout, lin_output_dim, lr, epochs_num, batch_size, momentum, best_val_acc) utils.print_summary(models_path, 'primary', accuracy_list_val) test_acc, predictions_dict_test = evaluate_test(model, device, embeddings_version, max_len, text_feature, batch_size) utils.save_predictions_to_df(predictions_dict_test, models_path, 'test', 'primary') utils.print_test_results(models_path, 'primary', test_acc) return models_df
def run_HAC(FLAGS, env, agent): experiment = Experiment(api_key="M03EcOc9o9kiG95hws4mq1uqI", project_name="HAC", workspace="antonwiehe") # Print task summary print_summary(FLAGS, env) # Determine training mode. If not testing and not solely training, interleave training and testing to track progress mix_train_test = False if not FLAGS.test and not FLAGS.train_only: mix_train_test = True for batch in range(NUM_BATCH): num_episodes = agent.other_params["num_exploration_episodes"] # Evaluate policy every TEST_FREQ batches if interleaving training and testing if mix_train_test and batch % TEST_FREQ == 0: print("\n--- TESTING ---") agent.FLAGS.test = True num_episodes = num_test_episodes # Reset successful episode counter successful_episodes = 0 for episode in range(num_episodes): print("\nBatch %d, Episode %d" % (batch, episode)) # Train for an episode success = agent.train(env, episode) if success: print("Batch %d, Episode %d End Goal Achieved\n" % (batch, episode)) # Increment successful episode counter if applicable if mix_train_test and batch % TEST_FREQ == 0: successful_episodes += 1 # Save agent agent.save_model(episode) # Finish evaluating policy if tested prior batch if mix_train_test and batch % TEST_FREQ == 0: # Log performance success_rate = successful_episodes / num_test_episodes * 100 print("\nTesting Success Rate %.2f%%" % success_rate) agent.log_performance(success_rate) agent.FLAGS.test = False experiment.set_step(batch) experiment.log_metric("Success rate", success_rate) success_list.append(success_rate) with open("successRates.csv", 'w', newline='') as myfile: wr = csv.writer(myfile, quoting=csv.QUOTE_ALL) wr.writerow(success_list) if success_rate > 95: print("Success rate over 95\%!") break print("\n--- END TESTING ---\n")