def train_eval(): """ train and eval the model """ global trainloader global testloader global net (x_train, y_train) = trainloader (x_test, y_test) = testloader # train procedure net.fit( x=x_train, y=y_train, batch_size=args.batch_size, validation_data=(x_test, y_test), epochs=args.epochs, shuffle=True, callbacks=[ SendMetrics(), EarlyStopping(min_delta=0.001, patience=10), TensorBoard(log_dir=TENSORBOARD_DIR), ], ) # trial report final acc to tuner _, acc = net.evaluate(x_test, y_test) logger.debug("Final result is: %d", acc) nni.report_final_result(acc)
def train(args, params): ''' Train model ''' model = build_model(params) X_train, Y1_train, Y2_train, X_test, Y1_test, Y2_test = load_dataset( args.dataset_name, args.num_train) print('Fitting model...') results = model.fit( X_train, [Y1_train, Y2_train], epochs=args.epochs, verbose=1, validation_data=(X_test, [Y1_test, Y2_test]), callbacks=[SendMetrics(), TensorBoard(log_dir=TENSORBOARD_DIR)]) _, _, _, cat_acc, subcat_acc = model.evaluate(X_test, [Y1_test, Y2_test], verbose=0) LOG.debug('Final result is: %d', subcat_acc) nni.report_final_result(subcat_acc) print('Final result is: %d', subcat_acc) model_id = nni.get_sequence_id() # serialize model to JSON model_json = model.to_json() with open("model-{}.json".format(model_id), "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights("model-{}.h5".format(model_id)) print("Saved model to disk")
def run(kf, data, model, label='flag_y'): """ Train model, predict on test set and get model performance. :param kf: :param data: :param model: :param label: :return: """ defaults, gini_trains, gini_tests, auc_trains, auc_tests, ks_trains, ks_tests, psis, models = [], [], [], [], [], \ [], [], [], [] # 交叉验证 kf_list = list(kf.split(data)) for i, index in enumerate(kf_list): # 训练 X = data.drop(columns=label) y = data[label] train_index, test_index = index[0], index[1] X_train, y_train = X.iloc[train_index], y.iloc[train_index] X_test, y_test = X.iloc[test_index], y.iloc[test_index] model.fit(X_train, y_train) y_pred = model.predict_proba(X_train)[:, 1] y_test_pred = model.predict_proba(X_test)[:, 1] # 计算指标 ks_train, ks_test = calc_ks(y_pred, y_train), calc_ks(y_test_pred, y_test) auc_train, auc_test = calc_auc(y_pred, y_train), calc_auc(y_test_pred, y_test) gini_train, gini_test = calc_gini(y_pred, y_train), calc_gini(y_test_pred, y_test) psi = calc_psi(y_pred, y_test_pred) default = 1.8 * ks_test - 0.8 * abs(ks_train - ks_test) defaults.append(default) gini_trains.append(gini_train) gini_tests.append(gini_test) auc_trains.append(auc_train) auc_tests.append(auc_test) ks_trains.append(ks_train) ks_tests.append(ks_test) psis.append(psi) models.append(model) # 整合结果 metrics = { 'gini_train': float(np.mean(gini_trains)), 'gini_test': float(np.mean(gini_tests)), 'auc_train': float(np.mean(auc_trains)), 'auc_test': float(np.mean(auc_tests)), 'ks_train': float(np.mean(ks_trains)), 'ks_test': float(np.mean(ks_tests)), 'psi': float(np.mean(psis)), 'default': float(np.mean(defaults)) } # 输出每套超参数最优模型 best_model_idx = np.argmax(defaults) dump_pkl(models[best_model_idx]) # 生成训练集测试集 train = data.iloc[kf_list[best_model_idx][0]] write_data(train, 'train.fea') test = data.iloc[kf_list[best_model_idx][1]] write_data(test, 'test.fea') LOG.debug(metrics) nni.report_final_result(metrics)
def run(X_train, X_test, y_train, y_test, model): '''Train model and predict result''' model.fit(X_train, y_train) testProba = model.predict_proba(X_test) score = roc_auc_score(y_test, testProba) LOG.debug('ROC-AUC score: %s' % score) nni.report_final_result(score)
def svd_training(params): """ Train Surprise SVD using the given hyper-parameters """ logger.debug("Start training...") train_data = pd.read_pickle(path=os.path.join(params['datastore'], params['train_datapath'])) validation_data = pd.read_pickle(path=os.path.join(params['datastore'], params['validation_datapath'])) svd_params = {p: params[p] for p in ['random_state', 'n_epochs', 'verbose', 'biased', 'n_factors', 'init_mean', 'init_std_dev', 'lr_all', 'reg_all', 'lr_bu', 'lr_bi', 'lr_pu', 'lr_qi', 'reg_bu', 'reg_bi', 'reg_pu', 'reg_qi']} svd = surprise.SVD(**svd_params) train_set = surprise.Dataset.load_from_df(train_data, reader=surprise.Reader(params['surprise_reader'])) \ .build_full_trainset() svd.fit(train_set) logger.debug("Evaluating...") metrics_dict = {} rating_metrics = params['rating_metrics'] if len(rating_metrics) > 0: predictions = compute_rating_predictions(svd, validation_data, usercol=params['usercol'], itemcol=params['itemcol']) for metric in rating_metrics: result = getattr(evaluation, metric)(validation_data, predictions) logger.debug("%s = %g", metric, result) if metric == params['primary_metric']: metrics_dict['default'] = result else: metrics_dict[metric] = result ranking_metrics = params['ranking_metrics'] if len(ranking_metrics) > 0: all_predictions = compute_ranking_predictions(svd, train_data, usercol=params['usercol'], itemcol=params['itemcol'], recommend_seen=params['recommend_seen']) k = params['k'] for metric in ranking_metrics: result = getattr(evaluation, metric)(validation_data, all_predictions, col_prediction='prediction', k=k) logger.debug("%s@%d = %g", metric, k, result) if metric == params['primary_metric']: metrics_dict['default'] = result else: metrics_dict[metric] = result if len(ranking_metrics) == 0 and len(rating_metrics) == 0: raise ValueError("No metrics were specified.") # Report the metrics nni.report_final_result(metrics_dict) # Save the metrics in a JSON file output_dir = os.environ.get('NNI_OUTPUT_DIR') with open(os.path.join(output_dir, 'metrics.json'), 'w') as fp: temp_dict = metrics_dict.copy() temp_dict[params['primary_metric']] = temp_dict.pop('default') json.dump(temp_dict, fp) return svd
def test(final=False): model.eval() z = model(data.x, data.edge_index) evaluator = MulticlassEvaluator() if args.dataset == 'WikiCS': accs = [] for i in range(20): acc = log_regression(z, dataset, evaluator, split=f'wikics:{i}', num_epochs=800)['acc'] accs.append(acc) acc = sum(accs) / len(accs) else: acc = log_regression(z, dataset, evaluator, split='rand:0.1', num_epochs=3000, preload_split=split)['acc'] if final and use_nni: nni.report_final_result(acc) elif use_nni: nni.report_intermediate_result(acc) return acc
def run(X_train, X_test, y_train, y_test, model): """ Train model, predict on test set and get model performance. :param X_train: train data :param X_test: :param y_train: train label :param y_test: test label :param model: specific model :return: report final result to nni """ # 训练 model.fit(X_train, y_train) y_pred = model.predict_proba(X_train)[:, 1] y_test_pred = model.predict_proba(X_test)[:, 1] # 计算指标 ks_train, ks_test = calc_ks(y_pred, y_train), calc_ks(y_test_pred, y_test) auc_train, auc_test = calc_auc(y_pred, y_train), calc_auc(y_test_pred, y_test) gini_train, gini_test = calc_gini(y_pred, y_train), calc_gini(y_test_pred, y_test) psi = calc_psi(y_pred, y_test_pred) # 整合结果 metrics = { 'gini_train': gini_train, 'gini_test': gini_test, 'auc_train': auc_train, 'auc_test': auc_test, 'ks_train': ks_train, 'ks_test': ks_test, 'psi': psi, 'default': 1.8 * ks_test - 0.8 * abs(ks_train - ks_test) } dump_pkl(model) LOG.debug(metrics) nni.report_final_result(metrics)
def main(params): """ Main program: - Build network - Prepare dataset - Train the model - Report accuracy to tuner """ model = MnistModel(conv_size=params['conv_size'], hidden_size=params['hidden_size'], dropout_rate=params['dropout_rate']) optimizer = Adam(learning_rate=params['learning_rate']) model.compile(optimizer=optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) _logger.info('Model built') (x_train, y_train), (x_test, y_test) = load_dataset() _logger.info('Dataset loaded') model.fit(x_train, y_train, batch_size=params['batch_size'], epochs=10, verbose=0, callbacks=[ReportIntermediates()], validation_data=(x_test, y_test)) _logger.info('Training completed') loss, accuracy = model.evaluate(x_test, y_test, verbose=0) nni.report_final_result( accuracy) # send final accuracy to NNI tuner and web UI _logger.info('Final accuracy reported: %s', accuracy)
def train(params): ''' Train model ''' x_train, y_train, x_test, y_test = load_mnist_data() model = create_mnist_model(params) epochs = 10 model.fit(x_train, y_train, batch_size=params['batch_size'], epochs=epochs, verbose=1, validation_data=(x_test, y_test), callbacks=[SendMetrics()]) _, acc = model.evaluate(x_test, y_test, verbose=0) logger.debug('Final result is: %d', acc) nni.report_final_result(acc) model_id = nni.get_sequence_id() model_json = model.to_json() with open('./ckpt/model-{}.json'.format(model_id), 'w') as json_file: json_file.write(model_json) model.save_weights('./ckpt/model-{}.h5'.format(model_id))
def run(x_train_res, y_train_res, x_train, y_train, params, model_num): bagging_predict_result = [] data_index = permutation(len(y_train)) if model_num == 1: model = CatBoostClassifier(learning_rate=params["learning_rate"], l2_leaf_reg=params["l2_leaf_reg"], depth=params["depth"], iterations=params["iterations"], border_count=params["border_count"], verbose=False) model.fit(X=x_train_res.iloc[data_index], y=y_train_res.iloc[data_index]) elif model_num == 2: model = XGBClassifier(use_label_encoder=False, n_estimators=params["n_estimators"], learning_rate=params["learning_rate"], min_child_weight=params["min_child_weight"], reg_lambda=params["reg_lambda"], gamma=params["gamma"], depth=params["depth"], verbosity=0) model.fit(x_train_res.iloc[data_index], y_train_res.iloc[data_index]) else: model = LGBMClassifier(n_estimators=params['n_estimators'], learning_rate=params['learning_rate'],subsample=params['subsample'],\ min_child_weight=params['min_child_weight'], reg_lambda =params['reg_lambda'], silent=True) model.fit(x_train_res.iloc[data_index], y_train_res.iloc[data_index]) y_pred = model.predict_proba(x_train) bagging_predict_result.append(y_pred) lloss = log_loss(y_train, y_pred) print( f'log loss:{lloss}\naccuracy: {accuracy_score(y_train, model.predict(x_train))}' ) nni.report_final_result(lloss)
def evaluate_mlp(agent, env, max_steps, use_nni=False, report_avg=None, eval_repeat=1): print("Evaluating agent over {} episodes".format(eval_repeat)) evaluation_returns = [] for _ in range(eval_repeat): state = env.reset() episode_reward = 0. for _ in range(max_steps): with torch.no_grad(): action, _, _, _ = agent.act(state, True) next_state, reward, done, _ = env.step(action) state = next_state episode_reward += reward if done: # currently all situations end with a done break evaluation_returns.append(episode_reward) eval_avg = sum(evaluation_returns) / len(evaluation_returns) print("Ave. evaluation return =", eval_avg) if use_nni: if eval_repeat == 1: nni.report_intermediate_result(eval_avg) elif eval_repeat > 1 and report_avg is not None: metric = (report_avg + eval_avg) / 2 nni.report_final_result(metric) return eval_avg
def execute_runner(runners, is_nni=False): train_losses = [] train_accuracies = [] test_intermediate_results = [] test_losses = [] test_accuracies = [] for idx_r, runner in enumerate(runners): rs = runner.run(verbose=2) train_losses.append(rs[0]) train_accuracies.append(rs[1]) test_intermediate_results.append(rs[2]) test_losses.append(rs[3]["loss"]) test_accuracies.append(rs[3]["acc"]) '''if idx_r == 0: plot_graphs(rs)''' if is_nni: mean_intermediate_res = np.mean(test_intermediate_results, axis=0) for i in mean_intermediate_res: nni.report_intermediate_result(i) nni.report_final_result(np.mean(test_accuracies)) runners[-1].logger.info("*" * 15 + "Final accuracy train: %3.4f" % np.mean(train_accuracies)) runners[-1].logger.info("*" * 15 + "Std accuracy train: %3.4f" % np.std(train_accuracies)) runners[-1].logger.info("*" * 15 + "Final accuracy test: %3.4f" % np.mean(test_accuracies)) runners[-1].logger.info("*" * 15 + "Std accuracy train: %3.4f" % np.std(train_accuracies)) runners[-1].logger.info("Finished") return
def main(): data_dir = '/tmp/tensorflow/mnist/input_data' mnist = input_data.read_data_sets(data_dir, one_hot=True) logger.debug('Mnist download data down.') mnist_network = MnistNetwork() mnist_network.build_network() logger.debug('Mnist build network done.') graph_location = tempfile.mkdtemp() logger.debug('Saving graph to: %s' % graph_location) train_writer = tf.summary.FileWriter(graph_location) train_writer.add_graph(tf.get_default_graph()) test_acc = 0.0 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) batch_num = 200 for i in range(batch_num): batch_size = nni.choice(50, 250, 500, name='batch_size') batch = mnist.train.next_batch(batch_size) dropout_rate = nni.choice(1, 5, name='dropout_rate') mnist_network.train_step.run(feed_dict={mnist_network.x: batch[ 0], mnist_network.y: batch[1], mnist_network.keep_prob: dropout_rate}) if i % 100 == 0: test_acc = mnist_network.accuracy.eval(feed_dict={ mnist_network.x: mnist.test.images, mnist_network.y: mnist.test.labels, mnist_network.keep_prob: 1.0}) nni.report_intermediate_result(test_acc) test_acc = mnist_network.accuracy.eval(feed_dict={mnist_network.x: mnist.test.images, mnist_network.y: mnist.test.labels, mnist_network.keep_prob: 1.0}) nni.report_final_result(test_acc)
def execute_runner(runners, is_nni=False): res = [runner.run(verbose=2) for runner in runners] all_final_results = [r[1] for r in res] if is_nni: # NNI reporting. now reporting -losses, trying to maximize this. It can also be done for AUCs. final_loss = np.mean([all_final_results[it]["all_loss_test"] for it in range(len(all_final_results))]) nni.report_final_result(np.exp(-final_loss)) # Reporting results to loggers aggr_final_results = {} for new_name, old_name in zip( ["auc_train", "loss_train", "auc_eval", "loss_eval", "auc_test", "loss_test", "epochs_done"], ["auc_train", "all_loss_train", "auc_eval", "all_loss_eval", "auc_test", "all_loss_test", "epochs_done"]): aggr_final_results[new_name] = [d[old_name] for d in all_final_results] runners[-1].logger.info("\nAggregated final results:") for name, vals in aggr_final_results.items(): runners[-1].logger.info("*" * 15 + f"mean {name}: {np.mean(vals):.4f}") runners[-1].logger.info("*" * 15 + f"std {name}: {np.std(vals):.4f}") runners[-1].logger.info("Finished") # If the NNI doesn't run, only the mean results dictionary will be built. No special plots. all_results = { "all_final_output_labels_train": [d["training_output_labels"] for d in all_final_results], "all_final_output_labels_eval": [d["eval_output_labels"] for d in all_final_results], "all_final_output_labels_test": [d["test_output_labels"] for d in all_final_results], "final_auc_train": np.mean([d["auc_train"] for d in all_final_results]), "final_loss_train": np.mean([d["all_loss_train"] for d in all_final_results]), "final_auc_eval": np.mean([d["auc_eval"] for d in all_final_results]), "final_loss_eval": np.mean([d["all_loss_eval"] for d in all_final_results]), "final_auc_test": np.mean([d["auc_test"] for d in all_final_results]), "final_loss_test": np.mean([d["all_loss_test"] for d in all_final_results]), "average_epochs_done": np.mean([d["epochs_done"] for d in all_final_results]) } return all_results
def run_trial(parameters): print("Starting Trial") print(parameters) model = Model(parameters) model.load_data() acc = [] for _ in range(parameters['trials']): # try: model.build_architecture() model.train() acc.append(model.test()) # except: # model.set_device("cpu") # # model.build_topological_edges() # model.build_architecture() # model.train() # acc.append(model.test()) avg_acc = np.mean(acc) Logger.info("average test acc: {:.3f}% \n std is: {}".format(avg_acc * 100, np.std(acc) * 100)) # output for nni - auto ml if parameters['is_nni']: avg_acc = max(avg_acc, MinValForNNI) nni.report_final_result(avg_acc) return
def evaluate_model(model_cls): # "model_cls" is a class, need to instantiate model = model_cls() device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') model.to(device) optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) transf = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) train_loader = DataLoader(MNIST('data/mnist', download=True, transform=transf), batch_size=64, shuffle=True) test_loader = DataLoader(MNIST('data/mnist', download=True, train=False, transform=transf), batch_size=64) for epoch in range(3): # train the model for one epoch train_epoch(model, device, train_loader, optimizer, epoch) # test the model for one epoch accuracy = test_epoch(model, device, test_loader) # call report intermediate result. Result can be float or dict nni.report_intermediate_result(accuracy) # report final test result nni.report_final_result(accuracy)
def fit(self, train_loader, validation_loader): for e in range(self.config.n_epochs): if self.config.verbose: lr = self.optimizer.param_groups[0]['lr'] timestamp = datetime.datetime.now().utcnow().isoformat() self.log(f'\n{timestamp}\nLR: {lr}') t = time.time() summary_loss = self.train_one_epoch(train_loader) self.log( f'[RESULT]: Train. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}') self.save(f'{self.base_dir}/last-checkpoint.bin') t = time.time() summary_loss = self.validation(validation_loader) self.log( f'[RESULT]: Val. Epoch: {self.epoch}, summary_loss: {summary_loss.avg:.5f}, time: {(time.time() - t):.5f}') nni.report_intermediate_result(summary_loss.avg) # logger.debug(summary_loss.avg) if summary_loss.avg < self.best_summary_loss: self.best_summary_loss = summary_loss.avg self.model.eval() self.save(f'{self.base_dir}/best-checkpoint-{str(self.epoch).zfill(3)}epoch.bin') # for path in sorted(glob(f'{self.base_dir}/best-checkpoint-*epoch.bin'))[:-3]: # os.remove(path) if self.config.validation_scheduler: self.scheduler.step(metrics=summary_loss.avg) self.epoch += 1 nni.report_final_result(summary_loss.avg)
def __call__(self): while self.cur_epoch < self.hp.max_epoch: try: if self.optimizer_config.lr_update: utils.adjust_learning_rate(self.optimizer, self.cur_epoch, self.hp.max_epoch, self.optimizer_config.lr) else: utils.set_learning_rate(self.optimizer, self.hp.lr, False) _ = self.train_epoch_dataset_first(self.cur_epoch, 'trn') except KeyboardInterrupt: self.handle_exception() print('Exit control menu.') # nni.report_intermediate_result(0.5) # if self.cur_epoch % self.config.val_interval == 0: try: ret_val, val_loss, val_eval = self.train_epoch_dataset_first( self.cur_epoch, 'val') ret_tst, tst_loss, tst_eval = self.train_epoch_dataset_first( self.cur_epoch, 'tst') if self.hp.evaluation == 'loss': self.undec = self.recoder.push_loss( self.cur_epoch, self.undec, val_loss, ret_tst) if self.hp.nni: nni.report_intermediate_result(tst_loss) elif self.hp.evaluation == 'acc': self.undec = self.recoder.push_eval( self.cur_epoch, self.undec, val_eval, ret_tst) if self.hp.nni: nni.report_intermediate_result(tst_eval) else: raise ValueError('Unknown evaluation.') if self.undec == 0: self.save_checkpoint() except KeyboardInterrupt: print(f'Skipping val and test for ctrl + c detected.') self.cur_epoch += 1 if self.undec >= self.hp.stop_val_dec: print( 'Val_loss hasn\'t decrease in the last [{}] epoches, stop training early.' .format(self.hp.stop_val_dec)) break if self.hp.evaluation == 'loss': fin_epoch, fin_loss = self.recoder.pop_via_loss() if self.hp.nni: nni.report_final_result(fin_loss) print( f'[{self.cur_epoch}] epoches complete, output results = {fin_loss} at epoch [{fin_epoch}], seed = {self.hp.seed}.' ) elif self.hp.evaluation == 'acc': fin_epoch, fin_eval = self.recoder.pop_via_eval() if self.hp.nni: nni.report_final_result(fin_eval) print( f'[{self.cur_epoch}] epoches complete, output results = {fin_eval} at epoch [{fin_epoch}], seed = {self.hp.seed}.' ) else: raise ValueError('Unknown evaluation.') self.evaluate()
def run(x_train, y_train, xtest, y_test, model): eval_set = [(x_test, y_test)] model.fit(x_train, y_train, early_stopping_rounds=10, eval_metric='auc', eval_set=eval_set, verbose=True) y_pred = model.predict_proba(x_test, ntree_limit=model.best_ntree_limit)[:, 1] auc = roc_auc_score(y_test, y_pred) score = roc_auc_score( y_train, model.predict_proba(x_train, ntree_limit=model.best_ntree_limit)[:, 1]) trail_id = get_experiment_id LOG.info('trail id :{}\ttrain auc:{}\ttest auc{}' % (trail_id, score, auc)) save_dir = args.save_dir model_best = glob.glob('{}/*.model'.format(save_folder)) if len(model_best) != 0: model_best = model_best[0] model_best_auc = float(model_best.strip('.model').split('_')[-1]) if auc > model_best_auc: model.save_model('{}/xgb_{}.model'.format(save_folder, auc)) os.system('rm {}'.format(model_best)) else: model.save_model('{}/xgb_{}.model'.format(save_folder, auc)) nni.report_final_result(auc)
def train(args, params): ''' Train model ''' data=data_read.yushan_data() train_x,train_y,test_x,test_y=data.get_train_data(tr_path='../data/yushan/yushan_tr_index.npy', tt_path='../data/yushan/yushan_tt_index.npy', data_type='3D') # data_aug_generator=data_read.data_aug(train_x,train_y) # rot_x,rot_y=data_aug_generator.rotate(train_x,train_y) # flip_x,flip_y=data_aug_generator.flip(train_x,train_y) # noise_x,noise_y=data_aug_generator.add_noise(train_x,train_y,sigma=0.5) # aug_data=get_mixup_data(train_x,train_y,[flip_x,flip_y],[rot_x,rot_y],[noise_x,noise_y],params) # train_x,train_y=aug_data train_y=data_read.label_to_onehot(train_y) test_y=data_read.label_to_onehot(test_y) model = create_model(params) SendMetric=SendMetrics(validation_data=(test_x,test_y)) model.fit(train_x, train_y, batch_size=args.batch_size, epochs=args.epochs, verbose=1, validation_data=(test_x, test_y), callbacks=[SendMetric, TensorBoard(log_dir=TENSORBOARD_DIR)]) y_pred=model.predict_proba(test_x) score = roc_auc_score(test_y[:,1], y_pred[:,1]) # _, acc = model.evaluate(x_test, y_test, verbose=0) LOG.debug('Final result is: %d', score) nni.report_final_result(score)
def main(params): """ Main program: - Build network - Prepare dataset - Train the model - Report accuracy to tuner """ model = dnnmodel(hidden_size=params['hidden_size']) optimizer = Adam(learning_rate=params['learning_rate']) model.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy']) _logger.info('Model built') train_x, train_y, test_x, test_y = load_data() _logger.info('Data loaded') model.fit(train_x, train_y, batch_size=params['batch_size'], epochs=100, verbose=0, callbacks=[ReportIntermediates()], validation_data=(test_x, test_y)) _logger.info('Training completed') loss, accuracy = model.evaluate(test_x, test_y, verbose=0) nni.report_final_result( accuracy) # send final accuracy to NNI tuner and web UI _logger.info('Final accuracy reported: %s', accuracy)
async def query_trial_metrics(self): start_t = getattr(self, "_trial_start_time", None) if start_t is None: logger.info(f"Trial({self.cfg_hash}) is not started!") else: logger.info( f"Trial({self.cfg_hash}) has started {(datetime.now() - start_t).total_seconds()} secs" ) curr_latest_epoch, intermediate_metrics, final_val = self.metrics_reporter.query_metrics( self.latest_epoch) if curr_latest_epoch is None: return if curr_latest_epoch is not None and intermediate_metrics is not None: for metrics in intermediate_metrics: logger.info(f"report_intermediate_result:{metrics}") if os.getenv(ENV_KEY_TRIAL_IN_NNI, None): nni.report_intermediate_result( metrics) # 目前测试阶段,还不能调用 nni 的接口 self.latest_epoch = curr_latest_epoch if final_val is not None and self.final_val is None: # 第一次读取到 final val self.final_val = final_val logger.info(f"report_final_result:{self.final_val}") if os.getenv(ENV_KEY_TRIAL_IN_NNI, None): nni.report_final_result(self.final_val) self._trial_finished_future.set_result(self.final_val)
def main(args, experiment_id, trial_id): use_cuda = not args['no_cuda'] and torch.cuda.is_available() torch.set_num_threads(4) torch.manual_seed(args['seed']) device = torch.device("cuda" if use_cuda else "cpu") batch_size = args['batch_size'] hidden_size = args['hidden_size'] train_loader, test_loader = data_loader(batch_size) model = Net(hidden_size=hidden_size).to(device) optimizer = optim.SGD(model.parameters(), lr=args['lr'], momentum=args['momentum']) for epoch in range(1, args['epochs'] + 1): train(args, model, device, train_loader, optimizer, epoch) test_acc = test(args, model, device, test_loader) # report intermediate result nni.report_intermediate_result(test_acc) logger.debug('test accuracy %g', test_acc) logger.debug('Pipe send intermediate result done.') torch.save( model.state_dict(), f'{os.path.join(os.getcwd())}/model_outputs/{experiment_id}-{trial_id}-model.pth' ) test_acc = test(args, model, device, test_loader) # report final result nni.report_final_result(test_acc) logger.debug('Final result is %g', test_acc) output_logger.info(f'{experiment_id}|{trial_id}|{params}|{test_acc:0.6f}') logger.debug('Send final result done.')
def run(params): """ Distributed Synchronous SGD Example """ rank = dist.get_rank() torch.manual_seed(1234) train_set, bsz = partition_dataset() model = Net() model = model optimizer = optim.SGD(model.parameters(), lr=params['learning_rate'], momentum=params['momentum']) num_batches = ceil(len(train_set.dataset) / float(bsz)) total_loss = 0.0 for epoch in range(3): epoch_loss = 0.0 for data, target in train_set: data, target = Variable(data), Variable(target) optimizer.zero_grad() output = model(data) loss = F.nll_loss(output, target) epoch_loss += loss.item() loss.backward() average_gradients(model) optimizer.step() #logger.debug('Rank: ', rank, ', epoch: ', epoch, ': ', epoch_loss / num_batches) if rank == 0: nni.report_intermediate_result(epoch_loss / num_batches) total_loss += (epoch_loss / num_batches) total_loss /= 3 logger.debug('Final loss: {}'.format(total_loss)) if rank == 0: nni.report_final_result(total_loss)
def main_nni(X, y, title, folder, result_type): # params = nni.get_next_parameter() params = { "hid_dim_0": 100, "hid_dim_1": 10, "reg": 0.68, "dims": [20, 40, 60, 2], "lr": 0.001, "test_size": 0.15, "batch_size": 4, "shuffle": 1, "num_workers": 4, "epochs": 150, "optimizer": 'SGD', "loss": 'MSE' } print(params) auc, loss = nn_nni_main(X, y, params, title, folder) if result_type == "loss": nni.report_final_result(loss) if result_type == "auc": nni.report_final_result(auc) else: raise Exception
def train_ml_flow_predictor(pre_model, dataset, test_set_choice, early_stopping_round, **kwargs): train_set = dataset.gen_flow_session(set_choice=0, **kwargs) eval_set = dataset.gen_flow_session(set_choice=1, **kwargs) test_set = dataset.gen_flow_session(set_choice=test_set_choice, **kwargs) def _split_set(input_set): _, seq = zip(*input_set) seq = np.array(seq) history_seq, label = seq[:, :-1], seq[:, -1] return history_seq, label train_history, train_label = _split_set(train_set) eval_history, eval_label = _split_set(eval_set) test_history, test_label = _split_set(test_set) try: pre_model = pre_model.fit(train_history, train_label, eval_set=[(eval_history, eval_label)], early_stopping_rounds=early_stopping_round) except TypeError: pre_model = pre_model.fit(train_history, train_label) pre = pre_model.predict(test_history) score_series = cal_regression_metric(pre, test_label) print(score_series) nni.report_final_result(score_series.loc['mae']) return score_series
def run(X_train, X_test, y_train, y_test, model_): '''Train model and predict result''' y_train_log = np.log1p(y_train) model_.fit(X_train, y_train_log) predict_y_train = model_.predict(X_train) predict_y_train = np.expm1(predict_y_train) predict_y_test = model_.predict(X_test) predict_y_test = np.expm1(predict_y_test) score_train = np.int(mean_absolute_error(y_train, predict_y_train)) r2_train = r2_score(y_train, predict_y_train) LOG.info('mean_absolute_error on train : %s' % score_train) LOG.info('r2 on train : %s' % r2_train) score_test = np.int(mean_absolute_error(y_test, predict_y_test)) r2_test = r2_score(y_test, predict_y_test) LOG.info('mean_absolute_error on test : %s' % score_test) LOG.info('r2 on test : %s' % r2_test) # add penalty if r2 train score is larger then r2 test score if r2_train/r2_test > 1: score_rate = (r2_train/r2_test)**2 else: score_rate = 1 LOG.info('score rate: %s' % score_rate) score = score_test * score_rate LOG.info('corrected mean_absolute_error on test score: %s' % score) nni.report_final_result(score)
def main(params): ''' Main function, build mnist network, run and send result to NNI. ''' # Import data mnist = input_data.read_data_sets(params['data_dir'], one_hot=True) print('Mnist download data down.') logger.debug('Mnist download data down.') # Create the model # Build the graph for the deep net mnist_network = MnistNetwork(channel_1_num=params['channel_1_num'], channel_2_num=params['channel_2_num'], conv_size=params['conv_size'], hidden_size=params['hidden_size'], pool_size=params['pool_size'], learning_rate=params['learning_rate']) mnist_network.build_network() logger.debug('Mnist build network done.') # Write log graph_location = tempfile.mkdtemp() logger.debug('Saving graph to: %s', graph_location) train_writer = tf.summary.FileWriter(graph_location) train_writer.add_graph(tf.get_default_graph()) test_acc = 0.0 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for i in range(params['batch_num']): batch = mnist.train.next_batch(params['batch_size']) mnist_network.train_step.run( feed_dict={ mnist_network.images: batch[0], mnist_network.labels: batch[1], mnist_network.keep_prob: 1 - params['dropout_rate'] }) if i % 100 == 0: test_acc = mnist_network.accuracy.eval( feed_dict={ mnist_network.images: mnist.test.images, mnist_network.labels: mnist.test.labels, mnist_network.keep_prob: 1.0 }) nni.report_intermediate_result(test_acc) logger.debug('test accuracy %g', test_acc) logger.debug('Pipe send intermediate result done.') test_acc = mnist_network.accuracy.eval( feed_dict={ mnist_network.images: mnist.test.images, mnist_network.labels: mnist.test.labels, mnist_network.keep_prob: 1.0 }) nni.report_final_result(test_acc) logger.debug('Final result is %g', test_acc) logger.debug('Send final result done.')
def train(params): num_units = params.get('num_units') dropout_rate = params.get('dropout_rate') lr = params.get('lr') activationOne = params.get('activationOne') activationTwo = params.get('activationTwo') batch_size = params.get('batch_size') test_size = params.get('test_size') dropout_rate = params.get('dropout_rate') vocab_size = params.get('vocab_size') embedding_dim = params.get('embedding_dim') lossF = params.get('lossF') model = EmbeddingNN(test_size, num_units, dropout_rate, lr, activationOne, activationTwo, vocab_size, embedding_dim, lossF) X_train, y_train, X_test, y_test, class_weights, input_shape = load_dataset( test_size) #Model fit model.fit(X_train, y_train, validation_data=(X_test, y_test), class_weight=class_weights, verbose=2, batch_size=int(batch_size)) threshold = true_acc(model, X_test, y_test) y_pred = model.predict(X_test) y_pred = np.array([1 if row > threshold else 0 for row in y_pred]) y_test = np.array(y_test) acc = np.mean(y_pred == y_test) # print(acc) nni.report_final_result(acc)
def run(X_train, X_test, y_train, y_test, PARAMS): '''Train model and predict result''' model.fit(X_train, y_train) predict_y = model.predict(X_test) score = r2_score(y_test, predict_y) LOG.debug('r2 score: %s' % score) nni.report_final_result(score)
def main(): _params = { "data_name": "ia-retweet-pol", "graphs_cutoff_number": 2, "net": GCNNet, "l1_lambda": 0, "epochs": 500, "gcn_dropout_rate": 0.7, "lstm_dropout_rate": 0, "gcn_hidden_sizes": [10, 10, 10, 10, 10, 10, 10, 10, 10], "learning_rate": 0.001, "weight_decay": 0, "gcn_latent_dim": 5, "lstm_hidden_size": 10, "lstm_num_layers": 1, "learned_label": DEFAULT_LABEL_TO_LEARN, } l1_lambda = [0, 1e-7] epochs = [500] gcn_dropout_rate = [0.3, 0.5] gcn_hidden_sizes = [[100, 100], [200, 200]] learning_rate = [1e-3, 1e-2, 3e-2] weight_decay = [5e-2, 1e-2] gcn_latent_dim = [50, 100] lstm_hidden_size = [50, 100] results = [] argparser = argparse.ArgumentParser() argparser.add_argument("--nni", action='store_true') args = argparser.parse_args() NNI = args.nni if NNI: p = nni.get_next_parameter() p["gcn_hidden_sizes"] = ast.literal_eval(p["gcn_hidden_sizes"]) _params.update(p) ( model_loss, model_accuracy, model_tot_accuracy, zero_model_tot_loss, zero_model_tot_accuracy, first_order_tot_loss, first_order_tot_accuracy, zero_model_diff_loss, zero_model_diff_accuracy, first_order_diff_loss, first_order_diff_accuracy ) = run_trial(_params) if NNI: nni.report_final_result( model_tot_accuracy[0]-zero_model_tot_accuracy[-1]) else: print(f"Final result: model tot accuracy: {model_tot_accuracy}, zero_model_tot_accuracy: {zero_model_tot_accuracy}, " f"first order tot accuracy: {first_order_tot_accuracy}, zero model diff accuracy: {zero_model_diff_accuracy}, " f"first order diff accuracy: {first_order_diff_accuracy}")