def run(args): # Download DROP_TASE_IO_SSE.tar.gz (trained from eladsegal/tag-based-multi-span-extraction) os.makedirs("downloaded", exist_ok=True) drop_weights_path = "downloaded/DROP_TASE_IO_SSE.tar.gz" if os.path.isfile(drop_weights_path): os.remove(drop_weights_path) wget.download( "https://storage.googleapis.com/ai2i/strategyqa/models/DROP_TASE_IO_SSE.tar.gz", drop_weights_path, ) # Train a QA model on twentyquestions os.environ["DROP_WEIGHTS"] = drop_weights_path sys.argv = ( ["train.py"] + (["-g", args.gpu] if args.gpu else []) + ["-c", "configs/for_STAR/1_twentyquestions.jsonnet"] ) print(" ".join(sys.argv)) twentyquestions_serialization_dir = train_main() # Continue to train the model on BoolQ sys.argv = ( ["train.py"] + (["-g", args.gpu] if args.gpu else []) + ["-c", "configs/for_STAR/2_boolq.jsonnet"] + ["-s", args.serialization_dir] + ["-w", os.path.join(twentyquestions_serialization_dir, "model.tar.gz")] ) print(" ".join(sys.argv)) return train_main()
def main(hyper_parameter): if hyper_parameter['train'] is True: is_train = True else: is_train = False if hyper_parameter['test'] is True: is_test = True else: is_test = False if is_train is False and is_test is False: return # 获取超参 hp = load_parameter(hyper_parameter['pre_parameter'], parameter_id=hyper_parameter['parameter_id']) # 训练模型 if is_train is True: train_main(hp) # 测试模型 if is_test is True: test_main(hp, save=False) # 保存超参 save_parameter(hp['data_name'], "1", hp)
def batch_train(cfgs, sleep_time=0, detector=True): for cfg in tqdm(cfgs): cfg_name = os.path.basename(cfg.work_dir) print('\ncfg: {}'.format(cfg_name)) # train train_params = dict(config=cfg, detector=detector) train_main(**train_params) print('{} train done!'.format(cfg_name)) time.sleep(sleep_time)
def main(args): ## set the gpu if args.gpu: os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu ## check the ckpt path path_create('ckpt') ## check the model path path_create('models') ## check the data path path_create('data') ## check the log path path_create('log') if args.type == 'train': train_main(args) elif args.type == 'eval': eval_main(args)
def TrainAndTest(tr_data, tr_label, te_data, te_label): from train import train_main # load my training function train = [tr_data, tr_label] test = [te_data, te_label] threshold = 0.5 model_num = 9 # model number test.append(threshold) train_main(train, test, model_num, dir) ttt = time.localtime(time.time()) with open(os.path.join(dir, 'time.txt'), 'a+') as f: f.write('finish time: {}m {}d {}h {}m {}s'.format( ttt.tm_mon, ttt.tm_mday, ttt.tm_hour, ttt.tm_min, ttt.tm_sec))
def voc_test(eval_steps): train_main('latest.pth', split='trainaug') eval_main('best_trainaug.pth', eval_steps, 'best') eval_main('final_trainaug.pth', eval_steps, 'final') settings.LR = settings.LR / 10 settings.ITER_MAX = settings.TEST_ITER_MAX settings.ITER_SAVE = settings.TEST_ITER_SAVE settings.ITER_VAL = settings.TEST_ITER_VAL train_main('final_trainaug.pth', split='trainval', val=True, reset_steps=True) eval_steps = [5, 6, 7] eval_main('best.pth', eval_steps, 'trainval_best') eval_main('final.pth', eval_steps, 'trainval_final')
def training_pipeline(raw_input_dir: str, training_data_dir: str, model_dir: str, tune_dir: str, whitelist: dict = {}, blacklist: dict = {}, epochs: int = 20, task_type: str = 'multilabel', with_others: bool = False, gen_data: bool = True, train_model: bool = True, tune_model: bool = True, num_workers: int = 8, batch_size: int = 512, model_type: str = 'textcnn', lr: float = 1e-3): if train_model: if not os.path.exists(model_dir): os.mkdir(model_dir) train_main(training_data_dir, model_dir, task_type=task_type, epochs=epochs, model_type=model_type, num_workers=num_workers, batch_size=batch_size, lr=lr) if tune_model: if not os.path.exists(tune_dir): os.mkdir(tune_dir) if model_type == 'textcnn': rf.tune_model(model_dir, training_data_dir, tune_dir + '/', TextCNN, task_type) elif model_type == 'transformer': rf.tune_model(model_dir, training_data_dir, tune_dir + '/', Transformer, task_type) else: rf.tune_model(model_dir, training_data_dir, tune_dir + '/', LSTM, task_type)
from train import main as train_main if __name__ == "__main__": train_main()
def main(): parser = argparse.ArgumentParser( description='RPL experiments master script') parser.add_argument('--num-folds', type=int, default=0, help="Number of folds used, 0 to disable folds") parser.add_argument('--data-dir', default='data/fmllr') parser.add_argument('--offset-dir', default='data') parser.add_argument('--target-dir', default='data/targets') parser.add_argument( '--ivector-dir', nargs=2, help= "Directory with i-vector files. Expects 2 arguments: train and test i-vector directories" ) parser.add_argument('--utt-list-dir', default='data', help='Directory with utterance lists') parser.add_argument('--recog-dir', default='recog') parser.add_argument('--output-dir', default='results') parser.add_argument('--data', default='data_{}.npy') parser.add_argument('--offsets', default='offsets_{}.npy') parser.add_argument('--targets', default='targets_{}.npy') parser.add_argument('--ivectors', default='ivectors_{}.npy') parser.add_argument('--ft', default='final.feature_transform') parser.add_argument('--output-id', default='tmp') parser.add_argument('--network-spec', default='-n lstm -l 4 -u 1024 --timedelay 5 -d 0.2') parser.add_argument('--batch-size', '-b', type=int, nargs='+', default=[256, 128]) parser.add_argument('--optimizer', '-o', nargs='+', default=['adam', 'momentumsgd']) parser.add_argument('--lr', type=float, nargs='+', default=[1e-2, 1e-3, 1e-4, 1e-5], help='Learning rate') parser.add_argument('--fold-data-dir', help='Directory with fold input data') parser.add_argument('--fold-output-dir', help='Directory with predicted fold output') parser.add_argument('--fold-model-dir', help='Directory with output fold model') parser.add_argument( '--fold-output-dev', default='data_dev.npy', help='Output file with predicted development data values') parser.add_argument( '--fold-data-pattern', default='data_{}.npy', help= 'Filename pattern of each fold data, {1} will be replaced by fold ID') parser.add_argument('--fold-offset-pattern', default='offsets_{}.npy', help='Filename pattern of each fold offset') parser.add_argument('--fold-target-pattern', default='targets_{}.npy', help='Filename pattern of each fold targets') parser.add_argument( '--fold-ivector-pattern', default='ivectors_{}.npy', help= 'Filename pattern of each fold i-vectors file, {} will be replaced by fold ID' ) parser.add_argument('--fold-output-pattern', default='data_{}.npy', help='Filename pattern of each fold network output') parser.add_argument('--fold-network-pattern', default='fold_{}.npz', help='Filename pattern of each fold network') parser.add_argument('--master-dir') parser.add_argument('--rpl-dir') parser.add_argument('--PIP', type=float, default=20) parser.add_argument('--LMW', type=float, default=1) parser.add_argument('--gen-folds', action='store_true') parser.add_argument('--no-train-master', action='store_true') parser.add_argument('--no-train-folds', action='store_true') parser.add_argument('--no-predict', action='store_true') parser.add_argument('--no-train-rpl', action='store_true') parser.add_argument('--no-eval', action='store_true') parser.add_argument('--eval-only-master', action='store_true') parser.add_argument('--no-progress', action='store_true') parser.add_argument('--eval-data', nargs='+', default=['test']) args = parser.parse_args() # set default arguments depending on other arguments if args.fold_data_dir is None: if args.ivector_dir: args.fold_data_dir = "{}/fold_data/{}/{}+{}".format( args.output_dir, args.num_folds, Path(args.data_dir).name, Path(args.ivector_dir[0]).name) else: args.fold_data_dir = "{}/fold_data/{}/{}".format( args.output_dir, args.num_folds, Path(args.data_dir).name) if args.fold_output_dir is None: args.fold_output_dir = "{}/fold_data_out/{}/{}".format( args.output_dir, args.num_folds, args.output_id) if args.fold_model_dir is None: args.fold_model_dir = "{}/models/folds/{}/{}".format( args.output_dir, args.num_folds, args.output_id) if args.master_dir is None: args.master_dir = "{}/models/master/{}/{}".format( args.output_dir, args.num_folds, args.output_id) if args.rpl_dir is None: args.rpl_dir = "{}/models/rpl/{}/{}".format(args.output_dir, args.num_folds, args.output_id) # generate folds if args.num_folds > 0 and args.gen_folds: print("==== Generating folds") cmd = [ "-n", args.num_folds, "--data-dir", args.data_dir, "--offset-dir", args.offset_dir, "--target-dir", args.target_dir, "--fold-data-dir", args.fold_data_dir, "--utt-list-dir", args.utt_list_dir, "--train-list", "train.list", "--data", args.data.format("train"), "--offsets", args.offsets.format("train"), "--targets", args.targets.format("train"), "--fold-data-pattern", args.fold_data_pattern, "--fold-offset-pattern", args.fold_offset_pattern, "--fold-target-pattern", args.fold_target_pattern ] if args.ivector_dir: cmd += [ '--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern", args.fold_ivector_pattern ] generate_folds_main(cmd) else: print("==== Skipping fold data generation") # train master network if not args.no_train_master: print("==== Training master network") cmd = ["--tri", "--noplot", "-b"] cmd += args.batch_size cmd += ["--epoch", 20, "-o"] cmd += args.optimizer cmd += ["--lr"] cmd += args.lr cmd += [ "--data-dir", args.data_dir, "--offset-dir", args.offset_dir, "--target-dir", args.target_dir, "--data", args.data, "--offsets", args.offsets, "--targets", args.targets, "--ivectors", args.ivectors, "--ft", args.ft, "--out", args.master_dir ] cmd += args.network_spec.split() if args.ivector_dir: cmd += ['--ivector-dir', args.ivector_dir[0]] if args.no_progress: cmd += ['--no-progress'] train_main(cmd) else: print("==== Skipping training master network") # train folds if args.num_folds > 0 and not args.no_train_folds: for fold in range(args.num_folds): print("==== Training fold {}".format(fold)) cmd = ["--tri", "--noplot", "-b"] cmd += args.batch_size cmd += ["--epoch", 20, "-o"] cmd += args.optimizer cmd += ["--lr"] cmd += args.lr cmd += [ "--data-dir", args.data_dir, "--offset-dir", args.offset_dir, "--target-dir", args.target_dir, "--data", args.data, "--offsets", args.offsets, "--targets", args.targets, "--ivectors", args.ivectors, "--ft", args.ft, "--train-fold", fold, "--fold-data-dir", args.fold_data_dir, "--fold-model-dir", args.fold_model_dir, "--fold-data-pattern", args.fold_data_pattern, "--fold-offset-pattern", args.fold_offset_pattern, "--fold-target-pattern", args.fold_target_pattern, "--out", "result_fold_tmp" ] cmd += args.network_spec.split() if args.ivector_dir: cmd += [ '--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern", args.fold_ivector_pattern ] if args.no_progress: cmd += ['--no-progress'] train_main(cmd) else: print("==== Skipping training folds") # get fold outputs if args.num_folds > 0 and not args.no_predict: print("==== Predicting training data") cmd = [ "--tri", "--ft", args.ft, "--fold-data-dir", args.fold_data_dir, "--fold-output-dir", args.fold_output_dir, "--fold-model-dir", args.fold_model_dir, "--fold-data-pattern", args.fold_data_pattern, "--fold-offset-pattern", args.fold_offset_pattern, "--fold-network-pattern", args.fold_network_pattern, "--fold-output-pattern", args.fold_output_pattern ] cmd += args.network_spec.split() if args.ivector_dir: cmd += [ '--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern", args.fold_ivector_pattern ] if args.no_progress: cmd += ['--no-progress'] predict_folds_main(cmd) print("==== Predicting development data") cmd = [ "--tri", "--ft", args.ft, "--data-dir", args.data_dir, "--offset-dir", args.offset_dir, "--data", args.data, "--offsets", args.offsets, "--fold-output-dir", args.fold_output_dir, "--fold-model-dir", args.fold_model_dir, "--fold-network-pattern", args.fold_network_pattern, "--fold-output-dev", args.fold_output_dev ] cmd += args.network_spec.split() if args.ivector_dir: cmd += [ '--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern", args.fold_ivector_pattern ] if args.no_progress: cmd += ['--no-progress'] predict_folds_main(cmd) else: print("==== Skipping predicting training and development data") # train RPL layer if args.num_folds > 0 and not args.no_train_rpl: print("==== Training RPL layer") cmd = [ "--train-rpl", "--tri", "-b", 1024, "--epoch", 20, "-o", "adam", "--lr", "1e-3", "--data-dir", args.fold_output_dir, "--target-dir", args.target_dir, "--data", args.fold_output_dev, "--targets", args.targets, "--fold-data-dir", args.fold_data_dir, "--fold-output-dir", args.fold_output_dir, "--fold-output-pattern", args.fold_output_pattern, "--fold-target-pattern", args.fold_target_pattern, "--out", args.rpl_dir ] if args.no_progress: cmd += ['--no-progress'] train_main(cmd) else: print("==== Skipping training RPL layer") # evaluate if not args.no_eval: for eval_data in args.eval_data: print("==== Evaluating {} data".format(eval_data)) for eval_folds in [False, True]: for eval_master in [False, True]: for eval_rpl in [False, True]: if (args.num_folds > 0 or args.eval_only_master) and ( eval_folds or not eval_master or eval_rpl): continue if eval_folds or eval_master: print("==== Evaluating {}folds {}master {}rpl". format("+" if eval_folds else "-", "+" if eval_master else "-", "+" if eval_rpl else "-")) cmd = [ "--tri", "--data-dir", args.data_dir, "--offset-dir", args.offset_dir, "--utt-list-dir", args.utt_list_dir, "--data", args.data, "--offsets", args.offsets, "--ivectors", args.ivectors, "--ft", args.ft, "--recog-dir", args.recog_dir, "--rpl", "--rpl-model", "{}/model".format(args.rpl_dir) if eval_rpl else "-", "--master-network", "{}/model".format(args.master_dir) if eval_master else "-", "--PIP", args.PIP, "--LMW", args.LMW, "--fold-model-dir", args.fold_model_dir, "--fold-network-pattern", args.fold_network_pattern if eval_folds else "-", "--test-or-dev", eval_data ] cmd += args.network_spec.split() if args.ivector_dir: cmd += ['--ivector-dir', args.ivector_dir[1]] if args.no_progress: cmd += ['--no-progress'] evaluate_main(cmd) else: print("==== Skipping evaluation")
def batch_train(cfgs, gpus='0'): for cfg in cfgs: if os.path.exists(cfg): cfg = import_module(cfg) cfg.gpus = gpus train_main(cfg)
out_dir = os.path.join("$HOME/dispatcher", run_id) out_dir = os.path.expandvars(out_dir) os.makedirs(out_dir, exist_ok=True) variant_generator = CustomVariantGenerator() unresolved_spec = {"config": cfgs} for i, variant in enumerate( variant_generator._generate_resolved_specs(1, unresolved_spec)): if args.run != "all" and i not in args.run: continue print("Dispatch job {}".format(variant["experiment_tag"])) cfg = variant["spec"]["config"] cfg["name"] = datetime.now().strftime( "%Y-%m-%d_%H-%M-%S") + variant["experiment_tag"] cfg["machine"] = args.machine cfg["training"]["log_path"] = os.path.join(cfg["training"]["log_path"], experiment_name) + "/" out_file = os.path.join(out_dir, variant["trial_id"] + ".yaml") with open(out_file, 'w') as of: yaml.safe_dump(cfg, of, default_flow_style=False) if not args.dry: try: if args.exp == 211: label_selection.label_selection_main(cfg) else: train.train_main(cfg) except Exception: print(traceback.format_exc()) print("Continue with next experiment.") torch.cuda.empty_cache()
from eval_checkpoints import main as eval_main if __name__ == '__main__': parser = ArgumentParser() parser.add_argument('--job-dir', type=str, required=True) parser.add_argument('--agent-net-size', type=int, default=256) parser.add_argument('--agent-net-conv', type=int, default=5) # parser.add_argument('--agent-net-dropout', type=float, default=0.2) parser.add_argument('--mcts-iter', type=int, default=50) parser.add_argument('--epochs', type=int, default=100) parser.add_argument('--epoch-games', type=int, default=5) parser.add_argument('--benchmark-games', type=int, default=10) parser.add_argument('--batch-size', type=int, default=128) parser.add_argument('--lr', type=float, default=1e-4) parser.add_argument('--lr-decay', type=float, default=1.0) parser.add_argument('--lr-decay-epochs', type=int, default=5) parser.add_argument('--reward-gamma', type=float, default=0.99) parser.add_argument('--num-cpus', type=int, default=cpu_count()) parser.add_argument('--checkpoint-gamma', type=float, default=0.2) parser.add_argument('--checkpoint-last-n', type=int, default=None) parser.add_argument('--contest-to-update', type=bool, default=False) parser.add_argument('--win-rate-threshold', type=float, default=0.6) parser.add_argument('--seed', type=int, default=None) parser.add_argument('--eval', type=bool, default=False) args = parser.parse_args() if args.eval: eval_main(args) else: train_main(args)
current_loss = 0 all_losses = [] start = time.time() patience = args.patience savePath = args.savePath train_path = "../data/dummy/classification_train.csv" test_path = "../data/dummy/classification_test.csv" for ei in range(args.max_epochs): bad_counter = 0 best_loss = -1.0 train_main(args, model, train_path, criterion, optimizer) dayloss, valid_loss = test(args, model, test_path, criterion) print("valid loss : {}".format(valid_loss)) print(dayloss) if valid_loss < best_loss or best_loss < 0: print("find best") bad_counter = 0 torch.save(model, args.saveModel) else: bad_counter += 1 if bad_counter > patience: print('Early Stopping') break # Draw a sample
def main(self): work_dir = dirname(abspath(__file__)) chdir(work_dir) servicemanager.LogInfoMsg('Module directory is %s' % work_dir) train_main(with_gui=False, check_stop=self.check_stop)
def test_train(cfg): cfg.train.n_gpu = 0 train_main(cfg)
f1s = [] for n in range(n_exp): model = RNN(args) print("Setting Model Complete") model.to(args.device) optimizer = "optim." + args.optim optimizer = eval(optimizer)(model.parameters(), lr=args.lr) #optimizer = optim.RMSprop(model.parameters()) #optimizer = optim.Adam(model.parameters()) #oprimizer = optim.SGD(model.parameters()) print("Setting optimizer Complete") train_main(model, args, train_data, valid_data, optimizer) acc, f1 = test(model, args, test_data) accs.append(acc) f1s.append(f1) print("=" * 90) print("results on testset at each trials:") for i in range(n_exp): print("exp {:d} | acc {:.3f} | f1 {:.3f}".format(i + 1, accs[i], f1s[i])) total_acc = sum(accs) / n_exp total_f1 = sum(f1s) / n_exp print("-" * 90)
def voc_val(eval_steps): train_main('latest.pth', split='trainaug') eval_main('best_trainaug.pth', eval_steps, 'best') eval_main('final_trainaug.pth', eval_steps, 'final')