def run(args):
# Download DROP_TASE_IO_SSE.tar.gz (trained from eladsegal/tag-based-multi-span-extraction)
os.makedirs("downloaded", exist_ok=True)
drop_weights_path = "downloaded/DROP_TASE_IO_SSE.tar.gz"
if os.path.isfile(drop_weights_path):
os.remove(drop_weights_path)
wget.download(
"https://storage.googleapis.com/ai2i/strategyqa/models/DROP_TASE_IO_SSE.tar.gz",
drop_weights_path,
)
# Train a QA model on twentyquestions
os.environ["DROP_WEIGHTS"] = drop_weights_path
sys.argv = (
["train.py"]
+ (["-g", args.gpu] if args.gpu else [])
+ ["-c", "configs/for_STAR/1_twentyquestions.jsonnet"]
)
print(" ".join(sys.argv))
twentyquestions_serialization_dir = train_main()
# Continue to train the model on BoolQ
sys.argv = (
["train.py"]
+ (["-g", args.gpu] if args.gpu else [])
+ ["-c", "configs/for_STAR/2_boolq.jsonnet"]
+ ["-s", args.serialization_dir]
+ ["-w", os.path.join(twentyquestions_serialization_dir, "model.tar.gz")]
)
print(" ".join(sys.argv))
return train_main()
def main(hyper_parameter):
if hyper_parameter['train'] is True:
is_train = True
else:
is_train = False
if hyper_parameter['test'] is True:
is_test = True
else:
is_test = False
if is_train is False and is_test is False:
return
# 获取超参
hp = load_parameter(hyper_parameter['pre_parameter'],
parameter_id=hyper_parameter['parameter_id'])
# 训练模型
if is_train is True:
train_main(hp)
# 测试模型
if is_test is True:
test_main(hp, save=False)
# 保存超参
save_parameter(hp['data_name'], "1", hp)
def batch_train(cfgs, sleep_time=0, detector=True):
for cfg in tqdm(cfgs):
cfg_name = os.path.basename(cfg.work_dir)
print('\ncfg: {}'.format(cfg_name))
# train
train_params = dict(config=cfg, detector=detector)
train_main(**train_params)
print('{} train done!'.format(cfg_name))
time.sleep(sleep_time)
def main(args):
## set the gpu
if args.gpu:
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
## check the ckpt path
path_create('ckpt')
## check the model path
path_create('models')
## check the data path
path_create('data')
## check the log path
path_create('log')
if args.type == 'train':
train_main(args)
elif args.type == 'eval':
eval_main(args)
def TrainAndTest(tr_data, tr_label, te_data, te_label):
from train import train_main # load my training function
train = [tr_data, tr_label]
test = [te_data, te_label]
threshold = 0.5
model_num = 9 # model number
test.append(threshold)
train_main(train, test, model_num, dir)
ttt = time.localtime(time.time())
with open(os.path.join(dir, 'time.txt'), 'a+') as f:
f.write('finish time: {}m {}d {}h {}m {}s'.format(
ttt.tm_mon, ttt.tm_mday, ttt.tm_hour, ttt.tm_min, ttt.tm_sec))
def voc_test(eval_steps):
train_main('latest.pth', split='trainaug')
eval_main('best_trainaug.pth', eval_steps, 'best')
eval_main('final_trainaug.pth', eval_steps, 'final')
settings.LR = settings.LR / 10
settings.ITER_MAX = settings.TEST_ITER_MAX
settings.ITER_SAVE = settings.TEST_ITER_SAVE
settings.ITER_VAL = settings.TEST_ITER_VAL
train_main('final_trainaug.pth',
split='trainval',
val=True,
reset_steps=True)
eval_steps = [5, 6, 7]
eval_main('best.pth', eval_steps, 'trainval_best')
eval_main('final.pth', eval_steps, 'trainval_final')
def training_pipeline(raw_input_dir: str,
training_data_dir: str,
model_dir: str,
tune_dir: str,
whitelist: dict = {},
blacklist: dict = {},
epochs: int = 20,
task_type: str = 'multilabel',
with_others: bool = False,
gen_data: bool = True,
train_model: bool = True,
tune_model: bool = True,
num_workers: int = 8,
batch_size: int = 512,
model_type: str = 'textcnn',
lr: float = 1e-3):
if train_model:
if not os.path.exists(model_dir):
os.mkdir(model_dir)
train_main(training_data_dir,
model_dir,
task_type=task_type,
epochs=epochs,
model_type=model_type,
num_workers=num_workers,
batch_size=batch_size,
lr=lr)
if tune_model:
if not os.path.exists(tune_dir):
os.mkdir(tune_dir)
if model_type == 'textcnn':
rf.tune_model(model_dir, training_data_dir, tune_dir + '/',
TextCNN, task_type)
elif model_type == 'transformer':
rf.tune_model(model_dir, training_data_dir, tune_dir + '/',
Transformer, task_type)
else:
rf.tune_model(model_dir, training_data_dir, tune_dir + '/', LSTM,
task_type)
from train import main as train_main
if __name__ == "__main__":
train_main()
def main():
parser = argparse.ArgumentParser(
description='RPL experiments master script')
parser.add_argument('--num-folds',
type=int,
default=0,
help="Number of folds used, 0 to disable folds")
parser.add_argument('--data-dir', default='data/fmllr')
parser.add_argument('--offset-dir', default='data')
parser.add_argument('--target-dir', default='data/targets')
parser.add_argument(
'--ivector-dir',
nargs=2,
help=
"Directory with i-vector files. Expects 2 arguments: train and test i-vector directories"
)
parser.add_argument('--utt-list-dir',
default='data',
help='Directory with utterance lists')
parser.add_argument('--recog-dir', default='recog')
parser.add_argument('--output-dir', default='results')
parser.add_argument('--data', default='data_{}.npy')
parser.add_argument('--offsets', default='offsets_{}.npy')
parser.add_argument('--targets', default='targets_{}.npy')
parser.add_argument('--ivectors', default='ivectors_{}.npy')
parser.add_argument('--ft', default='final.feature_transform')
parser.add_argument('--output-id', default='tmp')
parser.add_argument('--network-spec',
default='-n lstm -l 4 -u 1024 --timedelay 5 -d 0.2')
parser.add_argument('--batch-size',
'-b',
type=int,
nargs='+',
default=[256, 128])
parser.add_argument('--optimizer',
'-o',
nargs='+',
default=['adam', 'momentumsgd'])
parser.add_argument('--lr',
type=float,
nargs='+',
default=[1e-2, 1e-3, 1e-4, 1e-5],
help='Learning rate')
parser.add_argument('--fold-data-dir',
help='Directory with fold input data')
parser.add_argument('--fold-output-dir',
help='Directory with predicted fold output')
parser.add_argument('--fold-model-dir',
help='Directory with output fold model')
parser.add_argument(
'--fold-output-dev',
default='data_dev.npy',
help='Output file with predicted development data values')
parser.add_argument(
'--fold-data-pattern',
default='data_{}.npy',
help=
'Filename pattern of each fold data, {1} will be replaced by fold ID')
parser.add_argument('--fold-offset-pattern',
default='offsets_{}.npy',
help='Filename pattern of each fold offset')
parser.add_argument('--fold-target-pattern',
default='targets_{}.npy',
help='Filename pattern of each fold targets')
parser.add_argument(
'--fold-ivector-pattern',
default='ivectors_{}.npy',
help=
'Filename pattern of each fold i-vectors file, {} will be replaced by fold ID'
)
parser.add_argument('--fold-output-pattern',
default='data_{}.npy',
help='Filename pattern of each fold network output')
parser.add_argument('--fold-network-pattern',
default='fold_{}.npz',
help='Filename pattern of each fold network')
parser.add_argument('--master-dir')
parser.add_argument('--rpl-dir')
parser.add_argument('--PIP', type=float, default=20)
parser.add_argument('--LMW', type=float, default=1)
parser.add_argument('--gen-folds', action='store_true')
parser.add_argument('--no-train-master', action='store_true')
parser.add_argument('--no-train-folds', action='store_true')
parser.add_argument('--no-predict', action='store_true')
parser.add_argument('--no-train-rpl', action='store_true')
parser.add_argument('--no-eval', action='store_true')
parser.add_argument('--eval-only-master', action='store_true')
parser.add_argument('--no-progress', action='store_true')
parser.add_argument('--eval-data', nargs='+', default=['test'])
args = parser.parse_args()
# set default arguments depending on other arguments
if args.fold_data_dir is None:
if args.ivector_dir:
args.fold_data_dir = "{}/fold_data/{}/{}+{}".format(
args.output_dir, args.num_folds,
Path(args.data_dir).name,
Path(args.ivector_dir[0]).name)
else:
args.fold_data_dir = "{}/fold_data/{}/{}".format(
args.output_dir, args.num_folds,
Path(args.data_dir).name)
if args.fold_output_dir is None:
args.fold_output_dir = "{}/fold_data_out/{}/{}".format(
args.output_dir, args.num_folds, args.output_id)
if args.fold_model_dir is None:
args.fold_model_dir = "{}/models/folds/{}/{}".format(
args.output_dir, args.num_folds, args.output_id)
if args.master_dir is None:
args.master_dir = "{}/models/master/{}/{}".format(
args.output_dir, args.num_folds, args.output_id)
if args.rpl_dir is None:
args.rpl_dir = "{}/models/rpl/{}/{}".format(args.output_dir,
args.num_folds,
args.output_id)
# generate folds
if args.num_folds > 0 and args.gen_folds:
print("==== Generating folds")
cmd = [
"-n", args.num_folds, "--data-dir", args.data_dir, "--offset-dir",
args.offset_dir, "--target-dir", args.target_dir,
"--fold-data-dir", args.fold_data_dir, "--utt-list-dir",
args.utt_list_dir, "--train-list", "train.list", "--data",
args.data.format("train"), "--offsets",
args.offsets.format("train"), "--targets",
args.targets.format("train"), "--fold-data-pattern",
args.fold_data_pattern, "--fold-offset-pattern",
args.fold_offset_pattern, "--fold-target-pattern",
args.fold_target_pattern
]
if args.ivector_dir:
cmd += [
'--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern",
args.fold_ivector_pattern
]
generate_folds_main(cmd)
else:
print("==== Skipping fold data generation")
# train master network
if not args.no_train_master:
print("==== Training master network")
cmd = ["--tri", "--noplot", "-b"]
cmd += args.batch_size
cmd += ["--epoch", 20, "-o"]
cmd += args.optimizer
cmd += ["--lr"]
cmd += args.lr
cmd += [
"--data-dir", args.data_dir, "--offset-dir", args.offset_dir,
"--target-dir", args.target_dir, "--data", args.data, "--offsets",
args.offsets, "--targets", args.targets, "--ivectors",
args.ivectors, "--ft", args.ft, "--out", args.master_dir
]
cmd += args.network_spec.split()
if args.ivector_dir:
cmd += ['--ivector-dir', args.ivector_dir[0]]
if args.no_progress:
cmd += ['--no-progress']
train_main(cmd)
else:
print("==== Skipping training master network")
# train folds
if args.num_folds > 0 and not args.no_train_folds:
for fold in range(args.num_folds):
print("==== Training fold {}".format(fold))
cmd = ["--tri", "--noplot", "-b"]
cmd += args.batch_size
cmd += ["--epoch", 20, "-o"]
cmd += args.optimizer
cmd += ["--lr"]
cmd += args.lr
cmd += [
"--data-dir", args.data_dir, "--offset-dir", args.offset_dir,
"--target-dir", args.target_dir, "--data", args.data,
"--offsets", args.offsets, "--targets", args.targets,
"--ivectors", args.ivectors, "--ft", args.ft, "--train-fold",
fold, "--fold-data-dir", args.fold_data_dir,
"--fold-model-dir", args.fold_model_dir, "--fold-data-pattern",
args.fold_data_pattern, "--fold-offset-pattern",
args.fold_offset_pattern, "--fold-target-pattern",
args.fold_target_pattern, "--out", "result_fold_tmp"
]
cmd += args.network_spec.split()
if args.ivector_dir:
cmd += [
'--ivector-dir', args.ivector_dir[0],
"--fold-ivector-pattern", args.fold_ivector_pattern
]
if args.no_progress:
cmd += ['--no-progress']
train_main(cmd)
else:
print("==== Skipping training folds")
# get fold outputs
if args.num_folds > 0 and not args.no_predict:
print("==== Predicting training data")
cmd = [
"--tri", "--ft", args.ft, "--fold-data-dir", args.fold_data_dir,
"--fold-output-dir", args.fold_output_dir, "--fold-model-dir",
args.fold_model_dir, "--fold-data-pattern", args.fold_data_pattern,
"--fold-offset-pattern", args.fold_offset_pattern,
"--fold-network-pattern", args.fold_network_pattern,
"--fold-output-pattern", args.fold_output_pattern
]
cmd += args.network_spec.split()
if args.ivector_dir:
cmd += [
'--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern",
args.fold_ivector_pattern
]
if args.no_progress:
cmd += ['--no-progress']
predict_folds_main(cmd)
print("==== Predicting development data")
cmd = [
"--tri", "--ft", args.ft, "--data-dir", args.data_dir,
"--offset-dir", args.offset_dir, "--data", args.data, "--offsets",
args.offsets, "--fold-output-dir", args.fold_output_dir,
"--fold-model-dir", args.fold_model_dir, "--fold-network-pattern",
args.fold_network_pattern, "--fold-output-dev",
args.fold_output_dev
]
cmd += args.network_spec.split()
if args.ivector_dir:
cmd += [
'--ivector-dir', args.ivector_dir[0], "--fold-ivector-pattern",
args.fold_ivector_pattern
]
if args.no_progress:
cmd += ['--no-progress']
predict_folds_main(cmd)
else:
print("==== Skipping predicting training and development data")
# train RPL layer
if args.num_folds > 0 and not args.no_train_rpl:
print("==== Training RPL layer")
cmd = [
"--train-rpl", "--tri", "-b", 1024, "--epoch", 20, "-o", "adam",
"--lr", "1e-3", "--data-dir", args.fold_output_dir, "--target-dir",
args.target_dir, "--data", args.fold_output_dev, "--targets",
args.targets, "--fold-data-dir", args.fold_data_dir,
"--fold-output-dir", args.fold_output_dir, "--fold-output-pattern",
args.fold_output_pattern, "--fold-target-pattern",
args.fold_target_pattern, "--out", args.rpl_dir
]
if args.no_progress:
cmd += ['--no-progress']
train_main(cmd)
else:
print("==== Skipping training RPL layer")
# evaluate
if not args.no_eval:
for eval_data in args.eval_data:
print("==== Evaluating {} data".format(eval_data))
for eval_folds in [False, True]:
for eval_master in [False, True]:
for eval_rpl in [False, True]:
if (args.num_folds > 0 or args.eval_only_master) and (
eval_folds or not eval_master or eval_rpl):
continue
if eval_folds or eval_master:
print("==== Evaluating {}folds {}master {}rpl".
format("+" if eval_folds else "-",
"+" if eval_master else "-",
"+" if eval_rpl else "-"))
cmd = [
"--tri", "--data-dir", args.data_dir,
"--offset-dir", args.offset_dir,
"--utt-list-dir", args.utt_list_dir, "--data",
args.data, "--offsets", args.offsets,
"--ivectors", args.ivectors, "--ft", args.ft,
"--recog-dir", args.recog_dir, "--rpl",
"--rpl-model", "{}/model".format(args.rpl_dir)
if eval_rpl else "-", "--master-network",
"{}/model".format(args.master_dir)
if eval_master else "-", "--PIP", args.PIP,
"--LMW", args.LMW, "--fold-model-dir",
args.fold_model_dir, "--fold-network-pattern",
args.fold_network_pattern if eval_folds else
"-", "--test-or-dev", eval_data
]
cmd += args.network_spec.split()
if args.ivector_dir:
cmd += ['--ivector-dir', args.ivector_dir[1]]
if args.no_progress:
cmd += ['--no-progress']
evaluate_main(cmd)
else:
print("==== Skipping evaluation")
def batch_train(cfgs, gpus='0'):
for cfg in cfgs:
if os.path.exists(cfg):
cfg = import_module(cfg)
cfg.gpus = gpus
train_main(cfg)
out_dir = os.path.join("$HOME/dispatcher", run_id)
out_dir = os.path.expandvars(out_dir)
os.makedirs(out_dir, exist_ok=True)
variant_generator = CustomVariantGenerator()
unresolved_spec = {"config": cfgs}
for i, variant in enumerate(
variant_generator._generate_resolved_specs(1, unresolved_spec)):
if args.run != "all" and i not in args.run:
continue
print("Dispatch job {}".format(variant["experiment_tag"]))
cfg = variant["spec"]["config"]
cfg["name"] = datetime.now().strftime(
"%Y-%m-%d_%H-%M-%S") + variant["experiment_tag"]
cfg["machine"] = args.machine
cfg["training"]["log_path"] = os.path.join(cfg["training"]["log_path"],
experiment_name) + "/"
out_file = os.path.join(out_dir, variant["trial_id"] + ".yaml")
with open(out_file, 'w') as of:
yaml.safe_dump(cfg, of, default_flow_style=False)
if not args.dry:
try:
if args.exp == 211:
label_selection.label_selection_main(cfg)
else:
train.train_main(cfg)
except Exception:
print(traceback.format_exc())
print("Continue with next experiment.")
torch.cuda.empty_cache()
from eval_checkpoints import main as eval_main
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument('--job-dir', type=str, required=True)
parser.add_argument('--agent-net-size', type=int, default=256)
parser.add_argument('--agent-net-conv', type=int, default=5)
# parser.add_argument('--agent-net-dropout', type=float, default=0.2)
parser.add_argument('--mcts-iter', type=int, default=50)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--epoch-games', type=int, default=5)
parser.add_argument('--benchmark-games', type=int, default=10)
parser.add_argument('--batch-size', type=int, default=128)
parser.add_argument('--lr', type=float, default=1e-4)
parser.add_argument('--lr-decay', type=float, default=1.0)
parser.add_argument('--lr-decay-epochs', type=int, default=5)
parser.add_argument('--reward-gamma', type=float, default=0.99)
parser.add_argument('--num-cpus', type=int, default=cpu_count())
parser.add_argument('--checkpoint-gamma', type=float, default=0.2)
parser.add_argument('--checkpoint-last-n', type=int, default=None)
parser.add_argument('--contest-to-update', type=bool, default=False)
parser.add_argument('--win-rate-threshold', type=float, default=0.6)
parser.add_argument('--seed', type=int, default=None)
parser.add_argument('--eval', type=bool, default=False)
args = parser.parse_args()
if args.eval:
eval_main(args)
else:
train_main(args)
current_loss = 0
all_losses = []
start = time.time()
patience = args.patience
savePath = args.savePath
train_path = "../data/dummy/classification_train.csv"
test_path = "../data/dummy/classification_test.csv"
for ei in range(args.max_epochs):
bad_counter = 0
best_loss = -1.0
train_main(args, model, train_path, criterion, optimizer)
dayloss, valid_loss = test(args, model, test_path, criterion)
print("valid loss : {}".format(valid_loss))
print(dayloss)
if valid_loss < best_loss or best_loss < 0:
print("find best")
bad_counter = 0
torch.save(model, args.saveModel)
else:
bad_counter += 1
if bad_counter > patience:
print('Early Stopping')
break
# Draw a sample
def main(self):
work_dir = dirname(abspath(__file__))
chdir(work_dir)
servicemanager.LogInfoMsg('Module directory is %s' % work_dir)
train_main(with_gui=False, check_stop=self.check_stop)
def test_train(cfg):
cfg.train.n_gpu = 0
train_main(cfg)
f1s = []
for n in range(n_exp):
model = RNN(args)
print("Setting Model Complete")
model.to(args.device)
optimizer = "optim." + args.optim
optimizer = eval(optimizer)(model.parameters(), lr=args.lr)
#optimizer = optim.RMSprop(model.parameters())
#optimizer = optim.Adam(model.parameters())
#oprimizer = optim.SGD(model.parameters())
print("Setting optimizer Complete")
train_main(model, args, train_data, valid_data, optimizer)
acc, f1 = test(model, args, test_data)
accs.append(acc)
f1s.append(f1)
print("=" * 90)
print("results on testset at each trials:")
for i in range(n_exp):
print("exp {:d} | acc {:.3f} | f1 {:.3f}".format(i + 1, accs[i], f1s[i]))
total_acc = sum(accs) / n_exp
total_f1 = sum(f1s) / n_exp
print("-" * 90)
def voc_val(eval_steps):
train_main('latest.pth', split='trainaug')
eval_main('best_trainaug.pth', eval_steps, 'best')
eval_main('final_trainaug.pth', eval_steps, 'final')
