def run():
for n in make_index_list(load_config()):
base_dir = Path(str(n)).resolve()
base_dir.mkdir(exist_ok=True)
script = TrainRunScript(n, base_dir)
with open(base_dir / 'finetune.sh', 'w') as f:
f.write(str(script))
def run():
config = load_config()
first_trial = 0
for n in range(config['pretrain']['trials']):
base_dir = Path(str(n)).resolve()
base_dir.mkdir(exist_ok=True)
script = PretrainDataRunScript(base_dir, n, first_trial)
with open(base_dir / 'data.sh', 'w') as f:
f.write(str(script))
return first_trial
def run():
for n in make_index_list(load_config()):
base_dir = Path(str(n)).resolve()
base_dir.mkdir(exist_ok=True)
main_script = PretrainRunScript(n, base_dir)
worker_script = WorkerRunScript(n, base_dir)
with open(base_dir / 'pretrain.sh', 'w') as f:
f.write(str(main_script))
with open(base_dir / 'worker.sh', 'w') as f:
f.write(str(worker_script))
def sub(first_trial):
config = load_config()
indices = [n for n in range(config['pretrain']['trials'])]
if first_trial is not None:
indices = [n for n in indices if n != first_trial]
sub_script = DataSubScript([first_trial])
with open('first_sub.sh', 'w') as f:
f.write(str(sub_script))
sub_script = DataSubScript(indices)
with open('sub.sh', 'w') as f:
f.write(str(sub_script))
def valid_rescore_errant_results(dataset):
config = load_config()
result_list = []
for l in config['rescore']['lambda']:
base_dir = make_ensemble_base_dir(dataset, 'valid')
lmil = int(l * 1000)
output = base_dir / 'result.{}.cat2'.format(lmil)
if output.exists():
result = ErrantCat2Result(None, None, output, l = l)
result_list.append(result)
return result_list
def make_rescore_result_list(result_class, dataset, stage):
config = load_config()
result_list = []
for l in config['rescore']['lambda']:
base_dir = make_ensemble_base_dir(dataset, stage)
lmil = int(l * 1000)
output = base_dir / 'result.{}.res'.format(lmil)
if output.exists():
result = result_class(None, None, output, l=l)
result_list.append(result)
return result_list
def __init__(self, dataset, stage):
self.config = load_config()
self.dataset = dataset
self.stage = stage
self.make()
def __init__(self):
super().__init__()
self.config = load_config()
self.header()
self.make()