run.metadata['host'] = socket.gethostname()
# start training
run.workspace.add_file('stdout', 'stdout.txt')
run.workspace.add_file('stderr', 'stderr.txt')
run.workspace.add_file('command', 'command.txt')
with open(run.workspace.command, 'a') as fout:
fout.write(str(sys.argv))
if args.comment:
run.workspace.add_file('comment', 'comment.txt')
with open(run.workspace.comment, 'a') as fout:
fout.write(str(args.comment))
if args.profile:
from gtd.chrono import Profiling, Profiler
profiler = Profiler.default()
# To profile a class, do something like
# import phrasenode.foo
# profiler.add_module(phrasenode.foo)
Profiling.start()
with save_stdout(run.workspace.root):
try:
run.load_latest_model()
run.train()
finally:
run.close()
if args.profile:
Profiling.report()
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('exp_id', nargs='+')
arg_parser.add_argument('-c', '--check_commit', default='strict')
arg_parser.add_argument('-p', '--profile', action='store_true')
args = arg_parser.parse_args()
# create experiment
experiments = EditTrainingRuns(
check_commit=(args.check_commit == 'disable')) #'strict' wyl
exp_id = args.exp_id
if exp_id == ['default']:
# new default experiment
exp = experiments.new()
elif len(exp_id) == 1 and exp_id[0].isdigit():
# reload old experiment
exp = experiments[int(exp_id[0])]
else:
# new experiment according to configs
config = Config.from_file(exp_id[0])
for filename in exp_id[1:]:
config = Config.merge(config, Config.from_file(filename))
exp = experiments.new(config) # new experiment from config
# start training
exp.workspace.add_file('stdout', 'stdout.txt')
exp.workspace.add_file('stderr', 'stderr.txt')
with save_stdout(exp.workspace.root):
exp.train()
# profiler = Profiler.default()
# import gtd.ml.seq_batch; profiler.add_module(gtd.ml.seq_batch)
# import strongsup.decoder; profiler.add_module(strongsup.decoder)
# import strongsup.parse_model; profiler.add_module(strongsup.parse_model)
# import strongsup.parse_case; profiler.add_module(strongsup.parse_case)
# import strongsup.tables.predicates_computer; profiler.add_module(strongsup.tables.predicates_computer)
# import strongsup.tables.executor; profiler.add_module(strongsup.tables.executor); strongsup.tables.executor.add_decorated_methods(profiler)
# import strongsup.exploration_policy; profiler.add_module(strongsup.exploration_policy)
# Profiling.start()
################################
# start training
exp.workspace.add_file('stdout', 'stdout.txt')
exp.workspace.add_file('stderr', 'stderr.txt')
with save_stdout(exp.workspace.root):
try:
config = exp.config
if exp_mode == 'train':
print '\n===== TRAIN MODE ====='
if config.train_mode == 'semi-supervised':
exp.train()
elif config.train_mode == 'supervised':
exp.supervised_train()
else:
raise ValueError('Invalid train mode: {}'.format(config.train_mode))
elif exp_mode == 'eval':
print '\n===== EVALUATION MODE ====='
exp.big_evaluate(eval_samples)
else:
