def init(self,
desc_name,
expr_name,
run_name,
args=None,
highlight_args=None,
configs=None):
desc, _ = Desc.get_or_create(desc_name=desc_name)
expr, _ = Experiment.get_or_create(desc=desc, expr_name=expr_name)
run = Run(expr=expr, run_name=run_name, command=' '.join(sys.argv))
run.args = ''
run.highlight_args = ''
if args is not None:
run.args = io.dumps_json(args.__dict__)
if highlight_args is not None and hasattr(highlight_args,
'highlight_args'):
run.highlight_args = io.dumps_json(
get_highlight_args(args, highlight_args))
run.configs = ''
run.highlight_configs = ''
if configs is not None:
run.configs = io.dumps_json(configs)
if getattr(args, 'configs', None) is not None:
run.highlight_configs = io.dumps_json(args.configs.kvs)
save_retry(run)
self.desc = desc
self.expr = expr
self.run = run
def mark_as_trash_by_spec(specs):
failed = list()
for spec in specs:
try:
name = spec['desc']
x = Desc.get_or_none(desc_name=spec['desc'])
if x is None:
raise CannotDeleteException(name)
if 'expr' in spec:
name += '/' + spec['expr']
x = Experiment.get_or_none(desc=x, expr_name=spec['expr'])
if x is None:
raise CannotDeleteException(name)
else:
check_empty_record(x, 'exprs', name)
if 'run' in spec:
name += '/' + spec['run']
x = Run.get_or_none(expr=x, run_name=spec['run'])
if x is None:
raise CannotDeleteException(name)
else:
check_empty_record(x, 'runs', name)
mark_as_trash(x)
except CannotDeleteException as e:
failed.append(str(e))
return failed
def mark_as_trash_by_spec(specs):
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
mark_as_trash(run)
run.save()
def mark_as_stared_by_spec(specs, value=True):
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
mark_as_stared(run, value)
run.save()
def get():
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return None
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return desc
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return expr
return run
def start(self, specs):
logdirs = dict()
exprs = set()
descs = set()
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
key = run.expr.desc.desc_name + '/' + run.expr.expr_name + '/' + run.run_name + '/' + spec['highlight']
key = escape_tensorboard_name(key)
value = run.tb_dir
if value is not None and value != '':
logdirs[key] = value
descs.add(spec['desc'])
exprs.add((spec['desc'], str(spec['expr'])))
if len(logdirs) == 0:
return None
logdirs_string = ','.join(['{}:{}'.format(k, v) for k, v in logdirs.items()])
port = find_port()
cmd = ['tensorboard', '--logdir', logdirs_string, '--port', str(port)]
import tensorflow as tf
if tf.__version__ >= '2.0.0':
cmd[1] = '--logdir_spec'
cmd.append('--bind_all')
process = subprocess.Popen(cmd, stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w'))
logger.critical('Start tensorboard service: "{}".'.format(' '.join(cmd)))
record = dict(index=self.index, logdirs=logdirs, logdirs_string=logdirs_string, descs=descs, exprs=exprs, port=port, process=process)
self.index += 1
with self.mutex:
self.running_tensorboards.append(record)
return record
def main():
init_database('.')
ProjectMetainfo.set_all({
'title': 'Test',
'create_time': '2019-09-05 15:44:27',
'update_time': '2019-09-05 15:44:27'
})
print(ProjectMetainfo.get_all())
print('Create the desc')
desc, _ = Desc.get_or_create(desc_name='desc_xxx')
print('Create the expr')
expr, _ = Experiment.get_or_create(expr_name='expr1', desc=desc)
print('Create the runs')
train_run = Run(expr=expr,
run_name='train1' + str(time.time()),
command='jac-run xxx',
args='{xxx}',
configs='{xxx}',
highlight_args='{yyy}',
highlight_configs='{yyy}')
test_run = Run(expr=expr,
run_name='test1' + str(time.time()),
command='jac-run xxx',
args='{xxx}',
configs='{xxx}',
highlight_args='{yyy}',
highlight_configs='{yyy}',
refer=train_run)
from IPython import embed
embed()
train_run.save()
test_run.save()
def get(self):
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return
self.render('run.html',
run=run,
expr=expr,
desc=desc,
run_methods=get_run_methods())
def get(self):
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return
run_methods = get_run_methods()
cmd_name = self.get_argument('cmd', '')
method = None
for m in run_methods:
if m.__name__ == cmd_name:
method = m
if method is None:
return
import multiprocessing
import sys
self.write('<pre>')
self.flush()
sys.stdout = self
sys.stderr = self
proc = multiprocessing.Process(target=method, args=(run, ))
proc.start()
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
proc.join()
self.write('</pre>')
self.flush()
self.finish()
def start(self, specs):
logdirs = dict()
exprs = set()
descs = set()
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
key = run.expr.desc.desc_name + '/' + run.expr.expr_name + '/' + run.run_name + '/' + spec['highlight']
key = key.replace(';', '_').replace(' ', '')
value = run.tb_dir
if value is not None and value != '':
logdirs[key] = value
descs.add(spec['desc'])
exprs.add((spec['desc'], spec['expr']))
if len(logdirs) == 0:
return None
logdirs_string = ','.join(['{}:{}'.format(k, v) for k, v in logdirs.items()])
port = find_port()
process = subprocess.Popen(
['tensorboard', '--logdir', logdirs_string, '--port', str(port)],
stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w')
)
record = dict(index=self.index, logdirs=logdirs, logdirs_string=logdirs_string, descs=descs, exprs=exprs, port=port, process=process)
self.index += 1
with self.mutex:
self.running_tensorboards.append(record)
return record
def update_parent(self, parent, is_master=False):
self._refresh()
self.run.is_master = is_master
self.run.refer = Run.get_or_none(expr=self.expr, run_name=parent)
save_retry(self.run)
def _refresh(self):
self.desc = Desc.get_by_id(self.desc.get_id())
self.expr = Experiment.get_by_id(self.expr.get_id())
self.run = Run.get_by_id(self.run.get_id())