def mark_as_trash_by_spec(specs):
failed = list()
for spec in specs:
try:
name = spec['desc']
x = Desc.get_or_none(desc_name=spec['desc'])
if x is None:
raise CannotDeleteException(name)
if 'expr' in spec:
name += '/' + spec['expr']
x = Experiment.get_or_none(desc=x, expr_name=spec['expr'])
if x is None:
raise CannotDeleteException(name)
else:
check_empty_record(x, 'exprs', name)
if 'run' in spec:
name += '/' + spec['run']
x = Run.get_or_none(expr=x, run_name=spec['run'])
if x is None:
raise CannotDeleteException(name)
else:
check_empty_record(x, 'runs', name)
mark_as_trash(x)
except CannotDeleteException as e:
failed.append(str(e))
return failed
def mark_as_trash_by_spec(specs):
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
mark_as_trash(run)
run.save()
def mark_as_stared_by_spec(specs, value=True):
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
mark_as_stared(run, value)
run.save()
def get():
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return None
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return desc
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return expr
return run
def start(self, specs):
logdirs = dict()
exprs = set()
descs = set()
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
key = run.expr.desc.desc_name + '/' + run.expr.expr_name + '/' + run.run_name + '/' + spec['highlight']
key = escape_tensorboard_name(key)
value = run.tb_dir
if value is not None and value != '':
logdirs[key] = value
descs.add(spec['desc'])
exprs.add((spec['desc'], str(spec['expr'])))
if len(logdirs) == 0:
return None
logdirs_string = ','.join(['{}:{}'.format(k, v) for k, v in logdirs.items()])
port = find_port()
cmd = ['tensorboard', '--logdir', logdirs_string, '--port', str(port)]
import tensorflow as tf
if tf.__version__ >= '2.0.0':
cmd[1] = '--logdir_spec'
cmd.append('--bind_all')
process = subprocess.Popen(cmd, stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w'))
logger.critical('Start tensorboard service: "{}".'.format(' '.join(cmd)))
record = dict(index=self.index, logdirs=logdirs, logdirs_string=logdirs_string, descs=descs, exprs=exprs, port=port, process=process)
self.index += 1
with self.mutex:
self.running_tensorboards.append(record)
return record
def get(self):
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return
self.render('run.html',
run=run,
expr=expr,
desc=desc,
run_methods=get_run_methods())
def get(self):
desc_name = self.get_argument('desc', '')
desc = Desc.get_or_none(desc_name=desc_name)
if desc is None:
return
expr_name = self.get_argument('expr', '')
expr = Experiment.get_or_none(desc=desc, expr_name=expr_name)
if expr is None:
return
run_name = self.get_argument('run', '')
run = Run.get_or_none(expr=expr, run_name=run_name)
if run is None:
return
run_methods = get_run_methods()
cmd_name = self.get_argument('cmd', '')
method = None
for m in run_methods:
if m.__name__ == cmd_name:
method = m
if method is None:
return
import multiprocessing
import sys
self.write('<pre>')
self.flush()
sys.stdout = self
sys.stderr = self
proc = multiprocessing.Process(target=method, args=(run, ))
proc.start()
sys.stdout = sys.__stdout__
sys.stderr = sys.__stderr__
proc.join()
self.write('</pre>')
self.flush()
self.finish()
def start(self, specs):
logdirs = dict()
exprs = set()
descs = set()
for spec in specs:
desc = Desc.get_or_none(desc_name=spec['desc'])
if desc is None:
continue
expr = Experiment.get_or_none(desc=desc, expr_name=spec['expr'])
if expr is None:
continue
run = Run.get_or_none(expr=expr, run_name=spec['run'])
if run is None:
continue
key = run.expr.desc.desc_name + '/' + run.expr.expr_name + '/' + run.run_name + '/' + spec['highlight']
key = key.replace(';', '_').replace(' ', '')
value = run.tb_dir
if value is not None and value != '':
logdirs[key] = value
descs.add(spec['desc'])
exprs.add((spec['desc'], spec['expr']))
if len(logdirs) == 0:
return None
logdirs_string = ','.join(['{}:{}'.format(k, v) for k, v in logdirs.items()])
port = find_port()
process = subprocess.Popen(
['tensorboard', '--logdir', logdirs_string, '--port', str(port)],
stdout=open(os.devnull, 'w'), stderr=open(os.devnull, 'w')
)
record = dict(index=self.index, logdirs=logdirs, logdirs_string=logdirs_string, descs=descs, exprs=exprs, port=port, process=process)
self.index += 1
with self.mutex:
self.running_tensorboards.append(record)
return record
def update_parent(self, parent, is_master=False):
self._refresh()
self.run.is_master = is_master
self.run.refer = Run.get_or_none(expr=self.expr, run_name=parent)
save_retry(self.run)