def delete_from_id(task_id):
if type(task_id) is str:
t = Task.get(Task.id == task_id) # type: Task
t.delete_instance()
else:
assert type(task_id) is list
Task.delete().where(Task.id.in_(task_id)).execute()
def cancel_from_id(task_id):
if type(task_id) is str:
t = Task.get(Task.id == task_id)
if t.status.is_active():
t.cancel()
else:
assert type(task_id) is list
tasks = Task.select().where(Task.id.in_(task_id))
for t in tasks:
if t.status.is_active():
get_db().close() # avoid deadlock
t.cancel() # TODO one bulk ssh command
def get_tasks(platform: ComputePlatformType):
ExperimentManager.update_statuses(
platforms=[platform]) # TODO return tasks to avoid other db query?
tasks = Task.select().where(Task.platform_type == platform)
for t in tasks:
t.monitor()
return tasks
def update_statuses(platforms: Union[str, list] = 'all'):
if platforms == 'all':
platforms = [
ComputePlatformType.LOCAL, ComputePlatformType.HELIOS
] # FIXME dynamic
for ptype in platforms:
platform = get_platform(ptype)
tasks = Task.select().where(Task.platform_type == ptype)
job_ids = [t.job_id for t in tasks]
get_db().close() # Close db since the following may take time
statuses = platform.get_statuses(job_ids)
for t in tasks:
if t.status.is_active():
t.status = statuses[t.job_id]
t.save()
def monitor(task_id):
task = Task.get(Task.id == task_id)
task.monitor()
if 'out' in task.logs:
selected_log = 'out'
else:
selected_log = next(iter(task.logs.keys()))
viz_scripts, viz_divs = None, None
if len(task.metrics) > 0:
viz_scripts, viz_divs = viz.generate_plots(task.metrics)
return render_template('monitor.html',
task=task,
selected_log=selected_log,
viz_scripts=viz_scripts,
viz_divs=viz_divs)
def submit(platform: str, script_file: str, config_file: str):
# Load yaml config
config_file_path = Task.resolve_path(config_file)
yaml_config = yaml.load(config_file_path)
name = config_file_path.stem
# Handle hpsearch
if 'hpsearch' in yaml_config:
configs = generate_hpsearch(yaml_config, name)
else:
configs = {name: yaml_config}
# Make tasks
tasks = []
for name, config in configs.items():
t = Task(script_file=script_file,
config=config,
name=name,
platform_type=ComputePlatformType(platform))
t.save() # insert in database
tasks.append(t)
# Submit tasks
for t in tasks:
get_db().close() # avoid deadlock
t.submit() # FIXME submit all at once!
def submit(self, script_path: Path, config_file_path: Path):
t = Task(script_path, config_file_path)
t.submit()
self.tasks[t.task_id] = t
def get_all_tasks():
# NOTE: statuses are not updated here; they are requested asynchronously by the dashboard
all_tasks = list(Task.select())
return all_tasks