async def handle_delete(self, request):
"""
API endpoint for deleting an experiment.
Usage:
POST /v1/_delete/[id]
Returns:
The IDs of the deleted experiments.
"""
async def delete_one(the_id):
doc = await self.mldb.get(the_id)
try:
await self.store_mgr.delete(the_id, doc)
except Exception:
getLogger(__name__).debug(
'Failed to delete the storage for experiment %s',
the_id,
exc_info=True)
id = path_info_get(request, 'id', validator=validate_experiment_id)
delete_ids = sorted(set(await self.mldb.mark_delete(id)))
if not delete_ids:
raise web.HTTPNotFound()
await asyncio.gather(
*[delete_one(delete_id) for delete_id in delete_ids])
await self.mldb.complete_deletion(delete_ids)
return delete_ids
async def handle_heartbeat(self, request):
"""
API endpoint for experiment heartbeat.
Usage:
POST /v1/_heartbeat/[id]
Returns:
dict: An empty dict ``{}``.
"""
id = path_info_get(request, 'id', validator=validate_experiment_id)
await self.mldb.set_heartbeat(id)
return {}
async def handle_get(self, request):
"""
API endpoint for getting experiment document.
Usage:
GET /v1/_get/[id]
Returns:
The experiment document.
"""
id = path_info_get(request, 'id', validator=validate_experiment_id)
return await get_doc_or_error(self.mldb, self.store_mgr, id,
web.HTTPNotFound)
async def handle_update(self, request):
"""
API endpoint for updating experiment document.
Usage:
POST /v1/_update/[id] {...}
Returns:
The updated experiment document.
"""
id = path_info_get(request, 'id', validator=validate_experiment_id)
doc_fields = await request.json()
await self.mldb.update(id, doc_fields)
return await get_doc_or_error(self.mldb, self.store_mgr, id)
async def handle_listdir(self, request):
"""
API endpoint for listing a directory.
Usage:
GET /v1/_listdir/[id]/[path]
Returns:
The list of entries.
"""
path = path_info_get(request, 'path', '')
store = await get_file_store(request, self.mldb, self.store_mgr)
ret = []
for e in (await store.listdir_and_stat(path)):
ret.append(file_entry_to_dict(e))
return ret
async def handle_set_finished(self, request):
"""
API endpoint for setting the status of an experiment to be finished.
Usage:
POST /v1/_set_finished/[id] {"status": ..., ...}
Returns:
The updated experiment document.
"""
id = path_info_get(request, 'id', validator=validate_experiment_id)
doc_fields = await request.json()
if 'status' not in doc_fields:
raise web.HTTPBadRequest()
await self.mldb.set_finished(id, doc_fields['status'], doc_fields)
return await get_doc_or_error(self.mldb, self.store_mgr, id)
async def handle_getfile(self, request):
"""
API endpoint for getting a file.
Usage:
GET /v1/_getfile/[id]/[path]
Returns:
The file content.
"""
path = path_info_get(request, 'path', '')
store = await get_file_store(request, self.mldb, self.store_mgr)
if not (await store.isfile(path)):
raise web.HTTPNotFound()
headers = {}
# Special treatment for console.log: force it to be recognized
# as plain, UTF-8 text.
if path.endswith('console.log'):
headers['Content-Type'] = 'text/plain; charset=utf-8'
return web.FileResponse(store.resolve_path(path), headers=headers)
async def handle_tarball(self, request):
"""
API endpoint for downloading all files of an experiment as a tarball.
Usage:
GET /v1/_tarball/[id]
"""
id = path_info_get(request, 'id', validator=validate_experiment_id)
doc = await get_doc_or_error(self.mldb,
self.store_mgr,
id,
error_class=web.HTTPNotFound)
root_path = doc['storage_dir']
if not os.path.isdir(root_path):
raise web.HTTPNotFound()
tar_script = os.path.join(
os.path.split(os.path.abspath(__file__))[0], 'tar.py')
proc = await asyncio.create_subprocess_exec(
sys.executable,
tar_script,
root_path,
str(id),
stdout=asyncio.subprocess.PIPE)
try:
resp = web.StreamResponse(
headers={
'Content-Type': 'application/x-tar',
'Content-Disposition':
'attachment; filename={}.tar'.format(id)
})
await resp.prepare(request)
while True:
buf = await proc.stdout.read(8192)
if not buf:
break
await resp.write(buf)
await resp.write_eof()
finally:
if proc.returncode is None:
proc.terminate()
_ = proc.wait()
async def get_file_store(request, mldb, store_mgr):
id = path_info_get(request, 'id', validator=validate_experiment_id)
doc = await get_doc_or_error(mldb, store_mgr, id, web.HTTPNotFound)
store = await store_mgr.open(id, doc)
return store