async def get_task_by_request(self, request):
flow_id, run_number, step_name, task_id, attempt_id = \
get_pathspec_from_request(request)
run_id_key, run_id_value = translate_run_key(run_number)
task_id_key, task_id_value = translate_task_key(task_id)
conditions = [
"flow_id = %s", "{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s",
"{task_id_key} = %s".format(task_id_key=task_id_key)
]
values = [flow_id, run_id_value, step_name, task_id_value]
if attempt_id:
conditions.append("attempt_id = %s")
values.append(attempt_id)
# NOTE: Must enable joins so task has attempt_id present for filtering.
# Log cache action requires a task with an attempt_id,
# otherwise it defaults to attempt 0
db_response, *_ = await self.task_table.find_records(
fetch_single=True,
conditions=conditions,
values=values,
order=["attempt_id DESC"],
enable_joins=True,
expanded=True)
if db_response.response_code == 200:
return db_response.body
return None
async def get_metadata(self, request):
"""
---
description: Get all metadata of specified task
tags:
- Metadata
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
- $ref: '#/definitions/Params/Path/step_name'
- $ref: '#/definitions/Params/Path/task_id'
- $ref: '#/definitions/Params/Builtin/_page'
- $ref: '#/definitions/Params/Builtin/_limit'
- $ref: '#/definitions/Params/Builtin/_order'
- $ref: '#/definitions/Params/Builtin/_tags'
- $ref: '#/definitions/Params/Builtin/_group'
- $ref: '#/definitions/Params/Custom/flow_id'
- $ref: '#/definitions/Params/Custom/run_number'
- $ref: '#/definitions/Params/Custom/step_name'
- $ref: '#/definitions/Params/Custom/task_id'
- $ref: '#/definitions/Params/Custom/field_name'
- $ref: '#/definitions/Params/Custom/value'
- $ref: '#/definitions/Params/Custom/type'
- $ref: '#/definitions/Params/Custom/user_name'
- $ref: '#/definitions/Params/Custom/ts_epoch'
produces:
- application/json
responses:
"200":
description: Returns all metadata of specified task
schema:
$ref: '#/definitions/ResponsesMetadataList'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_id_key, run_id_value = translate_run_key(
request.match_info.get("run_number"))
step_name = request.match_info.get("step_name")
task_id_key, task_id_value = translate_task_key(
request.match_info.get("task_id"))
return await find_records(request,
self._async_table,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(
run_id_key=run_id_key),
"step_name = %s",
"{task_id_key} = %s".format(
task_id_key=task_id_key)],
initial_values=[
flow_name, run_id_value, step_name, task_id_value],
allowed_order=self._async_table.keys,
allowed_group=self._async_table.keys,
allowed_filters=self._async_table.keys + ["attempt_id"]
)
async def get_run_parameter_artifacts(self, flow_name, run_number, postprocess=None, invalidate_cache=False):
run_id_key, run_id_value = translate_run_key(run_number)
# '_parameters' step has all the parameters as artifacts. only pick the
# public parameters (no underscore prefix)
return await self.find_records(
conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s",
"name NOT LIKE %s",
"name <> %s",
"name <> %s"
],
values=[
flow_name,
run_id_value,
"_parameters",
r"\_%",
"name", # exclude the 'name' parameter as this always exists, and contains the FlowName
"script_name" # exclude the internally used 'script_name' parameter.
],
fetch_single=False,
expanded=True,
postprocess=postprocess,
invalidate_cache=invalidate_cache
)
async def get_run(self, flow_id: str, run_key: str):
"""
Fetch run with a given flow_id and run id or number from the DB.
Parameters
----------
flow_id : str
flow_id
run_key : str
run number or run id
Returns
-------
DBResponse
Containing a single run record, if one was found.
"""
run_id_key, run_id_value = translate_run_key(run_key)
result, *_ = await self.find_records(conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
],
values=[flow_id, run_id_value],
fetch_single=True,
enable_joins=True)
return result
async def get_tasks_for_run(self,
flow_id: str,
run_key: str,
postprocess: Callable = None) -> DBResponse:
"""
Fetches run tasks from DB.
Parameters
----------
flow_id : str
Flow id of the task
run_key : str
Run number or run id of the task
postprocess : Callable
A callback function for refining results.
Receives DBResponse as an argument, and should return a DBResponse
Returns
-------
DBResponse
"""
run_id_key, run_id_value = translate_run_key(run_key)
conditions = [
"flow_id = %s",
"{run_id_column} = %s".format(run_id_column=run_id_key)
]
values = [flow_id, run_id_value]
result, *_ = await self.find_records(conditions=conditions,
values=values,
fetch_single=False,
enable_joins=True,
expanded=False,
postprocess=postprocess)
return result
async def get_artifacts_by_run(self, request):
"""
---
description: Get all artifacts of specified run
tags:
- Artifact
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
- $ref: '#/definitions/Params/Builtin/_page'
- $ref: '#/definitions/Params/Builtin/_limit'
- $ref: '#/definitions/Params/Builtin/_order'
- $ref: '#/definitions/Params/Builtin/_tags'
- $ref: '#/definitions/Params/Builtin/_group'
- $ref: '#/definitions/Params/Custom/flow_id'
- $ref: '#/definitions/Params/Custom/run_number'
- $ref: '#/definitions/Params/Custom/step_name'
- $ref: '#/definitions/Params/Custom/task_id'
- $ref: '#/definitions/Params/Custom/name'
- $ref: '#/definitions/Params/Custom/type'
- $ref: '#/definitions/Params/Custom/ds_type'
- $ref: '#/definitions/Params/Custom/attempt_id'
- $ref: '#/definitions/Params/Custom/postprocess'
- $ref: '#/definitions/Params/Custom/invalidate'
- $ref: '#/definitions/Params/Custom/user_name'
- $ref: '#/definitions/Params/Custom/ts_epoch'
produces:
- application/json
responses:
"200":
description: Returns all artifacts of specified run
schema:
$ref: '#/definitions/ResponsesArtifactList'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_number = request.match_info.get("run_number")
run_id_key, run_id_value = translate_run_key(run_number)
return await find_records(
request,
self._async_table,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key)
],
initial_values=[flow_name, run_id_value],
allowed_order=self._async_table.keys,
allowed_group=self._async_table.keys,
allowed_filters=self._async_table.keys,
postprocess=postprocess_chain([
apply_run_tags_postprocess(flow_name, run_number,
self._async_run_table),
self.get_postprocessor(request)
]))
async def get_task_attempt(self,
flow_id: str,
run_key: str,
step_name: str,
task_key: str,
attempt_id: int = None,
postprocess: Callable = None) -> DBResponse:
"""
Fetches task attempt from DB. Specifying attempt_id will fetch the specific attempt.
Otherwise the newest attempt is returned.
Parameters
----------
flow_id : str
Flow id of the task
run_key : str
Run number or run id of the task
step_name : str
Step name of the task
task_key : str
task id or task name
attempt_id : int (optional)
The specific attempt of the task to be fetched. If not provided, the latest attempt is returned.
postprocess : Callable
A callback function for refining results.
Receives DBResponse as an argument, and should return a DBResponse
Returns
-------
DBResponse
"""
run_id_key, run_id_value = translate_run_key(run_key)
task_id_key, task_id_value = translate_task_key(task_key)
conditions = [
"flow_id = %s",
"{run_id_column} = %s".format(run_id_column=run_id_key),
"step_name = %s",
"{task_id_column} = %s".format(task_id_column=task_id_key)
]
values = [flow_id, run_id_value, step_name, task_id_value]
if attempt_id:
conditions.append("attempt_id = %s")
values.append(attempt_id)
result, *_ = await self.find_records(conditions=conditions,
values=values,
order=["attempt_id DESC"],
fetch_single=True,
enable_joins=True,
expanded=True,
postprocess=postprocess)
return result
async def get_task(self, request):
"""
---
description: Get one task
tags:
- Task
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
- $ref: '#/definitions/Params/Path/step_name'
- $ref: '#/definitions/Params/Path/task_id'
- $ref: '#/definitions/Params/Custom/postprocess'
- $ref: '#/definitions/Params/Custom/invalidate'
produces:
- application/json
responses:
"200":
description: Returns one task
schema:
$ref: '#/definitions/ResponsesTask'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_number = request.match_info.get("run_number")
run_id_key, run_id_value = translate_run_key(run_number)
step_name = request.match_info.get("step_name")
task_id_key, task_id_value = translate_task_key(
request.match_info.get("task_id"))
return await find_records(
request,
self._async_table,
fetch_single=True,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s",
"{task_id_key} = %s".format(task_id_key=task_id_key)
],
initial_values=[flow_name, run_id_value, step_name, task_id_value],
initial_order=["attempt_id DESC"],
enable_joins=True,
postprocess=postprocess_chain([
apply_run_tags_postprocess(flow_name, run_number,
self._async_run_table),
self.get_postprocessor(request)
]))
async def get_run_artifacts(self, flow_name, run_key, artifact_name):
"""
Find a set of artifacts to perform the search over.
Includes localstore artifacts as well, as we want to return that these could not be searched over.
"""
run_id_key, run_id_value = translate_run_key(run_key)
db_response = await self._artifact_table.get_records(
filter_dict={
"flow_id": flow_name,
run_id_key: run_id_value,
"name": artifact_name
})
db_response = await apply_run_tags_to_db_response(
flow_name, run_key, self._run_table, db_response)
return db_response.body
async def get_step(self, request):
"""
---
description: Get one step
tags:
- Step
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
- $ref: '#/definitions/Params/Path/step_name'
produces:
- application/json
responses:
"200":
description: Returns one step
schema:
$ref: '#/definitions/ResponsesStep'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_number = request.match_info.get("run_number")
run_id_key, run_id_value = translate_run_key(run_number)
step_name = request.match_info.get("step_name")
return await find_records(
request,
self._async_table,
fetch_single=True,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s"
],
initial_values=[flow_name, run_id_value, step_name],
enable_joins=True,
postprocess=apply_run_tags_postprocess(flow_name, run_number,
self._async_run_table))
async def get_expanded_run(self, run_key: str) -> DBResponse:
"""
Fetch run with a given id or number from the DB.
Parameters
----------
run_key : str
run number or run id
Returns
-------
DBResponse
Containing a single run record, if one was found.
"""
run_id_key, run_id_value = translate_run_key(run_key)
result, *_ = await self.find_records(
conditions=["{column} = %s".format(column=run_id_key)],
values=[run_id_value],
fetch_single=True,
enable_joins=True,
expanded=True)
return result
async def get_run_codepackage_metadata(self, flow_name: str, run_id: str) -> DBResponse:
"""
Tries to locate 'code-package' or 'code-package-url' in run metadata.
"""
run_id_key, run_id_value = translate_run_key(run_id)
# 'code-package' value contains json with dstype, sha1 hash and location
# 'code-package-url' value contains only location as a string
db_response, *_ = await self.find_records(
conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(
run_id_key=run_id_key),
"(field_name = %s OR field_name = %s)"
],
values=[
flow_name, run_id_value,
"code-package", "code-package-url"
],
fetch_single=True, expanded=True
)
return db_response
async def get_run_graph_info_artifact(self, flow_name: str, run_id: str) -> DBResponse:
"""
Tries to locate '_graph_info' in run artifacts
"""
run_id_key, run_id_value = translate_run_key(run_id)
db_response, *_ = await self.find_records(
conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(
run_id_key=run_id_key),
"step_name = %s",
"name = %s"
],
values=[
flow_name, run_id_value, "_parameters",
"_graph_info",
],
fetch_single=True, expanded=True
)
return db_response
async def get_task_by_request(self, request):
flow_id, run_number, step_name, task_id, _ = \
get_pathspec_from_request(request)
run_id_key, run_id_value = translate_run_key(run_number)
task_id_key, task_id_value = translate_task_key(task_id)
conditions = [
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s",
"{task_id_key} = %s".format(task_id_key=task_id_key)
]
values = [flow_id, run_id_value, step_name, task_id_value]
db_response, *_ = await self.db.task_table_postgres.find_records(
fetch_single=True,
conditions=conditions,
values=values,
expanded=True
)
if db_response.response_code == 200:
return db_response.body
return None
async def get_run(self, request):
"""
---
description: Get one run
tags:
- Run
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
produces:
- application/json
responses:
"200":
description: Returns one run
schema:
$ref: '#/definitions/ResponsesRun'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_id_key, run_id_value = translate_run_key(
request.match_info.get("run_number"))
return await find_records(request,
self._async_table,
fetch_single=True,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(
run_id_key=run_id_key),
],
initial_values=[flow_name, run_id_value],
enable_joins=True)
async def get_step_tasks(self, request):
"""
---
description: Get all tasks of specified step
tags:
- Task
parameters:
- $ref: '#/definitions/Params/Path/flow_id'
- $ref: '#/definitions/Params/Path/run_number'
- $ref: '#/definitions/Params/Path/step_name'
- $ref: '#/definitions/Params/Builtin/_page'
- $ref: '#/definitions/Params/Builtin/_limit'
- $ref: '#/definitions/Params/Builtin/_order'
- $ref: '#/definitions/Params/Builtin/_tags'
- $ref: '#/definitions/Params/Builtin/_group'
- $ref: '#/definitions/Params/Custom/flow_id'
- $ref: '#/definitions/Params/Custom/run_number'
- $ref: '#/definitions/Params/Custom/step_name'
- $ref: '#/definitions/Params/Custom/task_id'
- $ref: '#/definitions/Params/Custom/user_name'
- $ref: '#/definitions/Params/Custom/ts_epoch'
- $ref: '#/definitions/Params/Custom/finished_at'
- $ref: '#/definitions/Params/Custom/duration'
- $ref: '#/definitions/Params/Custom/postprocess'
- $ref: '#/definitions/Params/Custom/invalidate'
produces:
- application/json
responses:
"200":
description: Returns all tasks of specified step
schema:
$ref: '#/definitions/ResponsesTaskList'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
flow_name = request.match_info.get("flow_id")
run_number = request.match_info.get("run_number")
run_id_key, run_id_value = translate_run_key(run_number)
step_name = request.match_info.get("step_name")
return await find_records(
request,
self._async_table,
initial_conditions=[
"flow_id = %s",
"{run_id_key} = %s".format(run_id_key=run_id_key),
"step_name = %s"
],
initial_values=[flow_name, run_id_value, step_name],
initial_order=["attempt_id DESC"],
allowed_order=self._async_table.keys +
["finished_at", "duration", "attempt_id"],
allowed_group=self._async_table.keys,
allowed_filters=self._async_table.keys +
["finished_at", "duration", "attempt_id"],
enable_joins=True,
postprocess=postprocess_chain([
apply_run_tags_postprocess(flow_name, run_number,
self._async_run_table),
self.get_postprocessor(request)
]))