async def get_plugin_asset(self, request):
"""
---
description: Serve plugin asset
tags:
- Plugin
parameters:
- $ref: '#/definitions/Params/Path/plugin_name'
- $ref: '#/definitions/Params/Path/plugin_filename'
produces:
- application/json
responses:
"200":
description: Serve plugin asset, e.g. dist/index.html
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
plugin = _get_plugin_from_request(request)
if not plugin:
return web_response(404, "Plugin not found")
filename = request.match_info.get("filename")
try:
return plugin.serve(filename)
except:
return web_response(500, "Internal server error")
async def get_plugin(self, request):
"""
---
description: Get one plugin
tags:
- Plugin
parameters:
- $ref: '#/definitions/Params/Path/plugin_name'
produces:
- application/json
responses:
"200":
description: Returns one plugin
schema:
$ref: '#/definitions/ResponsesPlugin'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
plugin = _get_plugin_from_request(request)
if not plugin:
return web_response(404, "Plugin not found")
return web_response(200, dict(plugin))
async def get_config(self, request):
"""
---
description: Get all frontend configuration key-value pairs.
tags:
- Admin
produces:
- application/json
responses:
"200":
description: Returns all allowed configuration key-value pairs for the frontend.
schema:
type: object
properties:
"ALLOWED_CONFIG_VARIABLE":
type: string
example: "value-to-pass-frontend-1234"
description: "A frontend configuration variable from the server environment. These are exposed based on a whitelist on the server."
"405":
description: invalid HTTP Method
"""
config = {}
for key in ALLOWED_CONFIG_KEYS:
val = os.environ.get(key, None)
if val:
config[key] = val
return web_response(200, config)
async def get_task_log(self, request, logtype=STDOUT):
"fetches log and emits it as a list of rows wrapped in json"
task = await self.get_task_by_request(request)
if not task:
return web_response(404, {'data': []})
limit, page, reverse_order = get_pagination_params(request)
lines, page_count = await read_and_output(self.cache, task, logtype,
limit, page, reverse_order)
# paginated response
response = DBResponse(200, lines)
pagination = DBPagination(limit, limit * (page - 1),
len(response.body), page)
status, body = format_response_list(request, response, pagination,
page, page_count)
return web_response(status, body)
async def find_records(request: web.BaseRequest,
async_table=None,
initial_conditions: List[str] = [],
initial_values=[],
initial_order: List[str] = [],
allowed_order: List[str] = [],
allowed_group: List[str] = [],
allowed_filters: List[str] = [],
postprocess: Callable[[DBResponse], DBResponse] = None,
fetch_single=False,
enable_joins=False,
overwrite_select_from: str = None):
page, limit, offset, order, groups, group_limit = pagination_query(
request, allowed_order=allowed_order, allowed_group=allowed_group)
builtin_conditions, builtin_vals = builtin_conditions_query(request)
custom_conditions, custom_vals = custom_conditions_query(
request, allowed_keys=allowed_filters)
conditions = initial_conditions + builtin_conditions + custom_conditions
values = initial_values + builtin_vals + custom_vals
ordering = (initial_order or []) + (order or [])
benchmark = query_param_enabled(request, "benchmark")
invalidate_cache = query_param_enabled(request, "invalidate")
results, pagination, benchmark_result = await async_table.find_records(
conditions=conditions,
values=values,
limit=limit,
offset=offset,
order=ordering if len(ordering) > 0 else None,
groups=groups,
group_limit=group_limit,
fetch_single=fetch_single,
enable_joins=enable_joins,
expanded=True,
postprocess=postprocess,
invalidate_cache=invalidate_cache,
benchmark=benchmark,
overwrite_select_from=overwrite_select_from)
if fetch_single:
status, res = format_response(request, results)
else:
status, res = format_response_list(request, results, pagination, page)
if benchmark_result:
res["benchmark_result"] = benchmark_result
return web_response(status, res)
async def links(self, request):
"""
---
description: Provides custom navigation links for UI.
tags:
- Admin
produces:
- 'application/json'
responses:
"200":
description: Returns the custom navigation links for UI
schema:
$ref: '#/definitions/ResponsesLinkList'
"405":
description: invalid HTTP Method
"""
return web_response(status=200, body=self.navigation_links)
async def get_task_log_file(self, request, logtype=STDOUT):
"fetches log and emits it as a single file download response"
task = await self.get_task_by_request(request)
if not task:
return web_response(404, {'data': []})
log_filename = "{type}_{flow_id}_{run_number}_{step_name}_{task_id}-{attempt}.txt".format(
type="stdout" if logtype == STDOUT else "stderr",
flow_id=task['flow_id'],
run_number=task['run_number'],
step_name=task['step_name'],
task_id=task['task_id'],
attempt=task['attempt_id'])
lines, _ = await read_and_output(self.cache,
task,
logtype,
output_raw=True)
return file_download_response(log_filename, lines)
async def get_all_features(self, request):
"""
---
description: Get all of enabled/disabled features as key-value pairs.
tags:
- Admin
produces:
- application/json
responses:
"200":
description: Returns all features to be enabled or disabled by the frontend.
schema:
type: object
properties:
"FEATURE_*":
type: boolean
example: true
description: "An environment variable from the server with a FEATURE_ prefix, and its value as a boolean"
"405":
description: invalid HTTP Method
"""
return web_response(200, get_features())
async def get_plugins(self, request):
"""
---
description: List all plugins
tags:
- Plugin
produces:
- application/json
responses:
"200":
description: Returns list of all plugins
schema:
$ref: '#/definitions/ResponsesPluginList'
"405":
description: invalid HTTP Method
schema:
$ref: '#/definitions/ResponsesError405'
"""
plugins = []
for plugin in list_plugins():
plugins.append(dict(plugin))
return web_response(200, plugins)
async def get_notifications(self, request):
"""
---
description: Provides System Notifications for the UI
tags:
- Admin
produces:
- 'application/json'
responses:
"200":
description: Returns list of active system notification
schema:
$ref: '#/definitions/ResponsesNotificationList'
"405":
description: invalid HTTP Method
"""
processed_notifications = []
for notification in self.notifications:
try:
if "message" not in notification:
continue
# Created at is required and "start" is used by default if not value provided
# Notification will be ignored if both "created" and "start" are missing
created = notification.get("created",
notification.get("start", None))
if not created:
continue
processed_notifications.append({
"id":
notification.get(
"id",
hashlib.sha1(
str(notification).encode('utf-8')).hexdigest()),
"type":
notification.get("type", "info"),
"contentType":
notification.get("contentType", "text"),
"message":
notification.get("message", ""),
"url":
notification.get("url", None),
"urlText":
notification.get("urlText", None),
"created":
created,
"start":
notification.get("start", None),
"end":
notification.get("end", None)
})
except:
pass
# Filter notifications based on query parameters
# Supports eq,ne.lt,le,gt,ge operators for all the fields
def filter_notifications(notification):
comp_operators = {
"eq": lambda a, b: a == b,
"ne": lambda a, b: a != b,
"lt": lambda a, b: a < b,
"le": lambda a, b: a <= b,
"gt": lambda a, b: a > b,
"ge": lambda a, b: a >= b,
}
try:
for q in request.query.keys():
if ":" in q:
field, op = q.split(":", 1)
else:
field, op = q, "eq"
# Make sure compare operator is supported, otherwise ignore
# Compare value is typecasted to match field type
if op in comp_operators:
field_val = notification.get(field, None)
if not field_val:
continue
comp_val = type(field_val)(request.query.get(q, None))
if not comp_val:
continue
if not comp_operators[op](field_val, comp_val):
return False
except:
pass
return True
return web_response(status=200,
body=list(
filter(filter_notifications,
processed_notifications)))
async def status(self, request):
"""
---
description: Display system status information, such as cache
tags:
- Admin
produces:
- 'application/json'
responses:
"200":
description: Return system status information, such as cache
"405":
description: invalid HTTP Method
"""
cache_status = {}
for store in [
self.cache_store.artifact_cache, self.cache_store.dag_cache,
self.cache_store.log_cache
]:
try:
# Use client ping to verify communcation, True = ok
await store.cache.ping()
ping = True
except Exception as ex:
ping = str(ex)
try:
# Use Check -action to verify Cache communication, True = ok
await store.cache.request_and_return([store.cache.check()],
None)
check = True
except Exception as ex:
check = str(ex)
# Extract list of worker subprocesses
worker_list = []
cache_server_pid = store.cache._proc.pid if store.cache._proc else None
if cache_server_pid:
try:
proc = await asyncio.create_subprocess_shell(
"pgrep -P {}".format(cache_server_pid),
stdout=asyncio.subprocess.PIPE)
stdout, _ = await proc.communicate()
if stdout:
pids = stdout.decode().splitlines()
proc = await asyncio.create_subprocess_shell(
"ps -p {} -o pid,%cpu,%mem,stime,time,command".
format(",".join(pids)),
stdout=asyncio.subprocess.PIPE)
stdout, _ = await proc.communicate()
worker_list = stdout.decode().splitlines()
except Exception as ex:
worker_list = str(ex)
else:
worker_list = "Unable to get cache server pid"
# Extract current cache data usage in bytes
current_size = 0
try:
cache_data_path = os.path.abspath(store.cache._root)
proc = await asyncio.create_subprocess_shell(
"du -s {} | cut -f1".format(cache_data_path),
stdout=asyncio.subprocess.PIPE)
stdout, _ = await proc.communicate()
if stdout:
current_size = int(stdout.decode())
except Exception as ex:
current_size = str(ex)
cache_status[store.__class__.__name__] = {
"restart_requested":
store.cache._restart_requested,
"is_alive":
store.cache._is_alive,
"pending_requests":
list(store.cache.pending_requests),
"root":
store.cache._root,
"prev_is_alive":
store.cache._prev_is_alive,
"action_classes":
list(map(lambda cls: cls.__name__,
store.cache._action_classes)),
"max_actions":
store.cache._max_actions,
"max_size":
store.cache._max_size,
"current_size":
current_size,
"ping":
ping,
"check_action":
check,
"proc": {
"pid": store.cache._proc.pid,
"returncode": store.cache._proc.returncode,
} if store.cache._proc else None,
"workers":
worker_list
}
return web_response(status=200, body={"cache": cache_status})
async def get_all_tags(self, request):
db_response, _ = await self._async_table.get_tags()
return web_response(db_response.response_code, db_response.body)