async def get_plugin_asset(self, request): """ --- description: Serve plugin asset tags: - Plugin parameters: - $ref: '#/definitions/Params/Path/plugin_name' - $ref: '#/definitions/Params/Path/plugin_filename' produces: - application/json responses: "200": description: Serve plugin asset, e.g. dist/index.html "405": description: invalid HTTP Method schema: $ref: '#/definitions/ResponsesError405' """ plugin = _get_plugin_from_request(request) if not plugin: return web_response(404, "Plugin not found") filename = request.match_info.get("filename") try: return plugin.serve(filename) except: return web_response(500, "Internal server error")
async def get_plugin(self, request): """ --- description: Get one plugin tags: - Plugin parameters: - $ref: '#/definitions/Params/Path/plugin_name' produces: - application/json responses: "200": description: Returns one plugin schema: $ref: '#/definitions/ResponsesPlugin' "405": description: invalid HTTP Method schema: $ref: '#/definitions/ResponsesError405' """ plugin = _get_plugin_from_request(request) if not plugin: return web_response(404, "Plugin not found") return web_response(200, dict(plugin))
async def get_config(self, request): """ --- description: Get all frontend configuration key-value pairs. tags: - Admin produces: - application/json responses: "200": description: Returns all allowed configuration key-value pairs for the frontend. schema: type: object properties: "ALLOWED_CONFIG_VARIABLE": type: string example: "value-to-pass-frontend-1234" description: "A frontend configuration variable from the server environment. These are exposed based on a whitelist on the server." "405": description: invalid HTTP Method """ config = {} for key in ALLOWED_CONFIG_KEYS: val = os.environ.get(key, None) if val: config[key] = val return web_response(200, config)
async def get_task_log(self, request, logtype=STDOUT): "fetches log and emits it as a list of rows wrapped in json" task = await self.get_task_by_request(request) if not task: return web_response(404, {'data': []}) limit, page, reverse_order = get_pagination_params(request) lines, page_count = await read_and_output(self.cache, task, logtype, limit, page, reverse_order) # paginated response response = DBResponse(200, lines) pagination = DBPagination(limit, limit * (page - 1), len(response.body), page) status, body = format_response_list(request, response, pagination, page, page_count) return web_response(status, body)
async def find_records(request: web.BaseRequest, async_table=None, initial_conditions: List[str] = [], initial_values=[], initial_order: List[str] = [], allowed_order: List[str] = [], allowed_group: List[str] = [], allowed_filters: List[str] = [], postprocess: Callable[[DBResponse], DBResponse] = None, fetch_single=False, enable_joins=False, overwrite_select_from: str = None): page, limit, offset, order, groups, group_limit = pagination_query( request, allowed_order=allowed_order, allowed_group=allowed_group) builtin_conditions, builtin_vals = builtin_conditions_query(request) custom_conditions, custom_vals = custom_conditions_query( request, allowed_keys=allowed_filters) conditions = initial_conditions + builtin_conditions + custom_conditions values = initial_values + builtin_vals + custom_vals ordering = (initial_order or []) + (order or []) benchmark = query_param_enabled(request, "benchmark") invalidate_cache = query_param_enabled(request, "invalidate") results, pagination, benchmark_result = await async_table.find_records( conditions=conditions, values=values, limit=limit, offset=offset, order=ordering if len(ordering) > 0 else None, groups=groups, group_limit=group_limit, fetch_single=fetch_single, enable_joins=enable_joins, expanded=True, postprocess=postprocess, invalidate_cache=invalidate_cache, benchmark=benchmark, overwrite_select_from=overwrite_select_from) if fetch_single: status, res = format_response(request, results) else: status, res = format_response_list(request, results, pagination, page) if benchmark_result: res["benchmark_result"] = benchmark_result return web_response(status, res)
async def links(self, request): """ --- description: Provides custom navigation links for UI. tags: - Admin produces: - 'application/json' responses: "200": description: Returns the custom navigation links for UI schema: $ref: '#/definitions/ResponsesLinkList' "405": description: invalid HTTP Method """ return web_response(status=200, body=self.navigation_links)
async def get_task_log_file(self, request, logtype=STDOUT): "fetches log and emits it as a single file download response" task = await self.get_task_by_request(request) if not task: return web_response(404, {'data': []}) log_filename = "{type}_{flow_id}_{run_number}_{step_name}_{task_id}-{attempt}.txt".format( type="stdout" if logtype == STDOUT else "stderr", flow_id=task['flow_id'], run_number=task['run_number'], step_name=task['step_name'], task_id=task['task_id'], attempt=task['attempt_id']) lines, _ = await read_and_output(self.cache, task, logtype, output_raw=True) return file_download_response(log_filename, lines)
async def get_all_features(self, request): """ --- description: Get all of enabled/disabled features as key-value pairs. tags: - Admin produces: - application/json responses: "200": description: Returns all features to be enabled or disabled by the frontend. schema: type: object properties: "FEATURE_*": type: boolean example: true description: "An environment variable from the server with a FEATURE_ prefix, and its value as a boolean" "405": description: invalid HTTP Method """ return web_response(200, get_features())
async def get_plugins(self, request): """ --- description: List all plugins tags: - Plugin produces: - application/json responses: "200": description: Returns list of all plugins schema: $ref: '#/definitions/ResponsesPluginList' "405": description: invalid HTTP Method schema: $ref: '#/definitions/ResponsesError405' """ plugins = [] for plugin in list_plugins(): plugins.append(dict(plugin)) return web_response(200, plugins)
async def get_notifications(self, request): """ --- description: Provides System Notifications for the UI tags: - Admin produces: - 'application/json' responses: "200": description: Returns list of active system notification schema: $ref: '#/definitions/ResponsesNotificationList' "405": description: invalid HTTP Method """ processed_notifications = [] for notification in self.notifications: try: if "message" not in notification: continue # Created at is required and "start" is used by default if not value provided # Notification will be ignored if both "created" and "start" are missing created = notification.get("created", notification.get("start", None)) if not created: continue processed_notifications.append({ "id": notification.get( "id", hashlib.sha1( str(notification).encode('utf-8')).hexdigest()), "type": notification.get("type", "info"), "contentType": notification.get("contentType", "text"), "message": notification.get("message", ""), "url": notification.get("url", None), "urlText": notification.get("urlText", None), "created": created, "start": notification.get("start", None), "end": notification.get("end", None) }) except: pass # Filter notifications based on query parameters # Supports eq,ne.lt,le,gt,ge operators for all the fields def filter_notifications(notification): comp_operators = { "eq": lambda a, b: a == b, "ne": lambda a, b: a != b, "lt": lambda a, b: a < b, "le": lambda a, b: a <= b, "gt": lambda a, b: a > b, "ge": lambda a, b: a >= b, } try: for q in request.query.keys(): if ":" in q: field, op = q.split(":", 1) else: field, op = q, "eq" # Make sure compare operator is supported, otherwise ignore # Compare value is typecasted to match field type if op in comp_operators: field_val = notification.get(field, None) if not field_val: continue comp_val = type(field_val)(request.query.get(q, None)) if not comp_val: continue if not comp_operators[op](field_val, comp_val): return False except: pass return True return web_response(status=200, body=list( filter(filter_notifications, processed_notifications)))
async def status(self, request): """ --- description: Display system status information, such as cache tags: - Admin produces: - 'application/json' responses: "200": description: Return system status information, such as cache "405": description: invalid HTTP Method """ cache_status = {} for store in [ self.cache_store.artifact_cache, self.cache_store.dag_cache, self.cache_store.log_cache ]: try: # Use client ping to verify communcation, True = ok await store.cache.ping() ping = True except Exception as ex: ping = str(ex) try: # Use Check -action to verify Cache communication, True = ok await store.cache.request_and_return([store.cache.check()], None) check = True except Exception as ex: check = str(ex) # Extract list of worker subprocesses worker_list = [] cache_server_pid = store.cache._proc.pid if store.cache._proc else None if cache_server_pid: try: proc = await asyncio.create_subprocess_shell( "pgrep -P {}".format(cache_server_pid), stdout=asyncio.subprocess.PIPE) stdout, _ = await proc.communicate() if stdout: pids = stdout.decode().splitlines() proc = await asyncio.create_subprocess_shell( "ps -p {} -o pid,%cpu,%mem,stime,time,command". format(",".join(pids)), stdout=asyncio.subprocess.PIPE) stdout, _ = await proc.communicate() worker_list = stdout.decode().splitlines() except Exception as ex: worker_list = str(ex) else: worker_list = "Unable to get cache server pid" # Extract current cache data usage in bytes current_size = 0 try: cache_data_path = os.path.abspath(store.cache._root) proc = await asyncio.create_subprocess_shell( "du -s {} | cut -f1".format(cache_data_path), stdout=asyncio.subprocess.PIPE) stdout, _ = await proc.communicate() if stdout: current_size = int(stdout.decode()) except Exception as ex: current_size = str(ex) cache_status[store.__class__.__name__] = { "restart_requested": store.cache._restart_requested, "is_alive": store.cache._is_alive, "pending_requests": list(store.cache.pending_requests), "root": store.cache._root, "prev_is_alive": store.cache._prev_is_alive, "action_classes": list(map(lambda cls: cls.__name__, store.cache._action_classes)), "max_actions": store.cache._max_actions, "max_size": store.cache._max_size, "current_size": current_size, "ping": ping, "check_action": check, "proc": { "pid": store.cache._proc.pid, "returncode": store.cache._proc.returncode, } if store.cache._proc else None, "workers": worker_list } return web_response(status=200, body={"cache": cache_status})
async def get_all_tags(self, request): db_response, _ = await self._async_table.get_tags() return web_response(db_response.response_code, db_response.body)