async def collect_logs(request): owner = request.path_params["owner"] project = request.path_params["project"] run_uuid = request.path_params["run_uuid"] resource_name = get_resource_name(run_uuid=run_uuid) operation = get_run_instance(owner=owner, project=project, run_uuid=run_uuid) k8s_manager = AsyncK8SManager( namespace=settings.CLIENT_CONFIG.namespace, in_cluster=settings.CLIENT_CONFIG.in_cluster, ) await k8s_manager.setup() k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager, resource_name=resource_name) if not k8s_operation: raise HTTPException( detail="Run's logs was not collected, resource was not found.", status_code=status.HTTP_400_BAD_REQUEST, ) operation_logs, _ = await get_k8s_operation_logs(operation=operation, k8s_manager=k8s_manager, last_time=None) if k8s_manager: await k8s_manager.close() if not operation_logs: return Response() logs = operation_logs task = BackgroundTask(upload_logs, run_uuid=run_uuid, logs=logs) return Response(background=task)
async def collect_logs(request): run_uuid = request.path_params["run_uuid"] resource_name = get_resource_name(run_uuid=run_uuid) k8s_manager = AsyncK8SManager( namespace=settings.CLIENT_CONFIG.namespace, in_cluster=settings.CLIENT_CONFIG.in_cluster, ) await k8s_manager.setup() k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager, resource_name=resource_name) if not k8s_operation: raise HTTPException( detail="Run's logs was not collected, resource was not found.", status_code=status.HTTP_400_BAD_REQUEST, ) operation_logs, _ = await query_k8s_operation_logs(instance=run_uuid, k8s_manager=k8s_manager, last_time=None) if k8s_manager: await k8s_manager.close() if not operation_logs: return Response() try: await upload_logs(run_uuid=run_uuid, logs=operation_logs) except Exception as e: raise HTTPException( detail= "Run's logs was not collected, an error was raised while uploading the data %s." % e, status_code=status.HTTP_400_BAD_REQUEST, ) task = BackgroundTask(clean_tmp_logs, run_uuid=run_uuid) return Response(background=task)
def create_run(): click.echo("Creating a run.") try: compiled_operation = OperationSpecification.compile_operation( op_spec) run_name = compiled_operation.name or name resource = compiler.make( owner_name=owner, project_name=project_name, project_uuid=project_name, run_uuid=run_name, run_name=name, run_path=run_name, compiled_operation=compiled_operation, params=op_spec.params, default_sa=settings.AGENT_CONFIG.runs_sa, ) Spawner(namespace=settings.AGENT_CONFIG.namespace).create( run_uuid=run_name, run_kind=compiled_operation.get_run_kind(), resource=resource, ) # cache.cache(config_manager=RunConfigManager, response=response) run_job_uid = get_resource_name(run_name) Printer.print_success( "A new run `{}` was created".format(run_job_uid)) except (PolyaxonCompilerError, PolyaxonK8SError, PolypodException) as e: handle_cli_error(e, message="Could not create a run.") sys.exit(1)
async def get(self, run_uuid: str, run_kind: str): mixin = self._get_mixin_for_kind(kind=run_kind) resource_name = get_resource_name(run_uuid) return await self.k8s_manager.get_custom_object( name=resource_name, group=mixin.GROUP, version=mixin.API_VERSION, plural=mixin.PLURAL, )
def apply(self, run_uuid: str, run_kind: str, resource: Dict) -> Dict: mixin = self._get_mixin_for_kind(kind=run_kind) resource_name = get_resource_name(run_uuid) return self.k8s_manager.update_custom_object( name=resource_name, group=mixin.GROUP, version=mixin.API_VERSION, plural=mixin.PLURAL, body=resource, )
async def get_logs(request): owner = request.path_params["owner"] project = request.path_params["project"] run_uuid = request.path_params["run_uuid"] force = to_bool(request.query_params.get("force"), handle_none=True) resource_name = get_resource_name(run_uuid=run_uuid) operation = get_run_instance(owner=owner, project=project, run_uuid=run_uuid) last_time = QueryParams(request.url.query).get("last_time") if last_time: last_time = dt_parser.parse(last_time).astimezone() last_file = QueryParams(request.url.query).get("last_file") k8s_manager = None k8s_operation = None if not last_file: k8s_manager = AsyncK8SManager( namespace=settings.CLIENT_CONFIG.namespace, in_cluster=settings.CLIENT_CONFIG.in_cluster, ) await k8s_manager.setup() k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager, resource_name=resource_name) if not last_file and k8s_operation: last_file = None operation_logs, last_time = await get_k8s_operation_logs( operation=operation, last_time=last_time, k8s_manager=k8s_manager, stream=True, ) if k8s_operation["status"].get("completionTime"): last_time = None elif last_time: # Streaming should stop last_file = None last_time = None operation_logs = [] else: last_time = None operation_logs, last_file = await get_archived_operation_logs( run_uuid=run_uuid, last_file=last_file, check_cache=not force) if k8s_manager: await k8s_manager.close() response = V1Logs(last_time=last_time, last_file=last_file, logs=operation_logs) return UJSONResponse(response.to_dict())
async def get_logs(request): run_uuid = request.path_params["run_uuid"] force = to_bool(request.query_params.get("force"), handle_none=True) last_time = QueryParams(request.url.query).get("last_time") if last_time: last_time = dt_parser.parse(last_time).astimezone() last_file = QueryParams(request.url.query).get("last_file") files = [] if last_time: resource_name = get_resource_name(run_uuid=run_uuid) k8s_manager = AsyncK8SManager( namespace=settings.CLIENT_CONFIG.namespace, in_cluster=settings.CLIENT_CONFIG.in_cluster, ) await k8s_manager.setup() k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager, resource_name=resource_name) if k8s_operation: operation_logs, last_time = await get_operation_logs( k8s_manager=k8s_manager, k8s_operation=k8s_operation, instance=run_uuid, last_time=last_time, ) else: operation_logs, last_time = await get_tmp_operation_logs( run_uuid=run_uuid, last_time=last_time) if k8s_manager: await k8s_manager.close() else: operation_logs, last_file, files = await get_archived_operation_logs( run_uuid=run_uuid, last_file=last_file, check_cache=not force) response = V1Logs(last_time=last_time, last_file=last_file, logs=operation_logs, files=files) return UJSONResponse(response.to_dict())
def get_resource_name(self): return get_resource_name(self.run_uuid)