async def collect_logs(request):
owner = request.path_params["owner"]
project = request.path_params["project"]
run_uuid = request.path_params["run_uuid"]
resource_name = get_resource_name(run_uuid=run_uuid)
operation = get_run_instance(owner=owner,
project=project,
run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if not k8s_operation:
raise HTTPException(
detail="Run's logs was not collected, resource was not found.",
status_code=status.HTTP_400_BAD_REQUEST,
)
operation_logs, _ = await get_k8s_operation_logs(operation=operation,
k8s_manager=k8s_manager,
last_time=None)
if k8s_manager:
await k8s_manager.close()
if not operation_logs:
return Response()
logs = operation_logs
task = BackgroundTask(upload_logs, run_uuid=run_uuid, logs=logs)
return Response(background=task)
async def collect_logs(request):
run_uuid = request.path_params["run_uuid"]
resource_name = get_resource_name(run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if not k8s_operation:
raise HTTPException(
detail="Run's logs was not collected, resource was not found.",
status_code=status.HTTP_400_BAD_REQUEST,
)
operation_logs, _ = await query_k8s_operation_logs(instance=run_uuid,
k8s_manager=k8s_manager,
last_time=None)
if k8s_manager:
await k8s_manager.close()
if not operation_logs:
return Response()
try:
await upload_logs(run_uuid=run_uuid, logs=operation_logs)
except Exception as e:
raise HTTPException(
detail=
"Run's logs was not collected, an error was raised while uploading the data %s."
% e,
status_code=status.HTTP_400_BAD_REQUEST,
)
task = BackgroundTask(clean_tmp_logs, run_uuid=run_uuid)
return Response(background=task)
def create_run():
click.echo("Creating a run.")
try:
compiled_operation = OperationSpecification.compile_operation(
op_spec)
run_name = compiled_operation.name or name
resource = compiler.make(
owner_name=owner,
project_name=project_name,
project_uuid=project_name,
run_uuid=run_name,
run_name=name,
run_path=run_name,
compiled_operation=compiled_operation,
params=op_spec.params,
default_sa=settings.AGENT_CONFIG.runs_sa,
)
Spawner(namespace=settings.AGENT_CONFIG.namespace).create(
run_uuid=run_name,
run_kind=compiled_operation.get_run_kind(),
resource=resource,
)
# cache.cache(config_manager=RunConfigManager, response=response)
run_job_uid = get_resource_name(run_name)
Printer.print_success(
"A new run `{}` was created".format(run_job_uid))
except (PolyaxonCompilerError, PolyaxonK8SError,
PolypodException) as e:
handle_cli_error(e, message="Could not create a run.")
sys.exit(1)
async def get(self, run_uuid: str, run_kind: str):
mixin = self._get_mixin_for_kind(kind=run_kind)
resource_name = get_resource_name(run_uuid)
return await self.k8s_manager.get_custom_object(
name=resource_name,
group=mixin.GROUP,
version=mixin.API_VERSION,
plural=mixin.PLURAL,
)
def apply(self, run_uuid: str, run_kind: str, resource: Dict) -> Dict:
mixin = self._get_mixin_for_kind(kind=run_kind)
resource_name = get_resource_name(run_uuid)
return self.k8s_manager.update_custom_object(
name=resource_name,
group=mixin.GROUP,
version=mixin.API_VERSION,
plural=mixin.PLURAL,
body=resource,
)
async def get_logs(request):
owner = request.path_params["owner"]
project = request.path_params["project"]
run_uuid = request.path_params["run_uuid"]
force = to_bool(request.query_params.get("force"), handle_none=True)
resource_name = get_resource_name(run_uuid=run_uuid)
operation = get_run_instance(owner=owner,
project=project,
run_uuid=run_uuid)
last_time = QueryParams(request.url.query).get("last_time")
if last_time:
last_time = dt_parser.parse(last_time).astimezone()
last_file = QueryParams(request.url.query).get("last_file")
k8s_manager = None
k8s_operation = None
if not last_file:
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if not last_file and k8s_operation:
last_file = None
operation_logs, last_time = await get_k8s_operation_logs(
operation=operation,
last_time=last_time,
k8s_manager=k8s_manager,
stream=True,
)
if k8s_operation["status"].get("completionTime"):
last_time = None
elif last_time: # Streaming should stop
last_file = None
last_time = None
operation_logs = []
else:
last_time = None
operation_logs, last_file = await get_archived_operation_logs(
run_uuid=run_uuid, last_file=last_file, check_cache=not force)
if k8s_manager:
await k8s_manager.close()
response = V1Logs(last_time=last_time,
last_file=last_file,
logs=operation_logs)
return UJSONResponse(response.to_dict())
async def get_logs(request):
run_uuid = request.path_params["run_uuid"]
force = to_bool(request.query_params.get("force"), handle_none=True)
last_time = QueryParams(request.url.query).get("last_time")
if last_time:
last_time = dt_parser.parse(last_time).astimezone()
last_file = QueryParams(request.url.query).get("last_file")
files = []
if last_time:
resource_name = get_resource_name(run_uuid=run_uuid)
k8s_manager = AsyncK8SManager(
namespace=settings.CLIENT_CONFIG.namespace,
in_cluster=settings.CLIENT_CONFIG.in_cluster,
)
await k8s_manager.setup()
k8s_operation = await get_k8s_operation(k8s_manager=k8s_manager,
resource_name=resource_name)
if k8s_operation:
operation_logs, last_time = await get_operation_logs(
k8s_manager=k8s_manager,
k8s_operation=k8s_operation,
instance=run_uuid,
last_time=last_time,
)
else:
operation_logs, last_time = await get_tmp_operation_logs(
run_uuid=run_uuid, last_time=last_time)
if k8s_manager:
await k8s_manager.close()
else:
operation_logs, last_file, files = await get_archived_operation_logs(
run_uuid=run_uuid, last_file=last_file, check_cache=not force)
response = V1Logs(last_time=last_time,
last_file=last_file,
logs=operation_logs,
files=files)
return UJSONResponse(response.to_dict())
def get_resource_name(self):
return get_resource_name(self.run_uuid)
