async def experiment_job_logs_v2(request, ws, username, project_name,
experiment_id, job_id):
job, experiment, message = validate_experiment_job(
request=request,
username=username,
project_name=project_name,
experiment_id=experiment_id,
job_id=job_id)
if job is None:
await ws.send(get_error_message(message))
return
pod_id = job.pod_id
container_job_name = get_experiment_job_container_name(
backend=experiment.backend, framework=experiment.framework)
auditor.record(event_type=EXPERIMENT_JOB_LOGS_VIEWED,
instance=job,
actor_id=request.app.user.id,
actor_name=request.app.user.username)
# Stream logs
await log_job(request=request,
ws=ws,
job=job,
pod_id=pod_id,
container=container_job_name,
namespace=conf.get('K8S_NAMESPACE'))
def test_get_experiment_job_container_name(self):
assert conf.get('CONTAINER_NAME_EXPERIMENT_JOB') == get_experiment_job_container_name(
backend=None,
framework=None,
)
assert conf.get('CONTAINER_NAME_EXPERIMENT_JOB') == get_experiment_job_container_name(
backend=None,
framework='foo',
)
assert conf.get('CONTAINER_NAME_EXPERIMENT_JOB') == get_experiment_job_container_name(
backend='foo',
framework='foo',
)
assert conf.get('CONTAINER_NAME_EXPERIMENT_JOB') == get_experiment_job_container_name(
backend='foo',
framework=None,
)
assert conf.get('CONTAINER_NAME_EXPERIMENT_JOB') == get_experiment_job_container_name(
backend=ExperimentBackend.KUBEFLOW,
framework='foo',
)
assert conf.get('CONTAINER_NAME_TF_JOB') == get_experiment_job_container_name(
backend=ExperimentBackend.KUBEFLOW,
framework=ExperimentFramework.TENSORFLOW,
)
assert conf.get('CONTAINER_NAME_PYTORCH_JOB') == get_experiment_job_container_name(
backend=ExperimentBackend.KUBEFLOW,
framework=ExperimentFramework.PYTORCH,
)
def stream_logs(experiment: 'Experiment') -> Iterable[str]:
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=experiment.default_job_role,
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'), in_cluster=True)
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
return base.stream_logs(k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=container_job_name)
async def experiment_logs_v2(request, ws, username, project_name, experiment_id):
experiment, message = get_experiment(experiment_id=experiment_id)
if experiment is None:
await ws.send(get_error_message(message))
return
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
# Stream logs
await log_experiment(request=request,
ws=ws,
experiment=experiment,
container=container_job_name,
namespace=conf.get(K8S_NAMESPACE))
def process_logs(experiment: 'Experiment', temp: bool = True) -> None:
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=experiment.default_job_role,
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'), in_cluster=True)
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
log_lines = base.process_logs(k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=container_job_name)
safe_log_experiment(experiment_name=experiment.unique_name,
log_lines=log_lines,
temp=temp,
append=False)
def process_logs(experiment_job: 'ExperimentJob',
temp: bool = True,
k8s_manager: 'K8SManager' = None) -> None:
task_type = experiment_job.role
task_id = experiment_job.sequence
if not k8s_manager:
k8s_manager = K8SManager(namespace=conf.get(K8S_NAMESPACE), in_cluster=True)
container_job_name = get_experiment_job_container_name(
backend=experiment_job.experiment.backend,
framework=experiment_job.experiment.framework)
log_lines = base.process_logs(k8s_manager=k8s_manager,
pod_id=experiment_job.pod_id,
container_job_name=container_job_name,
task_type=task_type,
task_idx=task_id)
safe_log_experiment_job(experiment_job_name=experiment_job.unique_name,
log_lines=log_lines,
temp=temp,
append=False)
async def experiment_logs_v2(request, ws, username, project_name, experiment_id):
experiment, message = validate_experiment(request=request,
username=username,
project_name=project_name,
experiment_id=experiment_id)
if experiment is None:
await ws.send(get_error_message(message))
return
auditor.record(event_type=EXPERIMENT_LOGS_VIEWED,
instance=experiment,
actor_id=request.app.user.id,
actor_name=request.app.user.username)
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
# Stream logs
await log_experiment(request=request,
ws=ws,
experiment=experiment,
container=container_job_name,
namespace=conf.get(K8S_NAMESPACE))