def process_experiment_jobs_logs(experiment, temp=True):
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'),
in_cluster=True)
for experiment_job in experiment.jobs.all():
process_experiment_job_logs(experiment_job=experiment_job,
temp=temp,
k8s_manager=k8s_manager)
def logs_collect_experiment_job(experiment_job_uuid):
try:
experiment_job = ExperimentJob.objects.filter(
uuid=experiment_job_uuid).select_related('experiment').get()
experiment = experiment_job.experiment
except (ExperimentJob.DoesNotExist, Experiment.DoesNotExist):
return
if experiment.jobs.count() > 1:
process_experiment_job_logs(experiment_job=experiment_job, temp=False)
else:
process_experiment_logs(experiment=experiment, temp=False)
def get_experiment_job_logs_path(experiment: Experiment, job: ExperimentJob) -> Optional[str]:
if not job:
return None
job_name = job.unique_name
if experiment.is_done:
log_path = stores.get_experiment_job_logs_path(experiment_job_name=job_name, temp=False)
logs_path = archive_logs_file(
log_path=log_path,
namepath=job_name)
elif experiment.is_managed:
process_experiment_job_logs(experiment_job=job, temp=True)
logs_path = stores.get_experiment_job_logs_path(experiment_job_name=job_name, temp=True)
else:
logs_path = None
return logs_path
def process_experiment_jobs_logs(experiment: 'Experiment', temp: bool = True) -> None:
k8s_manager = K8SManager(namespace=conf.get(K8S_NAMESPACE), in_cluster=True)
for experiment_job in experiment.jobs.all():
process_experiment_job_logs(experiment_job=experiment_job,
temp=temp,
k8s_manager=k8s_manager)
