def handle_experiment_job_log(experiment_name, experiment_uuid, log_lines): if not Experiment.objects.filter(uuid=experiment_uuid).exists(): return logger.debug('handling log event for %s', experiment_uuid) safe_log_experiment(experiment_name=experiment_name, log_lines=log_lines, temp=False, append=True)
def handle_experiment_job_log(experiment_name: str, experiment_uuid: str, log_lines: Optional[Union[str, Iterable[str]]], temp: bool = True) -> None: if not Experiment.objects.filter(uuid=experiment_uuid).exists(): return logger.debug('handling log event for %s', experiment_uuid) safe_log_experiment(experiment_name=experiment_name, log_lines=log_lines, temp=temp, append=True)
def process_logs(experiment, temp=True): pod_id = EXPERIMENT_JOB_NAME_FORMAT.format( task_type=TaskType.MASTER, # We default to master task_idx=0, experiment_uuid=experiment.uuid.hex) k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE, in_cluster=True) log_lines = base.process_logs( k8s_manager=k8s_manager, pod_id=pod_id, container_job_name=settings.CONTAINER_NAME_EXPERIMENT_JOB) safe_log_experiment(experiment_name=experiment.unique_name, log_lines=log_lines, temp=temp, append=False)
def process_logs(experiment: 'Experiment', temp: bool = True) -> None: pod_id = EXPERIMENT_JOB_NAME_FORMAT.format( task_type=experiment.default_job_role, task_idx=0, experiment_uuid=experiment.uuid.hex) k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'), in_cluster=True) container_job_name = get_experiment_job_container_name(backend=experiment.backend, framework=experiment.framework) log_lines = base.process_logs(k8s_manager=k8s_manager, pod_id=pod_id, container_job_name=container_job_name) safe_log_experiment(experiment_name=experiment.unique_name, log_lines=log_lines, temp=temp, append=False)