def stream_logs(experiment):
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=TaskType.MASTER, # We default to master
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE, in_cluster=True)
return base.stream_logs(
k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=settings.CONTAINER_NAME_EXPERIMENT_JOB)
def stream_logs(experiment: 'Experiment') -> Iterable[str]:
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=TaskType.MASTER, # We default to master
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'),
in_cluster=True)
return base.stream_logs(
k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=conf.get('CONTAINER_NAME_EXPERIMENT_JOB'))
def stream_logs(experiment: 'Experiment') -> Iterable[str]:
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=experiment.default_job_role,
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'), in_cluster=True)
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
return base.stream_logs(k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=container_job_name)
def process_logs(experiment, temp=True):
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=TaskType.MASTER, # We default to master
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE, in_cluster=True)
log_lines = base.process_logs(
k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=settings.CONTAINER_NAME_EXPERIMENT_JOB)
safe_log_experiment(experiment_name=experiment.unique_name,
log_lines=log_lines,
temp=temp,
append=False)
def process_logs(experiment: 'Experiment', temp: bool = True) -> None:
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=experiment.default_job_role,
task_idx=0,
experiment_uuid=experiment.uuid.hex)
k8s_manager = K8SManager(namespace=conf.get('K8S_NAMESPACE'), in_cluster=True)
container_job_name = get_experiment_job_container_name(backend=experiment.backend,
framework=experiment.framework)
log_lines = base.process_logs(k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=container_job_name)
safe_log_experiment(experiment_name=experiment.unique_name,
log_lines=log_lines,
temp=temp,
append=False)
def process_logs(experiment_job, temp=True, k8s_manager=None):
task_type = experiment_job.role
task_id = experiment_job.sequence
pod_id = EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=task_type, # We default to master
task_idx=task_id,
experiment_uuid=experiment_job.experiment.uuid.hex)
if not k8s_manager:
k8s_manager = K8SManager(namespace=settings.K8S_NAMESPACE, in_cluster=True)
log_lines = base.process_logs(k8s_manager=k8s_manager,
pod_id=pod_id,
container_job_name=settings.CONTAINER_NAME_EXPERIMENT_JOB,
task_type=task_type,
task_idx=task_id)
safe_log_experiment_job(experiment_job_name=experiment_job.unique_name,
log_lines=log_lines,
temp=temp,
append=False)
def get_resource_name(self, task_type, task_idx): # pylint:disable=arguments-differ
return EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=task_type,
task_idx=task_idx,
experiment_uuid=self.experiment_uuid)
def pod_id(self) -> str:
return EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=self.role,
task_idx=self.sequence,
experiment_uuid=self.experiment.uuid.hex)
def get_job_name(self, task_type, task_idx):
return EXPERIMENT_JOB_NAME_FORMAT.format(
task_type=task_type,
task_idx=task_idx,
experiment_uuid=self.experiment_uuid)
