def test_experiment_logs_path_creation_deletion(self):
experiment_logs_path = get_experiment_logs_path(
self.experiment.unique_name)
filepath = get_experiment_logs_path(self.experiment.unique_name)
open(filepath, '+w')
# Should be true, created by the signal
assert os.path.exists(experiment_logs_path) is True
assert os.path.exists(filepath) is True
delete_experiment_logs(self.experiment.unique_name)
assert os.path.exists(filepath) is False
def test_handle_events_job_logs_create_one_handler(self):
with patch(
'experiments.tasks.build_experiment.apply_async') as mock_fct:
experiment = ExperimentFactory()
params = dict(experiment_name=experiment.unique_name,
experiment_uuid=uuid.uuid4().hex,
job_uuid=uuid.uuid4().hex,
log_line='First test',
persist=True,
task_type=TaskType.MASTER,
task_idx=0)
handle_events_job_logs(**params)
# Check new log path is created
log_path = get_experiment_logs_path(experiment.unique_name)
assert os.path.exists(log_path) is True
# Check the logger has a file handler
xp_logger = logging.getLogger(experiment.unique_name)
assert len(xp_logger.handlers) == 1
assert self.file_line_count(log_path) == 1
# Calling again the task should not reuse handler, and create a new line
handle_events_job_logs(**params)
# Check the logger has a file handler
xp_logger = logging.getLogger(experiment.unique_name)
assert len(xp_logger.handlers) == 1
assert self.file_line_count(log_path) == 2
def test_project_logs_path_creation_deletion(self):
with patch('experiments.tasks.build_experiment.apply_async') as _:
experiment = ExperimentFactory(user=self.project.user, project=self.project)
experiment_logs_path = get_experiment_logs_path(experiment.unique_name)
project_logs_path = get_project_logs_path(self.project.unique_name)
# Should be true, created by the signal
assert os.path.exists(experiment_logs_path) is True
assert os.path.exists(project_logs_path) is True
delete_project_logs(self.project.unique_name)
assert os.path.exists(experiment_logs_path) is False
assert os.path.exists(project_logs_path) is False
def handle_events_job_logs(experiment_name,
experiment_uuid,
job_uuid,
log_line,
persist,
task_type=None,
task_idx=None):
# must persist resources if logs according to the config
logger.debug('handling log event for {} {} {}'.format(
experiment_uuid, job_uuid, persist))
if task_type and task_idx:
log_line = '{}.{} -- {}'.format(task_type, int(task_idx) + 1, log_line)
xp_logger = logging.getLogger('experiment_name')
log_path = get_experiment_logs_path(experiment_name)
log_handler = logging.FileHandler(log_path)
log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
log_handler.setFormatter(log_formatter)
xp_logger.addHandler(log_handler)
xp_logger.setLevel(logging.INFO)
xp_logger.info(log_line)
def get_config_map(namespace,
project_name,
experiment_group_name,
experiment_name,
project_uuid,
experiment_group_uuid,
experiment_uuid,
cluster_def,
declarations,
log_level):
name = constants.CONFIG_MAP_NAME.format(experiment_uuid=experiment_uuid)
labels = get_map_labels(project_name,
experiment_group_name,
experiment_name,
project_uuid,
experiment_group_uuid,
experiment_uuid)
metadata = client.V1ObjectMeta(name=name, labels=labels, namespace=namespace)
experiment_outputs_path = get_experiment_outputs_path(experiment_name)
experiment_logs_path = get_experiment_logs_path(experiment_name)
experiment_data_path = get_project_data_path(project_name)
data = {
constants.CONFIG_MAP_CLUSTER_KEY_NAME: json.dumps(cluster_def),
constants.CONFIG_MAP_DECLARATIONS_KEY_NAME: json.dumps(declarations) or '{}',
constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME: json.dumps(labels),
constants.CONFIG_MAP_LOG_LEVEL_KEY_NAME: log_level,
constants.CONFIG_MAP_API_KEY_NAME: 'http://{}:{}'.format(settings.POLYAXON_K8S_API_HOST,
settings.POLYAXON_K8S_API_PORT),
constants.CONFIG_MAP_EXPERIMENT_OUTPUTS_PATH_KEY_NAME: experiment_outputs_path,
constants.CONFIG_MAP_EXPERIMENT_LOGS_PATH_KEY_NAME: experiment_logs_path,
constants.CONFIG_MAP_EXPERIMENT_DATA_PATH_KEY_NAME: experiment_data_path,
}
return client.V1ConfigMap(api_version=k8s_constants.K8S_API_VERSION_V1,
kind=k8s_constants.K8S_CONFIG_MAP_KIND,
metadata=metadata,
data=data)
