def test_experiment_logs_path_creation_deletion(self): experiment_logs_path = get_experiment_logs_path( self.experiment.unique_name) filepath = get_experiment_logs_path(self.experiment.unique_name) open(filepath, '+w') # Should be true, created by the signal assert os.path.exists(experiment_logs_path) is True assert os.path.exists(filepath) is True delete_experiment_logs(self.experiment.unique_name) assert os.path.exists(filepath) is False
def test_handle_events_job_logs_create_one_handler(self): with patch( 'experiments.tasks.build_experiment.apply_async') as mock_fct: experiment = ExperimentFactory() params = dict(experiment_name=experiment.unique_name, experiment_uuid=uuid.uuid4().hex, job_uuid=uuid.uuid4().hex, log_line='First test', persist=True, task_type=TaskType.MASTER, task_idx=0) handle_events_job_logs(**params) # Check new log path is created log_path = get_experiment_logs_path(experiment.unique_name) assert os.path.exists(log_path) is True # Check the logger has a file handler xp_logger = logging.getLogger(experiment.unique_name) assert len(xp_logger.handlers) == 1 assert self.file_line_count(log_path) == 1 # Calling again the task should not reuse handler, and create a new line handle_events_job_logs(**params) # Check the logger has a file handler xp_logger = logging.getLogger(experiment.unique_name) assert len(xp_logger.handlers) == 1 assert self.file_line_count(log_path) == 2
def test_project_logs_path_creation_deletion(self): with patch('experiments.tasks.build_experiment.apply_async') as _: experiment = ExperimentFactory(user=self.project.user, project=self.project) experiment_logs_path = get_experiment_logs_path(experiment.unique_name) project_logs_path = get_project_logs_path(self.project.unique_name) # Should be true, created by the signal assert os.path.exists(experiment_logs_path) is True assert os.path.exists(project_logs_path) is True delete_project_logs(self.project.unique_name) assert os.path.exists(experiment_logs_path) is False assert os.path.exists(project_logs_path) is False
def handle_events_job_logs(experiment_name, experiment_uuid, job_uuid, log_line, persist, task_type=None, task_idx=None): # must persist resources if logs according to the config logger.debug('handling log event for {} {} {}'.format( experiment_uuid, job_uuid, persist)) if task_type and task_idx: log_line = '{}.{} -- {}'.format(task_type, int(task_idx) + 1, log_line) xp_logger = logging.getLogger('experiment_name') log_path = get_experiment_logs_path(experiment_name) log_handler = logging.FileHandler(log_path) log_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') log_handler.setFormatter(log_formatter) xp_logger.addHandler(log_handler) xp_logger.setLevel(logging.INFO) xp_logger.info(log_line)
def get_config_map(namespace, project_name, experiment_group_name, experiment_name, project_uuid, experiment_group_uuid, experiment_uuid, cluster_def, declarations, log_level): name = constants.CONFIG_MAP_NAME.format(experiment_uuid=experiment_uuid) labels = get_map_labels(project_name, experiment_group_name, experiment_name, project_uuid, experiment_group_uuid, experiment_uuid) metadata = client.V1ObjectMeta(name=name, labels=labels, namespace=namespace) experiment_outputs_path = get_experiment_outputs_path(experiment_name) experiment_logs_path = get_experiment_logs_path(experiment_name) experiment_data_path = get_project_data_path(project_name) data = { constants.CONFIG_MAP_CLUSTER_KEY_NAME: json.dumps(cluster_def), constants.CONFIG_MAP_DECLARATIONS_KEY_NAME: json.dumps(declarations) or '{}', constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME: json.dumps(labels), constants.CONFIG_MAP_LOG_LEVEL_KEY_NAME: log_level, constants.CONFIG_MAP_API_KEY_NAME: 'http://{}:{}'.format(settings.POLYAXON_K8S_API_HOST, settings.POLYAXON_K8S_API_PORT), constants.CONFIG_MAP_EXPERIMENT_OUTPUTS_PATH_KEY_NAME: experiment_outputs_path, constants.CONFIG_MAP_EXPERIMENT_LOGS_PATH_KEY_NAME: experiment_logs_path, constants.CONFIG_MAP_EXPERIMENT_DATA_PATH_KEY_NAME: experiment_data_path, } return client.V1ConfigMap(api_version=k8s_constants.K8S_API_VERSION_V1, kind=k8s_constants.K8S_CONFIG_MAP_KIND, metadata=metadata, data=data)