def test_experiment_outputs_path_creation_deletion(self):
experiment_outputs_path = get_experiment_outputs_path(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is False
create_experiment_outputs_path(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is True
delete_experiment_outputs(self.experiment.unique_name)
assert os.path.exists(experiment_outputs_path) is False
def test_copying_an_experiment(self):
with patch('runner.tasks.experiments.build_experiment.apply_async'
) as _: # noqa
experiment1 = ExperimentFactory()
# We create some outputs files for the experiment
path = create_experiment_outputs_path(experiment1.unique_name)
open(os.path.join(path, 'file'), 'w+')
# Create a new experiment that is a clone of the previous
with patch('runner.tasks.experiments.build_experiment.apply_async'
) as _: # noqa
experiment2 = ExperimentFactory(original_experiment=experiment1)
# Check that outputs path for experiment2 does not exist yet
experiment2_outputs_path = get_experiment_outputs_path(
experiment2.unique_name)
assert os.path.exists(experiment2_outputs_path) is False
# Handle restart should create the outputs and copy the content of experiment 1
copy_experiment(experiment2)
assert os.path.exists(experiment2_outputs_path) is True
assert os.path.exists(os.path.join(experiment2_outputs_path,
'file')) is True
def get_env_vars(self, task_type, task_idx):
tf_config = {
'cluster': self.get_cluster(),
'task': {'type': task_type, 'index': task_idx},
'model_dir': get_experiment_outputs_path(self.experiment_name),
'environment': 'cloud'
}
return get_env_var(name='TF_CONFIG', value=tf_config)
def test_project_outputs_path_creation_deletion(self):
with patch('experiments.tasks.build_experiment.apply_async') as _:
experiment = ExperimentFactory(user=self.project.user, project=self.project)
create_experiment_outputs_path(experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(experiment.unique_name)
project_outputs_path = get_project_outputs_path(self.project.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(project_outputs_path) is True
delete_project_outputs(self.project.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(project_outputs_path) is False
def test_experiment_group_outputs_path_creation_deletion(self):
experiment = ExperimentFactory(user=self.project.user,
project=self.project,
experiment_group=self.experiment_group)
create_experiment_outputs_path(experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(
experiment.unique_name)
experiment_group_outputs_path = get_experiment_group_outputs_path(
self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(experiment_group_outputs_path) is True
delete_experiment_group_outputs(self.experiment_group.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(experiment_group_outputs_path) is False
def get_config_map(namespace, project_name, experiment_group_name,
experiment_name, project_uuid, experiment_group_uuid,
experiment_uuid, original_name, cloning_strategy,
cluster_def, declarations, log_level):
name = constants.CONFIG_MAP_NAME.format(experiment_uuid=experiment_uuid)
labels = get_map_labels(project_name, experiment_group_name,
experiment_name, project_uuid,
experiment_group_uuid, experiment_uuid)
metadata = client.V1ObjectMeta(name=name,
labels=labels,
namespace=namespace)
experiment_outputs_path = get_experiment_outputs_path(
experiment_name=experiment_name,
original_name=original_name,
cloning_strategy=cloning_strategy)
experiment_logs_path = get_experiment_logs_path(experiment_name)
experiment_data_path = get_project_data_path(project_name)
data = {
constants.CONFIG_MAP_CLUSTER_KEY_NAME:
json.dumps(cluster_def),
constants.CONFIG_MAP_DECLARATIONS_KEY_NAME:
json.dumps(declarations) or '{}',
constants.CONFIG_MAP_EXPERIMENT_INFO_KEY_NAME:
json.dumps(labels),
constants.CONFIG_MAP_LOG_LEVEL_KEY_NAME:
log_level,
constants.CONFIG_MAP_API_KEY_NAME:
'http://{}:{}'.format(settings.POLYAXON_K8S_API_HOST,
settings.POLYAXON_K8S_API_PORT),
constants.CONFIG_MAP_EXPERIMENT_OUTPUTS_PATH_KEY_NAME:
experiment_outputs_path,
constants.CONFIG_MAP_EXPERIMENT_LOGS_PATH_KEY_NAME:
experiment_logs_path,
constants.CONFIG_MAP_EXPERIMENT_DATA_PATH_KEY_NAME:
experiment_data_path,
}
return client.V1ConfigMap(api_version=k8s_constants.K8S_API_VERSION_V1,
kind=k8s_constants.K8S_CONFIG_MAP_KIND,
metadata=metadata,
data=data)