def test_project_outputs_path_creation_deletion(self):
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa
experiment = ExperimentFactory(user=self.project.user, project=self.project)
create_experiment_outputs_path(experiment.unique_name)
create_experiment_logs_path(experiment.unique_name)
experiment_outputs_path = get_experiment_outputs_path(experiment.unique_name)
project_outputs_path = get_project_outputs_path(self.project.unique_name)
assert os.path.exists(experiment_outputs_path) is True
assert os.path.exists(project_outputs_path) is True
delete_project_outputs(self.project.unique_name)
assert os.path.exists(experiment_outputs_path) is False
assert os.path.exists(project_outputs_path) is False
def outputs_path(self):
if self.experiment:
from libs.paths.experiments import get_experiment_outputs_path
return get_experiment_outputs_path(
experiment_name=self.experiment.unique_name,
original_name=self.experiment.original_unique_name,
cloning_strategy=self.experiment.cloning_strategy)
if self.experiment_group:
from libs.paths.experiment_groups import get_experiment_group_outputs_path
return get_experiment_group_outputs_path(
experiment_group_name=self.experiment_group.unique_name)
from libs.paths.projects import get_project_outputs_path
return get_project_outputs_path(project_name=self.project.unique_name)
def start_tensorboard(self, image, resources=None, node_selectors=None):
ports = [self.request_tensorboard_port()]
target_ports = [self.PORT]
volumes, volume_mounts = get_pod_volumes()
outputs_path = get_project_outputs_path(project_name=self.project_name)
deployment = deployments.get_deployment(
namespace=self.namespace,
app=settings.APP_LABELS_TENSORBOARD,
name=self.TENSORBOARD_JOB_NAME,
project_name=self.project_name,
project_uuid=self.project_uuid,
job_name=self.job_name,
job_uuid=self.job_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=[
"tensorboard --logdir={} --port={}".format(
outputs_path, self.PORT)
],
ports=target_ports,
container_name=settings.CONTAINER_NAME_PLUGIN_JOB,
resources=resources,
node_selector=node_selectors,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_name = constants.JOB_NAME.format(
name=self.TENSORBOARD_JOB_NAME, job_uuid=self.job_uuid)
deployment_labels = deployments.get_labels(
app=settings.APP_LABELS_TENSORBOARD,
project_name=self.project_name,
project_uuid=self.project_uuid,
job_name=self.job_name,
job_uuid=self.job_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
dep_resp, _ = self.create_or_update_deployment(name=deployment_name,
data=deployment)
service = services.get_service(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
service_resp, _ = self.create_or_update_service(name=deployment_name,
data=service)
results = {
'deployment': dep_resp.to_dict(),
'service': service_resp.to_dict()
}
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path':
'/tensorboard/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)
return results
def start_tensorboard(self, image, resources=None, node_selectors=None):
ports = [self.request_tensorboard_port()]
target_ports = [self.PORT]
volumes, volume_mounts = get_pod_volumes()
outputs_path = get_project_outputs_path(project_name=self.project_name)
deployment = deployments.get_deployment(
namespace=self.namespace,
app=settings.APP_LABELS_TENSORBOARD,
name=self.TENSORBOARD_JOB_NAME,
project_name=self.project_name,
project_uuid=self.project_uuid,
job_name=self.job_name,
job_uuid=self.job_uuid,
volume_mounts=volume_mounts,
volumes=volumes,
image=image,
command=["/bin/sh", "-c"],
args=["tensorboard --logdir={} --port={}".format(outputs_path, self.PORT)],
ports=target_ports,
container_name=settings.CONTAINER_NAME_PLUGIN_JOB,
resources=resources,
node_selector=node_selectors,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
deployment_name = constants.JOB_NAME.format(name=self.TENSORBOARD_JOB_NAME,
job_uuid=self.job_uuid)
deployment_labels = deployments.get_labels(app=settings.APP_LABELS_TENSORBOARD,
project_name=self.project_name,
project_uuid=self.project_uuid,
job_name=self.job_name,
job_uuid=self.job_uuid,
role=settings.ROLE_LABELS_DASHBOARD,
type=settings.TYPE_LABELS_EXPERIMENT)
dep_resp, _ = self.create_or_update_deployment(name=deployment_name, data=deployment)
service = services.get_service(
namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
ports=ports,
target_ports=target_ports,
service_type=self._get_service_type())
service_resp, _ = self.create_or_update_service(name=deployment_name, data=service)
results = {'deployment': dep_resp.to_dict(), 'service': service_resp.to_dict()}
if self._use_ingress():
annotations = json.loads(settings.K8S_INGRESS_ANNOTATIONS)
paths = [{
'path': '/tensorboard/{}'.format(self.project_name.replace('.', '/')),
'backend': {
'serviceName': deployment_name,
'servicePort': ports[0]
}
}]
ingress = ingresses.get_ingress(namespace=self.namespace,
name=deployment_name,
labels=deployment_labels,
annotations=annotations,
paths=paths)
self.create_or_update_ingress(name=deployment_name, data=ingress)
return results