def validate_experiment_spec_config(config):
try:
spec = ExperimentSpecification.read(config)
except (PolyaxonfileError, PolyaxonConfigurationError):
raise ValidationError('Received non valid specification config.')
return spec
def test_create_experiment_with_valid_spec(self, spawner_mock):
config = ExperimentSpecification.read(experiment_spec_content)
mock_instance = spawner_mock.return_value
mock_instance.start_experiment.return_value = start_experiment_value
mock_instance.spec = config
experiment = ExperimentFactory(config=config.parsed_data)
assert experiment.is_independent is True
assert ExperimentStatus.objects.filter(
experiment=experiment).count() == 3
assert list(
ExperimentStatus.objects.filter(experiment=experiment).values_list(
'status', flat=True)) == [
ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED,
ExperimentLifeCycle.STARTING
]
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.STARTING
# Assert 1 job was created
assert ExperimentJob.objects.filter(experiment=experiment).count() == 1
assert JobResources.objects.count() == 0
jobs_statuses = ExperimentJob.objects.values_list('statuses__status',
flat=True)
assert set(jobs_statuses) == {
JobLifeCycle.CREATED,
}
jobs = ExperimentJob.objects.filter(experiment=experiment)
assert experiment.calculated_status == ExperimentLifeCycle.STARTING
for job in jobs:
# Assert the jobs status is created
assert job.last_status == JobLifeCycle.CREATED
def test_get_with_resource_reg_90(self):
# Fix issue#90:
# Failed to getting experiment when specify resources without framework in environment
spec_content = """---
version: 1
kind: experiment
environment:
resources:
gpu:
requests: 1
limits: 1
build:
image: my_image
run:
cmd: video_prediction_train --model=DNA --num_masks=1
"""
spec_parsed_content = ExperimentSpecification.read(spec_content)
project = ProjectFactory(user=self.auth_client.user)
exp = self.factory_class(project=project,
config=spec_parsed_content.parsed_data)
url = '/{}/{}/{}/experiments/{}/'.format(API_V1, project.user.username,
project.name, exp.sequence)
resp = self.auth_client.get(url)
assert resp.status_code == status.HTTP_200_OK
exp.refresh_from_db()
assert resp.data == self.serializer_class(exp).data
def test_independent_experiment_creation_with_run_triggers_experiment_building_scheduling(
self):
config = ExperimentSpecification.read(exec_experiment_spec_content)
# Create a repo for the project
repo = RepoFactory()
with patch('runner.dockerizer.builders.experiments.build_experiment'
) as mock_docker_build:
experiment = ExperimentFactory(config=config.parsed_data,
project=repo.project)
assert mock_docker_build.call_count == 1
assert experiment.project.repo is not None
assert experiment.is_independent is True
assert ExperimentStatus.objects.filter(
experiment=experiment).count() == 3
assert list(
ExperimentStatus.objects.filter(experiment=experiment).values_list(
'status', flat=True)) == [
ExperimentLifeCycle.CREATED, ExperimentLifeCycle.BUILDING,
ExperimentLifeCycle.SCHEDULED
]
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.SCHEDULED
def test_serialize_with_environment_section(self):
spec_content = """---
version: 1
kind: experiment
environment:
resources:
cpu:
requests: 2
limits: 4
memory:
requests: 4096
limits: 10240
pytorch:
n_workers: 2
default_worker:
resources:
cpu:
requests: 2
limits: 4
memory:
requests: 4096
limits: 10240
run:
image: my_image
cmd: video_prediction_train --model=DNA --num_masks=1
"""
spec = ExperimentSpecification.read(spec_content)
obj = self.factory_class(config=spec.parsed_data)
serializer = self.serializer_class(obj)
data = serializer.data
assert 'resources' in data
def test_serialize_cluster_resources(self):
spec_content = """---
version: 1
kind: experiment
environment:
resources:
cpu:
requests: 2
limits: 4
memory:
requests: 4096
limits: 10240
pytorch:
n_workers: 2
default_worker_resources:
cpu:
requests: 2
limits: 4
memory:
requests: 4096
limits: 10240
run:
image: my_image
cmd: video_prediction_train --model=DNA --num_masks=1
"""
spec = ExperimentSpecification.read(spec_content)
obj = self.factory_class(config=spec.parsed_data)
self.serializer_class(obj).data # pylint:disable=expression-not-assigned
def experiments_stop(project_name,
project_uuid,
experiment_name,
experiment_group_name,
experiment_group_uuid,
experiment_uuid,
specification,
update_status=True):
specification = ExperimentSpecification.read(specification)
experiment_scheduler.stop_experiment(
project_name=project_name,
project_uuid=project_uuid,
experiment_name=experiment_name,
experiment_group_name=experiment_group_name,
experiment_group_uuid=experiment_group_uuid,
experiment_uuid=experiment_uuid,
specification=specification,
)
if not update_status:
return
experiment = get_valid_experiment(experiment_uuid=experiment_uuid)
if not experiment:
_logger.info(
'Something went wrong, '
'the Experiment `%s` does not exist anymore.', experiment_uuid)
return
# Update experiment status to show that its stopped
experiment.set_status(ExperimentLifeCycle.STOPPED)
def test_get_with_resource_reg_90(self):
# Fix issue#90:
# Failed to getting experiment when specify resources without framework in environment
spec_content = """---
version: 1
kind: experiment
environment:
resources:
gpu:
requests: 1
limits: 1
build:
image: my_image
run:
cmd: video_prediction_train --model=DNA --num_masks=1
"""
spec_parsed_content = ExperimentSpecification.read(spec_content)
project = ProjectFactory(user=self.auth_client.user)
exp = self.factory_class(project=project, config=spec_parsed_content.parsed_data)
url = '/{}/{}/{}/experiments/{}/'.format(API_V1,
project.user.username,
project.name,
exp.sequence)
resp = self.auth_client.get(url)
assert resp.status_code == status.HTTP_200_OK
exp.refresh_from_db()
assert resp.data == self.serializer_class(exp).data
def validate_experiment_spec_config(config, raise_for_rest=False):
try:
spec = ExperimentSpecification.read(config)
except (PolyaxonfileError, PolyaxonConfigurationError) as e:
message_error = 'Received non valid specification config. %s' % e
if raise_for_rest:
raise ValidationError(message_error)
else:
raise DjangoValidationError(message_error)
return spec
def test_set_metrics(self):
config = ExperimentSpecification.read(experiment_spec_content)
experiment = ExperimentFactory(config=config.parsed_data)
assert experiment.metrics.count() == 0
create_at = timezone.now()
experiments_set_metrics(experiment_uuid=experiment.uuid.hex,
created_at=create_at,
metrics={'accuracy': 0.9, 'precision': 0.9})
assert experiment.metrics.count() == 1
def test_independent_experiment_creation_triggers_experiment_scheduling(self):
content = ExperimentSpecification.read(experiment_spec_content)
experiment = ExperimentFactory(config=content.parsed_data)
assert experiment.is_independent is True
assert ExperimentStatus.objects.filter(experiment=experiment).count() == 2
assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list(
'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED]
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.SCHEDULED
# Assert also that experiment is monitored
assert experiment.last_status == ExperimentLifeCycle.SCHEDULED
def reduce_configs(self):
"""Reduce the experiments to restart."""
experiment_ids = self.get_reduced_configs()
experiments = self.experiment_group.experiments.filter(
id__in=experiment_ids)
self.create_iteration(experiment_ids=experiment_ids)
iteration_config = self.experiment_group.iteration_config
params_config = self.experiment_group.params_config
n_resources = self.experiment_group.search_manager.get_resources_for_iteration(
iteration=iteration_config.iteration)
resource_value = self.experiment_group.search_manager.get_n_resources(
n_resources=n_resources,
bracket_iteration=iteration_config.bracket_iteration)
resource_name = params_config.hyperband.resource.name
resource_value = params_config.hyperband.resource.cast_value(
resource_value)
# Check if we need to resume or restart the experiments
for experiment in experiments:
declarations = experiment.declarations
declarations[resource_name] = resource_value
declarations_spec = {'declarations': declarations}
specification = ExperimentSpecification(values=[
experiment.specification.parsed_data, declarations_spec
])
status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
iteration_config.iteration, iteration_config.bracket_iteration)
if params_config.hyperband.resume:
experiment.resume(declarations=declarations,
config=specification.parsed_data,
message=status_message)
else:
experiment.restart(experiment_group=self.experiment_group,
declarations=declarations,
config=specification.parsed_data)
def create_experiment(self, config):
config = ExperimentSpecification.read(config)
return ExperimentFactory(config=config.parsed_data,
project=self.project)
- Flatten:
- Dense:
units: 10
activation: softmax
train:
data_pipeline:
TFRecordImagePipeline:
batch_size: 64
num_epochs: 1
shuffle: true
dynamic_pad: false
data_files: ["../data/mnist/mnist_train.tfrecord"]
meta_data_file: "../data/mnist/meta_data.json"
"""
experiment_spec_parsed_content = ExperimentSpecification.read(experiment_spec_content)
exec_experiment_spec_content = """---
version: 1
kind: experiment
project:
name: project1
run:
image: my_image
cmd: video_prediction_train --model=DNA --num_masks=1
"""
exec_experiment_spec_parsed_content = ExperimentSpecification.read(exec_experiment_spec_content)
def specification(self):
return ExperimentSpecification(
values=self.config) if self.config else None
def test_cluster_def_without_framwork(self):
spec = ExperimentSpecification.read(os.path.abspath('tests/fixtures/env_without_framework.yml'))
self.assertEqual(spec.cluster_def, ({TaskType.MASTER: 1}, False))