def test_independent_experiment_creation_with_run_triggers_experiment_building_scheduling(self): config = ExperimentSpecification.read(exec_experiment_spec_content) # Create a repo for the project repo = RepoFactory() with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_build: experiment = ExperimentFactory(config=config.parsed_data, project=repo.project) assert mock_build.call_count == 1 assert experiment.project.repo is not None assert experiment.is_independent is True assert ExperimentStatus.objects.filter(experiment=experiment).count() == 1 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED] with patch('dockerizer.builders.experiments.build_experiment') as mock_build: build_experiment(experiment_id=experiment.id) assert mock_build.call_count == 1 assert ExperimentStatus.objects.filter(experiment=experiment).count() == 4 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.BUILDING, ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.FAILED] experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.FAILED
def test_create_experiment_with_resources_spec(self, spawner_mock): config = ExperimentSpecification.read(exec_experiment_resources_content) mock_instance = spawner_mock.return_value mock_instance.start_experiment.return_value = start_experiment_value mock_instance.spec = config experiment = ExperimentFactory(config=config.parsed_data) assert experiment.is_independent is True assert ExperimentStatus.objects.filter(experiment=experiment).count() == 3 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.STARTING] experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STARTING # Assert 3 jobs were created with resources assert ExperimentJob.objects.filter(experiment=experiment).count() == 3 assert JobResources.objects.count() == 3 jobs_statuses = ExperimentJob.objects.values_list('statuses__status', flat=True) assert set(jobs_statuses) == {JobLifeCycle.CREATED, } jobs = ExperimentJob.objects.filter(experiment=experiment) assert experiment.calculated_status == ExperimentLifeCycle.STARTING for job in jobs: # Assert the jobs status is created assert job.last_status == JobLifeCycle.CREATED
def test_set_metrics(self): config = ExperimentSpecification.read(experiment_spec_content) experiment = ExperimentFactory(config=config.parsed_data) assert experiment.metrics.count() == 0 create_at = timezone.now() experiments_set_metrics(experiment_id=experiment.id, data={ 'created_at': create_at, 'values': { 'accuracy': 0.9, 'precision': 0.9 } }) assert experiment.metrics.count() == 1 experiments_set_metrics(experiment_id=experiment.id, data=[{ 'created_at': create_at, 'values': { 'accuracy': 0.9, 'precision': 0.9 } }, { 'created_at': create_at, 'values': { 'accuracy': 0.9, 'precision': 0.9 } }]) assert experiment.metrics.count() == 3
def experiments_stop(project_name, project_uuid, experiment_name, experiment_group_name, experiment_group_uuid, experiment_uuid, specification, update_status=True): specification = ExperimentSpecification.read(specification) experiment_scheduler.stop_experiment( project_name=project_name, project_uuid=project_uuid, experiment_name=experiment_name, experiment_group_name=experiment_group_name, experiment_group_uuid=experiment_group_uuid, experiment_uuid=experiment_uuid, specification=specification, ) if not update_status: return experiment = get_valid_experiment(experiment_uuid=experiment_uuid) if not experiment: _logger.info( 'Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_uuid) return # Update experiment status to show that its stopped experiment.set_status(ExperimentLifeCycle.STOPPED, message='Experiment was stopped')
def test_serialize_with_environment_section(self): spec_content = """--- version: 1 kind: experiment environment: resources: cpu: requests: 2 limits: 4 memory: requests: 4096 limits: 10240 pytorch: n_workers: 2 default_worker: resources: cpu: requests: 2 limits: 4 memory: requests: 4096 limits: 10240 run: image: my_image cmd: video_prediction_train --model=DNA --num_masks=1 """ spec = ExperimentSpecification.read(spec_content) obj = self.factory_class(config=spec.parsed_data) serializer = self.serializer_class(obj) data = serializer.data assert 'resources' in data
def test_create_experiment_with_resources_spec(self, spawner_mock): config = ExperimentSpecification.read(exec_experiment_resources_content) mock_instance = spawner_mock.return_value mock_instance.start_experiment.return_value = start_experiment_value mock_instance.job_uuids = {'master': ['fa6203c189a855dd977019854a7ffcc3'], 'worker': ['3a9c9b0bd56b5e9fbdbd1a3d43d57960'], 'ps': ['59e3601232b85a3d8be2511f23a62945']} mock_instance.spec = config experiment = ExperimentFactory(config=config.parsed_data) assert experiment.is_independent is True assert ExperimentStatus.objects.filter(experiment=experiment).count() == 3 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.STARTING] experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.STARTING # Assert 3 jobs were created with resources assert ExperimentJob.objects.filter(experiment=experiment).count() == 3 assert JobResources.objects.count() == 3 jobs_statuses = ExperimentJob.objects.values_list('statuses__status', flat=True) assert set(jobs_statuses) == {JobLifeCycle.CREATED, } jobs = ExperimentJob.objects.filter(experiment=experiment) assert experiment.calculated_status == ExperimentLifeCycle.STARTING for job in jobs: # Assert the jobs status is created assert job.last_status == JobLifeCycle.CREATED
def test_independent_experiment_creation_with_run_triggers_experiment_scheduling(self): config = ExperimentSpecification.read(exec_experiment_spec_content) # Create a repo for the project repo = RepoFactory() with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_build: experiment = ExperimentFactory(config=config.parsed_data, project=repo.project) assert mock_build.call_count == 1 assert experiment.project.repo is not None assert experiment.is_independent is True assert ExperimentStatus.objects.filter(experiment=experiment).count() == 1 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED] with patch('scheduler.dockerizer_scheduler.create_build_job') as mock_start: build = BuildJobFactory() BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build) mock_start.return_value = build, True, True experiments_build(experiment_id=experiment.id) assert mock_start.call_count == 1 assert ExperimentStatus.objects.filter(experiment=experiment).count() == 3 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.FAILED] experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.FAILED
def test_independent_experiment_creation_triggers_experiment_scheduling(self): content = ExperimentSpecification.read(experiment_spec_content) experiment = ExperimentFactory(config=content.parsed_data) assert experiment.is_independent is True assert ExperimentStatus.objects.filter(experiment=experiment).count() == 3 assert list(ExperimentStatus.objects.filter(experiment=experiment).values_list( 'status', flat=True)) == [ExperimentLifeCycle.CREATED, ExperimentLifeCycle.SCHEDULED, ExperimentLifeCycle.FAILED] experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.FAILED
def experiments_stop(self, project_name, project_uuid, experiment_name, experiment_group_name, experiment_group_uuid, experiment_uuid, specification, update_status=True, collect_logs=True, message=None): if collect_logs: try: collectors.logs_collect_experiment_jobs( experiment_uuid=experiment_uuid) except OSError: _logger.warning( 'Scheduler could not collect ' 'the logs for experiment `%s`.', experiment_name) if specification: specification = ExperimentSpecification.read(specification) deleted = experiment_scheduler.stop_experiment( project_name=project_name, project_uuid=project_uuid, experiment_name=experiment_name, experiment_group_name=experiment_group_name, experiment_group_uuid=experiment_group_uuid, experiment_uuid=experiment_uuid, specification=specification, ) else: deleted = True if not deleted and self.request.retries < 2: _logger.info('Trying again to delete job `%s` in experiment.', experiment_name) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return if not update_status: return experiment = get_valid_experiment(experiment_uuid=experiment_uuid, include_deleted=True) if not experiment: _logger.info( 'Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_uuid) return # Update experiment status to show that its stopped experiment.set_status(ExperimentLifeCycle.STOPPED, message=message or 'Experiment was stopped')
def validate_experiment_spec_config(config, raise_for_rest: bool = False): try: spec = ExperimentSpecification.read(config) except (MarshmallowValidationError, PolyaxonfileError, PolyaxonConfigurationError) as e: message_error = 'Received non valid specification config. %s' % e if raise_for_rest: raise ValidationError(message_error) else: raise DjangoValidationError(message_error) return spec
def test_set_metrics(self): config = ExperimentSpecification.read(experiment_spec_content) experiment = ExperimentFactory(config=config.parsed_data) assert experiment.metrics.count() == 0 create_at = timezone.now() experiments_set_metrics(experiment_uuid=experiment.uuid.hex, created_at=create_at, metrics={ 'accuracy': 0.9, 'precision': 0.9 }) assert experiment.metrics.count() == 1
def test_serialize_with_environment_section(self): spec_content = """--- version: 1 kind: experiment framework: pytorch environment: resources: cpu: requests: 2 limits: 4 memory: requests: 4096 limits: 10240 replicas: n_workers: 2 default_worker: resources: cpu: requests: 2 limits: 4 memory: requests: 4096 limits: 10240 build: image: foo run: cmd: video_prediction_train --model=DNA --num_masks=1 """ spec = ExperimentSpecification.read(spec_content) obj = self.factory_class(config=spec.parsed_data) obj1_query = queries.experiments_details.get(id=obj.id) serializer = self.serializer_class(obj1_query) data = serializer.data assert 'resources' in data
def experiments_stop(self, project_name, project_uuid, experiment_name, experiment_group_name, experiment_group_uuid, experiment_uuid, specification, update_status=True): specification = ExperimentSpecification.read(specification) deleted = experiment_scheduler.stop_experiment( project_name=project_name, project_uuid=project_uuid, experiment_name=experiment_name, experiment_group_name=experiment_group_name, experiment_group_uuid=experiment_group_uuid, experiment_uuid=experiment_uuid, specification=specification, ) if not deleted and self.request.retries < 2: _logger.info('Trying again to delete job `%s` in experiment.', experiment_name) self.retry(countdown=Intervals.EXPERIMENTS_SCHEDULER) return if not update_status: return experiment = get_valid_experiment(experiment_uuid=experiment_uuid) if not experiment: _logger.info( 'Something went wrong, ' 'the Experiment `%s` does not exist anymore.', experiment_uuid) return # Update experiment status to show that its stopped experiment.set_status(ExperimentLifeCycle.STOPPED, message='Experiment was stopped')
- Flatten: - Dense: units: 10 activation: softmax train: data_pipeline: TFRecordImagePipeline: batch_size: 64 num_epochs: 1 shuffle: true dynamic_pad: false data_files: ["../data/mnist/mnist_train.tfrecord"] meta_data_file: "../data/mnist/meta_data.json" """ experiment_spec_parsed_content = ExperimentSpecification.read( experiment_spec_content) exec_experiment_spec_content = """--- version: 1 kind: experiment tags: [fixtures] build: image: my_image run: cmd: video_prediction_train --model=DNA --num_masks=1 """
def specification(self): return ExperimentSpecification( values=self.config) if self.config else None
def create_experiment(self, config): config = ExperimentSpecification.read(config) return ExperimentFactory(config=config.parsed_data, project=self.project)