def test_object_hyperparameters(self): hyperparameters_str = '{"hp1": {"innerkey": "innerval"}}' invalid_args = self.parser.parse_args( required_args + ['--static_parameters', hyperparameters_str]) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(invalid_args))
def test_spot_lesser_wait_time(self): args = self.parser.parse_args(required_args + [ '--spot_instance', 'True', '--max_wait_time', '86399', '--checkpoint_config', '{"S3Uri": "s3://fake-uri/", "LocalPath": "local-path"}' ]) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(args))
def test_spot_bad_args(self): no_max_wait_args = self.parser.parse_args(required_args + ['--spot_instance', 'True']) no_checkpoint_args = self.parser.parse_args(required_args + ['--spot_instance', 'True', '--max_wait_time', '3600']) no_s3_uri_args = self.parser.parse_args(required_args + ['--spot_instance', 'True', '--max_wait_time', '3600', '--checkpoint_config', '{}']) for arg in [no_max_wait_args, no_checkpoint_args, no_s3_uri_args]: with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(arg))
def test_either_warm_start_or_parents_args(self): # It will generate an exception if either warm_start_type or parent hpo jobs is being passed missing_parent_hpo_jobs_args = self.parser.parse_args(required_args + ['--warm_start_type', 'TransferLearning']) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(missing_parent_hpo_jobs_args)) missing_warm_start_type_args = self.parser.parse_args(required_args + ['--parent_hpo_jobs', 'A,B,C']) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(missing_warm_start_type_args))
def test_no_channels(self): no_channels_args = required_args.copy() channels_index = required_args.index('--channels') # Replace the value after the flag with an empty list no_channels_args[channels_index + 1] = '[]' parsed_args = self.parser.parse_args(no_channels_args) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(parsed_args))
def test_no_defined_image(self): # Pass the image to pass the parser no_image_args = required_args.copy() image_index = no_image_args.index('--image') # Cut out --image and it's associated value no_image_args = no_image_args[:image_index] + no_image_args[image_index+2:] parsed_args = self.parser.parse_args(no_image_args) with self.assertRaises(Exception): _utils.create_hyperparameter_tuning_job_request(vars(parsed_args))
def test_empty_hyperparameters(self): hyperparameters_str = '{}' good_args = self.parser.parse_args(required_args + ['--static_parameters', hyperparameters_str]) response = _utils.create_hyperparameter_tuning_job_request(vars(good_args)) self.assertEqual(response['TrainingJobDefinition']['StaticHyperParameters'], {})
def test_empty_string(self): good_args = self.parser.parse_args(required_args + [ '--spot_instance', 'True', '--max_wait_time', '86400', '--checkpoint_config', '{"S3Uri": "s3://fake-uri/"}' ]) response = _utils.create_hyperparameter_tuning_job_request( vars(good_args)) test_utils.check_empty_string_values(response)
def test_first_party_algorithm(self): algorithm_name_args = self.parser.parse_args(required_args + ['--algorithm_name', 'first-algorithm']) # Should not throw an exception response = _utils.create_hyperparameter_tuning_job_request(vars(algorithm_name_args)) self.assertIn('TrainingJobDefinition', response) self.assertIn('TrainingImage', response['TrainingJobDefinition']['AlgorithmSpecification']) self.assertNotIn('AlgorithmName', response['TrainingJobDefinition']['AlgorithmSpecification'])
def test_training_mode(self): required_vpc_args = self.parser.parse_args( required_args + ['--training_input_mode', 'Pipe']) response = _utils.create_hyperparameter_tuning_job_request( vars(required_vpc_args)) self.assertEqual( response['TrainingJobDefinition']['AlgorithmSpecification'] ['TrainingInputMode'], 'Pipe')
def test_vpc_configuration(self): required_vpc_args = self.parser.parse_args(required_args + ['--vpc_security_group_ids', 'sg1,sg2', '--vpc_subnets', 'subnet1,subnet2']) response = _utils.create_hyperparameter_tuning_job_request(vars(required_vpc_args)) self.assertIn('TrainingJobDefinition', response) self.assertIn('VpcConfig', response['TrainingJobDefinition']) self.assertIn('sg1', response['TrainingJobDefinition']['VpcConfig']['SecurityGroupIds']) self.assertIn('sg2', response['TrainingJobDefinition']['VpcConfig']['SecurityGroupIds']) self.assertIn('subnet1', response['TrainingJobDefinition']['VpcConfig']['Subnets']) self.assertIn('subnet2', response['TrainingJobDefinition']['VpcConfig']['Subnets'])
def test_reasonable_required_args(self): response = _utils.create_hyperparameter_tuning_job_request(vars(self.parser.parse_args(required_args))) # Ensure all of the optional arguments have reasonable default values self.assertFalse(response['TrainingJobDefinition']['EnableManagedSpotTraining']) self.assertDictEqual(response['TrainingJobDefinition']['StaticHyperParameters'], {}) self.assertNotIn('VpcConfig', response['TrainingJobDefinition']) self.assertNotIn('MetricDefinitions', response['TrainingJobDefinition']) self.assertEqual(response['Tags'], []) self.assertEqual(response['TrainingJobDefinition']['AlgorithmSpecification']['TrainingInputMode'], 'File') self.assertEqual(response['TrainingJobDefinition']['OutputDataConfig']['S3OutputPath'], 'test-output-location')
def test_warm_start_and_parents_args(self): # specifying both params good_args = self.parser.parse_args(required_args + ['--warm_start_type', 'TransferLearning'] + ['--parent_hpo_jobs', 'A,B,C']) response = _utils.create_hyperparameter_tuning_job_request(vars(good_args)) self.assertIn('WarmStartConfig', response) self.assertIn('ParentHyperParameterTuningJobs', response['WarmStartConfig']) self.assertIn('WarmStartType', response['WarmStartConfig']) self.assertEqual(response['WarmStartConfig']['ParentHyperParameterTuningJobs'][0]['HyperParameterTuningJobName'], 'A') self.assertEqual(response['WarmStartConfig']['ParentHyperParameterTuningJobs'][1]['HyperParameterTuningJobName'], 'B') self.assertEqual(response['WarmStartConfig']['ParentHyperParameterTuningJobs'][2]['HyperParameterTuningJobName'], 'C') self.assertEqual(response['WarmStartConfig']['WarmStartType'], 'TransferLearning')
def test_spot_local_path(self): args = self.parser.parse_args(required_args + [ '--spot_instance', 'True', '--max_wait_time', '86400', '--checkpoint_config', '{"S3Uri": "s3://fake-uri/", "LocalPath": "local-path"}' ]) response = _utils.create_hyperparameter_tuning_job_request(vars(args)) self.assertEqual( response['TrainingJobDefinition']['CheckpointConfig']['S3Uri'], 's3://fake-uri/') self.assertEqual( response['TrainingJobDefinition']['CheckpointConfig']['LocalPath'], 'local-path')
def test_metric_definitions(self): metric_definition_args = self.parser.parse_args(required_args + ['--metric_definitions', '{"metric1": "regexval1", "metric2": "regexval2"}']) response = _utils.create_hyperparameter_tuning_job_request(vars(metric_definition_args)) self.assertIn('MetricDefinitions', response['TrainingJobDefinition']['AlgorithmSpecification']) response_metric_definitions = response['TrainingJobDefinition']['AlgorithmSpecification']['MetricDefinitions'] self.assertEqual(response_metric_definitions, [{ 'Name': "metric1", 'Regex': "regexval1" }, { 'Name': "metric2", 'Regex': "regexval2" }])
def test_spot_good_args(self): good_args = self.parser.parse_args(required_args + [ '--spot_instance', 'True', '--max_wait_time', '86400', '--checkpoint_config', '{"S3Uri": "s3://fake-uri/"}' ]) response = _utils.create_hyperparameter_tuning_job_request( vars(good_args)) self.assertTrue( response['TrainingJobDefinition']['EnableManagedSpotTraining']) self.assertEqual( response['TrainingJobDefinition']['StoppingCondition'] ['MaxWaitTimeInSeconds'], 86400) self.assertEqual( response['TrainingJobDefinition']['CheckpointConfig']['S3Uri'], 's3://fake-uri/')
def test_known_algorithm_value(self): # This passes an algorithm that is a known SageMaker algorithm name known_algorithm_args = required_args + ['--algorithm_name', 'seq2seq'] image_index = required_args.index('--image') # Cut out --image and it's associated value known_algorithm_args = known_algorithm_args[:image_index] + known_algorithm_args[image_index+2:] parsed_args = self.parser.parse_args(known_algorithm_args) # Patch retrieve _utils.retrieve = MagicMock() _utils.retrieve.return_value = "seq2seq-url" response = _utils.create_hyperparameter_tuning_job_request(vars(parsed_args)) _utils.retrieve.assert_called_with('seq2seq', 'us-west-2') self.assertEqual(response['TrainingJobDefinition']['AlgorithmSpecification']['TrainingImage'], "seq2seq-url")
def test_unknown_algorithm(self): known_algorithm_args = required_args + ['--algorithm_name', 'unknown algorithm'] image_index = required_args.index('--image') # Cut out --image and it's associated value known_algorithm_args = known_algorithm_args[:image_index] + known_algorithm_args[image_index+2:] parsed_args = self.parser.parse_args(known_algorithm_args) # Patch retrieve _utils.retrieve = MagicMock() _utils.retrieve.return_value = "unknown-url" response = _utils.create_hyperparameter_tuning_job_request(vars(parsed_args)) # Should just place the algorithm name in regardless _utils.retrieve.assert_not_called() self.assertEqual(response['TrainingJobDefinition']['AlgorithmSpecification']['AlgorithmName'], "unknown algorithm")
def test_valid_hyperparameters(self): hyperparameters_str = '{"hp1": "val1", "hp2": "val2", "hp3": "val3"}' categorical_params = '[{"Name" : "categorical", "Values": ["A", "B"]}]' integer_params = '[{"MaxValue": "integer_val1", "MinValue": "integer_val2", "Name": "integer", "ScalingType": "test_integer"}]' continuous_params = '[{"MaxValue": "continuous_val1", "MinValue": "continuous_val2", "Name": "continuous", "ScalingType": "test_continuous"}]' good_args = self.parser.parse_args(required_args + ['--static_parameters', hyperparameters_str] + ['--integer_parameters', integer_params] + ['--continuous_parameters', continuous_params] + ['--categorical_parameters', categorical_params]) response = _utils.create_hyperparameter_tuning_job_request(vars(good_args)) self.assertIn('hp1', response['TrainingJobDefinition']['StaticHyperParameters']) self.assertIn('hp2', response['TrainingJobDefinition']['StaticHyperParameters']) self.assertIn('hp3', response['TrainingJobDefinition']['StaticHyperParameters']) self.assertEqual(response['TrainingJobDefinition']['StaticHyperParameters']['hp1'], "val1") self.assertEqual(response['TrainingJobDefinition']['StaticHyperParameters']['hp2'], "val2") self.assertEqual(response['TrainingJobDefinition']['StaticHyperParameters']['hp3'], "val3") self.assertIn('ParameterRanges', response['HyperParameterTuningJobConfig']) self.assertIn('IntegerParameterRanges', response['HyperParameterTuningJobConfig']['ParameterRanges']) self.assertIn('ContinuousParameterRanges', response['HyperParameterTuningJobConfig']['ParameterRanges']) self.assertIn('CategoricalParameterRanges', response['HyperParameterTuningJobConfig']['ParameterRanges']) self.assertIn('Name', response['HyperParameterTuningJobConfig']['ParameterRanges']['CategoricalParameterRanges'][0]) self.assertIn('Values', response['HyperParameterTuningJobConfig']['ParameterRanges']['CategoricalParameterRanges'][0]) self.assertIn('MaxValue', response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]) self.assertIn('MinValue', response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]) self.assertIn('Name', response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]) self.assertIn('ScalingType', response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]) self.assertIn('MaxValue', response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]) self.assertIn('MinValue', response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]) self.assertIn('Name', response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]) self.assertIn('ScalingType', response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]) self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['CategoricalParameterRanges'][0]['Name'], "categorical") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['CategoricalParameterRanges'][0]["Values"][0], "A") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['CategoricalParameterRanges'][0]["Values"][1], "B") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]['MaxValue'], "integer_val1") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]['MinValue'], "integer_val2") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]['Name'], "integer") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['IntegerParameterRanges'][0]['ScalingType'], "test_integer") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]['MaxValue'], "continuous_val1") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]['MinValue'], "continuous_val2") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]['Name'], "continuous") self.assertEqual(response['HyperParameterTuningJobConfig']['ParameterRanges']['ContinuousParameterRanges'][0]['ScalingType'], "test_continuous")
def test_tags(self): args = self.parser.parse_args( required_args + ['--tags', '{"key1": "val1", "key2": "val2"}']) response = _utils.create_hyperparameter_tuning_job_request(vars(args)) self.assertIn({'Key': 'key1', 'Value': 'val1'}, response['Tags']) self.assertIn({'Key': 'key2', 'Value': 'val2'}, response['Tags'])