def createCounter(self, retries_count, connection_retries_count): client = HubApiClient( hub_app_url='https://some-url.com', retries_count=retries_count, connection_retries_count=connection_retries_count) return HubApiClient.RetryCounter(client)
def __init__(self, url, token, debug=False): super(RestApi, self).__init__() self.hub_client = HubApiClient(hub_app_url=url, token=token, debug=debug) self.api_url = url self.debug = debug
def setUp(self): self.hub_project_api_token = '0d5a55cb795f2039922689e647cb3c5d24a0992bf58d941dad864ef7c371f8fc' self.client = HubApiClient( hub_app_url='http://localhost:5000', optimizers_url='http://localhost:7777', retries_count=1, hub_project_api_token=self.hub_project_api_token)
def test_auth_token_invalid(self, sleep_mock): client = HubApiClient(hub_app_url='http://localhost:5000', token='wrong') with self.assertRaises(HubApiClient.FatalApiError) as context: client.get_experiments() self.assertUnauthenticatedResponse(context.exception.metadata())
def test_network_unavailable(self, sleep_mock): client = HubApiClient( hub_app_url='http://ivalid-url', token='some-token', retries_count=2, ) with self.assertRaises(HubApiClient.NetworkError) as context: client.get_trials()
def test_auth_token_valid(self, sleep_mock): client = HubApiClient( hub_app_url='http://localhost:5000', token= 'eyJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6ImhlckBtYWlsLmNvbSIsImV4cCI6MTU1NTIzNjI0NH0.SyZ5e1-zbuFEy7Q176fcuToehpdwrSIa-CK-qTs0D_E' ) res = client.get_experiments() self.assertIndexResponse(res, 'experiment')
def test_counter_connection_retry(self): counter = self.createCounter(retries_count=1, connection_retries_count=2) self.assertEqual(counter.is_retries_available(), True) counter.count_retry(HubApiClient.NetworkError('some error')) self.assertEqual(counter.is_retries_available(), True) counter.count_retry(HubApiClient.NetworkError('some error')) self.assertEqual(counter.is_retries_available(), False)
def test_server_unavailable(self, sleep_mock): client = HubApiClient( hub_app_url='https://optimizers-service-prod.herokuapp.com', token='some-token', retries_count=2, ) with self.assertRaises(HubApiClient.RetryableApiError) as context: client.get_trials() self.assertServerErrorResponse(context.exception)
def test_counter_unlnown_retry(self): counter = self.createCounter(retries_count=1, connection_retries_count=2) with self.assertRaises(RuntimeError) as context: counter.count_retry(HubApiClient.InvalidParamsError('some error')) self.assertIn('Unsupported kind of error', str(context.error))
def build_hub_client_for_optimizer(self, optimizers_url='http://localhost:7777', token=None): if not token: token = 'eyJhbGciOiJIUzI1NiJ9.eyJ4IjoxfQ.NLc1yhn2PDZhJaWTI2dtHkHzn2r8cND1MwiwrVtNlx0' return HubApiClient(hub_app_url='http://localhost:5000', optimizers_url=optimizers_url, retries_count=1, hub_project_api_token=token)
def __init__(self, url, token): super(RestApi, self).__init__() self.hub_client = HubApiClient(hub_app_url=url, token=token) self.api_url = url
class RestApi(object): """Warapper around Auger Cloud Rest Api.""" def __init__(self, url, token): super(RestApi, self).__init__() self.hub_client = HubApiClient(hub_app_url=url, token=token) self.api_url = url def get_status(self, obj, obj_id): return self.hub_client.get_status(object=obj, id=obj_id) def call_ex(self, method, params={}): params = params.copy() if params.get('id') and not method.startswith('create_'): oid = params['id'] del params['id'] return getattr(self.hub_client, method)(oid, **params) else: return getattr(self.hub_client, method)(**params) def call(self, method, params={}): result = self.call_ex(method, params) if 'data' in result: return result['data'] raise AugerException("Call of Auger API method %s failed." % keys) def request_list(self, record_type, params): offset = params.get('offset', 0) limit = params.get('limit', REQUEST_LIMIT) p = params.copy() while limit > 0: p['offset'] = offset p['limit'] = limit response = self.call_ex('get_' + record_type, p) if not 'data' in response or not 'meta' in response: raise AugerException("Read list of %s failed." % record_type) for item in response['data']: yield item received = len(response['data']) offset += received limit -= received if offset >= response['meta']['pagination']['total']: break def wait_for_object_status(self, get_status, progress, object_readable_name, post_check_status=None, log_status=None): def _log_status(obj_status): pass log_status = log_status if log_status else _log_status status = get_status() last_status = '' while status in progress: if status != last_status: last_status = status log_status(status) while status == last_status: time.sleep(STATE_POLL_INTERVAL) status = get_status() if status == 'processed_with_error': raise AugerException( '%s processed with error' % object_readable_name) elif status == 'error' or status == "failure": raise AugerException('Auger Cloud return error...') if post_check_status: post_check_status(status) return status
class TestHubApiClient(unittest.TestCase): def setUp(self): self.hub_project_api_token = '0d5a55cb795f2039922689e647cb3c5d24a0992bf58d941dad864ef7c371f8fc' self.client = HubApiClient( hub_app_url='http://*****:*****@vcr.use_cassette('auth/token_valid.yaml') def test_auth_token_valid(self, sleep_mock): client = HubApiClient( hub_app_url='http://*****:*****@vcr.use_cassette('auth/token_invalid.yaml') def test_auth_token_invalid(self, sleep_mock): client = HubApiClient(hub_app_url='http://*****:*****@vcr.use_cassette('general_errors/network_unavailable.yaml') def test_network_unavailable(self, sleep_mock): client = HubApiClient( hub_app_url='http://ivalid-url', token='some-token', retries_count=2, ) with self.assertRaises(HubApiClient.NetworkError) as context: client.get_trials() # 503 error and similar errors @vcr.use_cassette('general_errors/server_unavailable.yaml') def test_server_unavailable(self, sleep_mock): client = HubApiClient( hub_app_url='https://optimizers-service-prod.herokuapp.com', token='some-token', retries_count=2, ) with self.assertRaises(HubApiClient.RetryableApiError) as context: client.get_trials() self.assertServerErrorResponse(context.exception) # Cluster statuses @vcr.use_cassette('cluster_statuses/index.yaml') def test_get_cluster_statuses(self, sleep_mock): res = self.client.get_cluster_statuses(cluster_id=340) self.assertIndexResponse(res, 'cluster_status') # Cluster tasks @vcr.use_cassette('cluster_tasks/show.yaml') def test_get_cluster_task(self, sleep_mock): res = self.client.get_cluster_task(25) self.assertResourceResponse(res, 'cluster_task') @vcr.use_cassette('cluster_tasks/index.yaml') def test_get_cluster_tasks(self, sleep_mock): res = self.client.get_cluster_tasks() self.assertIndexResponse(res, 'cluster_task') @vcr.use_cassette('cluster_tasks/create_valid.yaml') def test_create_cluster_task_valid(self, sleep_mock): res = self.client.create_cluster_task( name='auger_ml.tasks_queue.tasks.list_project_files', queue=None, args=None) self.assertResourceResponse(res, 'cluster_task') @vcr.use_cassette('cluster_tasks/update_valid.yaml') def test_update_cluster_task_valid(self, sleep_mock): res = self.client.update_cluster_task( id=55, status='success', result={ 'x': 1, 'y': 2 }, traceback='Some str\nSome more data') self.assertResourceResponse(res, 'cluster_task') # Dataset manifests @vcr.use_cassette('dataset_manifests/show.yaml') def test_get_dataset_manifest(self, sleep_mock): res = self.client.get_dataset_manifest(100500) self.assertResourceResponse(res, 'dataset_manifest') @vcr.use_cassette('dataset_manifests/index.yaml') def test_get_dataset_manifests(self, sleep_mock): res = self.client.get_dataset_manifests() self.assertIndexResponse(res, 'dataset_manifest') @vcr.use_cassette('dataset_manifests/all_index.yaml') def test_iterate_all_dataset_manifests(self, sleep_mock): self.client.iterate_all_dataset_manifests( lambda item: self.assertEqual(item['object'], 'dataset_manifest'), limit=1) @vcr.use_cassette('dataset_manifests/create_invalid.yaml') def test_create_dataset_manifest_invalid(self, sleep_mock): with self.assertRaises(HubApiClient.InvalidParamsError) as context: self.client.create_dataset_manifest() self.assertInvalidParams(context.exception.metadata(), ['id', 'name', 'statistics']) @vcr.use_cassette('dataset_manifests/create_valid.yaml') def test_create_dataset_manifest_valid(self, sleep_mock): res = self.client.create_dataset_manifest(id=100500, name='test', statistics={ 'x': 1, 'y': 2 }) self.assertResourceResponse(res, 'dataset_manifest') @vcr.use_cassette('dataset_manifests/update_valid.yaml') def test_update_dataset_manifest_valid(self, sleep_mock): res = self.client.update_dataset_manifest( 123123, dataset_url='s3://bucket/path') self.assertResourceResponse(res, 'dataset_manifest') # Endpoints @vcr.use_cassette('endpoints/show.yaml') def test_get_endpoint(self, sleep_mock): res = self.client.get_endpoint('ddc968ac-43d5-4aa4-9929-1edba7cefc8f') self.assertResourceResponse(res, 'endpoint') @vcr.use_cassette('endpoints/index.yaml') def test_get_endpoints(self, sleep_mock): res = self.client.get_endpoints() self.assertIndexResponse(res, 'endpoint') @vcr.use_cassette('endpoints/create_valid.yaml') def test_create_endpoint_valid(self, sleep_mock): res = self.client.create_endpoint( pipeline_id='118DCF6B1A2A44A', name='test-endpoint', ) self.assertResourceResponse(res, 'endpoint') @vcr.use_cassette('endpoints/update_valid.yaml') def test_update_endpoint_valid(self, sleep_mock): res = self.client.update_endpoint( '3a0cfb34-b03b-468c-8e13-2befe3e78819', name='my super endpoint') self.assertResourceResponse(res, 'endpoint') @vcr.use_cassette('endpoints/delete.yaml') def test_delete_endpoint_valid(self, sleep_mock): res = self.client.delete_endpoint( '3a0cfb34-b03b-468c-8e13-2befe3e78819') self.assertResourceResponse(res, 'endpoint') # Endpoint pipelines @vcr.use_cassette('endpoint_pipelines/create_valid.yaml') def test_create_endpoint_pipeline_valid(self, sleep_mock): res = self.client.create_endpoint_pipeline( endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f', pipeline_id='118DCF6B1A2A44A', active=True, ) self.assertResourceResponse(res, 'endpoint_pipeline') @vcr.use_cassette('endpoint_pipelines/update_valid.yaml') def test_update_endpoint_pipeline_valid(self, sleep_mock): res = self.client.update_endpoint_pipeline(7, active=True) self.assertResourceResponse(res, 'endpoint_pipeline') @vcr.use_cassette('endpoint_pipelines/delete.yaml') def test_delete_endpoint_pipeline_valid(self, sleep_mock): res = self.client.delete_endpoint_pipeline(3) self.assertResourceResponse(res, 'endpoint_pipeline') # Endpoint predictions + actuals @vcr.use_cassette('endpoint_predictions/show.yaml') def test_get_endpoint_prediction(self, sleep_mock): res = self.client.get_endpoint_prediction( '46da6a3f-085b-4169-8157-4d20deba3bd1', endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f') self.assertResourceResponse(res, 'prediction_group') @vcr.use_cassette('endpoint_predictions/index.yaml') def test_get_endpoint_predictions(self, sleep_mock): res = self.client.get_endpoint_predictions( endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f') self.assertIndexResponse(res, 'prediction_group') @vcr.use_cassette('endpoint_predictions/create_valid.yaml') def test_create_endpoint_prediction_valid(self, sleep_mock): res = self.client.create_endpoint_prediction( endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f', records=[[1.1, 1.2, 1.3, 1.4]], features=[ 'sepal_length', 'sepal_length', 'petal_length', 'petal_width' ]) self.assertResourceResponse(res, 'prediction_group') @vcr.use_cassette('endpoint_actuals/create_valid.yaml') def test_create_endpoint_actuals_valid(self, sleep_mock): res = self.client.create_endpoint_actual( endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f', prediction_group_id='46da6a3f-085b-4169-8157-4d20deba3bd1', actuals=[{ 'prediction_id': '1', 'endpoint_actual': 1 }, { 'prediction_id': '2', 'endpoint_actual': 1 }]) self.assertResourceEmptyResponse(res, 'actual') @vcr.use_cassette('endpoint_actuals/delete.yaml') def test_delete_endpoint_actuals_valid(self, sleep_mock): res = self.client.delete_endpoint_actuals( 'ddc968ac-43d5-4aa4-9929-1edba7cefc8f', **{ 'from': '2020-08-10', 'to': '2020-08-20', 'with_predictions': True, }) self.assertIndexResponse(res, 'cluster_task') # Endpoint ROI validations @vcr.use_cassette('endpoint_roi_validations/create.yaml') def test_create_endpoint_roi_validations(self, sleep_mock): res = self.client.create_endpoint_roi_validation( endpoint_id='8cbb0609-f0b9-42a6-952a-373c20ab7806', expressions=['P = true', '$1000', '@if(A = true, $1050, $0)'], ) self.assertResourceResponse(res, 'cluster_task') # Experiments @vcr.use_cassette('experiments/show.yaml') def test_get_experiment(self, sleep_mock): res = self.client.get_experiment('0138f8da7adf76') self.assertResourceResponse(res, 'experiment') @vcr.use_cassette('experiments/index.yaml') def test_get_experiments(self, sleep_mock): res = self.client.get_experiments() self.assertIndexResponse(res, 'experiment') @vcr.use_cassette('experiments/create_valid.yaml') def test_create_experiment_valid(self, sleep_mock): res = self.client.create_experiment(id='a0138f7adf78d6', dataset_manifest_id='c993ab1107', name='Some sort of experiment') self.assertResourceResponse(res, 'experiment') @vcr.use_cassette('experiments/update_valid.yaml') def test_update_experiment_valid(self, sleep_mock): res = self.client.update_experiment('a0138f7adf78d6', name='Real experiment') self.assertResourceResponse(res, 'experiment') @vcr.use_cassette('experiments/delete_valid.yaml') def test_delete_experiment_valid(self, sleep_mock): res = self.client.delete_experiment('a0138f7adf78d6') self.assertResourceResponse(res, 'experiment') @vcr.use_cassette('experiments/delete_not_existing.yaml') def test_delete_not_existing(self, sleep_mock): with self.assertRaises(HubApiClient.FatalApiError) as context: self.client.delete_experiment('a0138f7adf78d6') # Experiment Sessions @vcr.use_cassette('experiment_sessions/show.yaml') def test_get_experiment_session(self, sleep_mock): res = self.client.get_experiment_session('5772fd1ec439b7') self.assertResourceResponse(res, 'experiment_session') @vcr.use_cassette('experiment_sessions/index.yaml') def test_get_experiment_sessions(self, sleep_mock): res = self.client.get_experiment_sessions() self.assertIndexResponse(res, 'experiment_session') @vcr.use_cassette('experiment_sessions/index_with_filter.yaml') def test_get_experiment_sessions_with_filter(self, sleep_mock): res = self.client.get_experiment_sessions(project_id=31) self.assertIndexResponse(res, 'experiment_session') @vcr.use_cassette('experiment_sessions/create_malformed_json.yaml') def test_create_experiment_session_malformed_json(self, sleep_mock): with self.assertRaises(HubApiClient.FatalApiError) as context: self.client.create_experiment_session( id='1984368722', experiment_id='afaf-dfgdfhg-gdfgdg', dataset_manifest_id='c993ab1107', status='running', leaderbord={ 'a1': 1, 'a2': 2 }, model_settings={'x': float('inf')}, message='Some sort of message') @vcr.use_cassette('experiment_sessions/create_valid.yaml') def test_create_experiment_session_valid(self, sleep_mock): res = self.client.create_experiment_session( id='1984368722', experiment_id='0138f8da7adf76', dataset_manifest_id='c993ab1107', status='started', leaderbord={ 'a1': 1, 'a2': 2 }, model_settings={ 'x': 1, 'y': 2 }, message='Some sort of message') self.assertResourceResponse(res, 'experiment_session') @vcr.use_cassette('experiment_sessions/update_valid.yaml') def test_update_experiment_session_valid(self, sleep_mock): res = self.client.update_experiment_session('1984368722', status='completed') self.assertResourceResponse(res, 'experiment_session') # Hyperparameters @vcr.use_cassette('hyperparameters/show.yaml') def test_get_hyperparameter(self, sleep_mock): res = self.client.get_hyperparameter(1) self.assertResourceResponse(res, 'hyperparameter') @vcr.use_cassette('hyperparameters/index.yaml') def test_get_hyperparameters(self, sleep_mock): res = self.client.get_hyperparameters() self.assertIndexResponse(res, 'hyperparameter') @vcr.use_cassette('hyperparameters/create_valid.yaml') def test_create_hyperparameter_valid(self, sleep_mock): res = self.client.create_hyperparameter(algorithm_name='SVM', algorithm_params={ 'x': 1, 'y': 2, }) self.assertResourceResponse(res, 'hyperparameter') # Instance types @vcr.use_cassette('instance_types/index.yaml') def test_get_instance_types(self, sleep_mock): res = self.client.get_instance_types() self.assertIndexResponse(res, 'instance_type') # Organizations @vcr.use_cassette('organizations/show.yaml') def test_get_organization(self, sleep_mock): res = self.client.get_organization(23) self.assertResourceResponse(res, 'organization') @vcr.use_cassette('organizations/index.yaml') def test_get_organizations(self, sleep_mock): res = self.client.get_organizations() self.assertIndexResponse(res, 'organization') @vcr.use_cassette('organizations/create_valid.yaml') def test_create_organization_valid(self, sleep_mock): res = self.client.create_organization(name='my-organization') self.assertResourceResponse(res, 'organization') @vcr.use_cassette('organizations/update_valid.yaml') def test_update_organization_valid(self, sleep_mock): res = self.client.update_organization( 50, role_to_assume_arn= 'arn:aws:iam::529880471834:role/auger-hub-role-alex') self.assertResourceResponse(res, 'organization') @vcr.use_cassette('organizations/delete_valid.yaml') def test_delete_organization_valid(self, sleep_mock): res = self.client.delete_organization(50) self.assertResourceResponse(res, 'organization') # Pipelines @vcr.use_cassette('pipelines/show.yaml') def test_get_pipeline(self, sleep_mock): res = self.client.get_pipeline('c356ff9b6ecc7364') self.assertResourceResponse(res, 'pipeline') @vcr.use_cassette('pipelines/index.yaml') def test_get_pipelines(self, sleep_mock): res = self.client.get_pipelines() self.assertIndexResponse(res, 'pipeline') @vcr.use_cassette('pipelines/create_valid.yaml') def test_create_pipeline_valid(self, sleep_mock): res = self.client.create_pipeline(trial_id='1231231') self.assertResourceResponse(res, 'pipeline') @vcr.use_cassette('pipelines/update_valid.yaml') def test_update_pipeline_valid(self, sleep_mock): res = self.client.update_pipeline('c356ff9b6ecc7364', status='packaging') self.assertResourceResponse(res, 'pipeline') # Pipeline file @vcr.use_cassette('pipeline_files/show.yaml') def test_get_pipeline_file(self, sleep_mock): res = self.client.get_pipeline_file(2) self.assertResourceResponse(res, 'pipeline_file') @vcr.use_cassette('pipeline_files/index.yaml') def test_get_pipeline_files(self, sleep_mock): res = self.client.get_pipeline_files() self.assertIndexResponse(res, 'pipeline_file') @vcr.use_cassette('pipeline_files/create_valid.yaml') def test_create_pipeline_file_valid(self, sleep_mock): res = self.client.create_pipeline_file(trial_id='3D1E99741D37422') self.assertResourceResponse(res, 'pipeline_file') # Predictions @vcr.use_cassette('predictions/show.yaml') def test_get_prediction(self, sleep_mock): res = self.client.get_prediction(2) self.assertResourceResponse(res, 'prediction') @vcr.use_cassette('predictions/index.yaml') def test_get_predictions(self, sleep_mock): res = self.client.get_predictions() self.assertIndexResponse(res, 'prediction') @vcr.use_cassette('actuals/create_valid.yaml') def test_create_actuals_valid(self, sleep_mock): res = self.client.create_actual(pipeline_id='46188658d308607a', actuals=[{ 'prediction_id': '1', 'actual': 1 }, { 'prediction_id': '2', 'actual': 1 }]) self.assertResourceEmptyResponse(res, 'actual') @vcr.use_cassette('actuals/delete_valid.yaml') def test_delete_actuals_valid(self, sleep_mock): res = self.client.delete_actuals( **{ 'pipeline_id': '118DCF6B1A2A44A', 'from': '2020-08-10', 'to': '2020-08-20', 'with_predictions': True }) self.assertResourceResponse(res, 'cluster_task') @vcr.use_cassette('predictions/create_valid_with_nils.yaml') def test_create_prediction_valid_with_nils(self, sleep_mock): res = self.client.create_prediction(pipeline_id='46188658d308607a', records=[[1.1, 1.2, 1.3], [2.1, None, None]], features=['x1', 'x2', 'x3']) self.assertResourceResponse(res, 'prediction') @vcr.use_cassette('predictions/create_invalid.yaml') def test_create_prediction_invalid(self, sleep_mock): with self.assertRaises(HubApiClient.InvalidParamsError) as context: self.client.create_prediction(pipeline_id='46188658d308607a', records=[[1.1, 1.2, 1.3], [2.1, 2.2]], features=['x1', 'x2', 'x3']) self.assertIn( 'each records size should be equal to the features count', str(context.exception)) self.assertIn('"request_params": {', str(context.exception)) self.assertIsInstance(context.exception.metadata()['request_params'], dict) self.assertEqual( json.dumps(context.exception.metadata()['request_params']), '{"pipeline_id": "46188658d308607a", "records": [[1.1, 1.2, 1.3], [2.1, 2.2]],' ' "features": ["x1", "x2", "x3"], "project_api_token": "****",' ' "prediction": {"pipeline_id": "46188658d308607a"}, "project_id": 1}' ) # Response from server in dev mode @vcr.use_cassette('predictions/create_invalid_with_nans.yaml') def test_create_prediction_invalid_with_nans(self, sleep_mock): with self.assertRaises(HubApiClient.FatalApiError) as context: self.client.create_prediction( pipeline_id='46188658d308607a', records=[[1.1, 1.2, 1.3], [2.1, float('NaN'), float('NaN')]], features=['x1', 'x2', 'x3']) self.assertIn('ActionDispatch::Http::Parameters::ParseError', str(context.exception)) # Response from server in production mode @vcr.use_cassette('predictions/create_invalid_with_nan_blank_response.yaml' ) def test_create_prediction_invalid_with_nans_blank_response( self, sleep_mock): with self.assertRaises(HubApiClient.FatalApiError) as context: self.client.create_prediction( pipeline_id='46188658d308607a', records=[[1.1, 1.2, 1.3], [2.1, float('NaN'), float('NaN')]], features=['x1', 'x2', 'x3']) self.assertIn('Bad Request', str(context.exception)) # Prediction groups @vcr.use_cassette('prediction_groups/show.yaml') def test_get_prediction_group(self, sleep_mock): res = self.client.get_prediction_group(7) self.assertResourceResponse(res, 'prediction_group') @vcr.use_cassette('prediction_groups/index.yaml') def test_get_prediction_groups(self, sleep_mock): res = self.client.get_prediction_groups() self.assertIndexResponse(res, 'prediction_group') @vcr.use_cassette('prediction_groups/create_valid.yaml') def test_create_prediction_group_valid(self, sleep_mock): res = self.client.create_prediction_group( pipeline_id='118DCF6B1A2A44A', records=[[1.1, 1.2, 1.3, 1.4]], features=[ 'sepal_length', 'sepal_length', 'petal_length', 'petal_width' ]) self.assertResourceResponse(res, 'prediction_group') # Projects @vcr.use_cassette('projects/show.yaml') def test_get_project(self, sleep_mock): res = self.client.get_project(31) self.assertResourceResponse(res, 'project') @vcr.use_cassette('projects/index.yaml') def test_get_projects(self, sleep_mock): res = self.client.get_projects() self.assertIndexResponse(res, 'project') @vcr.use_cassette('projects/create_valid.yaml') def test_create_project_valid(self, sleep_mock): res = self.client.create_project(name='my-project', organization_id=23) self.assertResourceResponse(res, 'project') @vcr.use_cassette('projects/update_valid.yaml') def test_update_project_valid(self, sleep_mock): res = self.client.update_project( 43, default_worker_nodes_count=5, default_kubernetes_stack='experimental') self.assertResourceResponse(res, 'project') @vcr.use_cassette('projects/delete_valid.yaml') def test_delete_project_valid(self, sleep_mock): res = self.client.delete_project(43) self.assertResourceResponse(res, 'project') @vcr.use_cassette('projects/get_logs.yaml') def test_get_project_logs(self, sleep_mock): res = self.client.get_project_logs(31) self.assertIsInstance(res, string_type) # Pod logs for project @vcr.use_cassette('pod_logs/index.yaml') def test_get_pod_logs(self, sleep_mock): res = self.client.get_pod_logs(project_id=696) self.assertIndexResponse(res, 'pod_log') @vcr.use_cassette('projects/deploy_project.yaml') def test_deploy_project(self, sleep_mock): res = self.client.deploy_project(1, worker_type_id=1, workers_count=2) self.assertResourceResponse(res, 'project') @vcr.use_cassette('projects/undeploy_project.yaml') def test_deploy_project(self, sleep_mock): res = self.client.undeploy_project(1) self.assertResourceResponse(res, 'project') # Project files @vcr.use_cassette('project_files/show.yaml') def test_get_project_file(self, sleep_mock): res = self.client.get_project_file(68) self.assertResourceResponse(res, 'project_file') @vcr.use_cassette('project_files/index.yaml') def test_get_project_files(self, sleep_mock): res = self.client.get_project_files() self.assertIndexResponse(res, 'project_file') @vcr.use_cassette('project_files/create_valid.yaml') def test_create_project_file_valid(self, sleep_mock): res = self.client.create_project_file(file_name='my-project_file.csv', name='File with data', project_id=31, url='files/data.csv') self.assertResourceResponse(res, 'project_file') @vcr.use_cassette('project_files/delete_valid.yaml') def test_delete_project_file_valid(self, sleep_mock): res = self.client.delete_project_file(69) self.assertResourceResponse(res, 'project_file') # Project file URLs @vcr.use_cassette('project_file_urls/create_valid.yaml') def test_create_project_file_url_valid(self, sleep_mock): res = self.client.create_project_file_url( project_id=1, file_path='workspace/projects/mt-test/files/test.csv', ) self.assertEqual(res['meta']['status'], 200) self.assertIsInstance(res['data'], dict) self.assertTrue( re.match(r'https://[\w\d\-]+.s3.[\w\d\-]+.amazonaws.com', res['data']['url'])) self.assertIsInstance(res['data']['fields'], dict) self.assertEqual(res['data']['fields']['key'], 'workspace/projects/mt-test/files/test.csv') @vcr.use_cassette('project_file_urls/create_valid_relative.yaml') def test_create_project_file_url_valid_relative(self, sleep_mock): res = self.client.create_project_file_url( project_id=1, file_path='test.csv', ) self.assertEqual(res['meta']['status'], 200) self.assertIsInstance(res['data'], dict) self.assertTrue( re.match(r'https://[\w\d\-]+.s3.[\w\d\-]+.amazonaws.com', res['data']['url'])) self.assertIsInstance(res['data']['fields'], dict) self.assertEqual(res['data']['fields']['key'], 'workspace/projects/mt-test/files/test.csv') @vcr.use_cassette('project_file_urls/show.yaml') def test_show_project_file_url(self, sleep_mock): res = self.client.get_project_file_url( project_id=1, file_path='test.csv', ) self.assertEqual(res['meta']['status'], 200) self.assertIsInstance(res['data'], dict) matcher = r'https://[\w\d\-]+.s3.[\w\d\-]+.amazonaws.com.*files/test.csv?' self.assertTrue(re.match(matcher, res['data']['url'])) # Review alerts @vcr.use_cassette('review_alerts/show.yaml') def test_get_review_alert(self, sleep_mock): res = self.client.get_review_alert(2) self.assertResourceResponse(res, 'review_alert') @vcr.use_cassette('review_alerts/index.yaml') def test_get_review_alerts(self, sleep_mock): res = self.client.get_review_alerts() self.assertIndexResponse(res, 'review_alert') @vcr.use_cassette('review_alerts/create_valid.yaml') def test_create_review_alert_valid(self, sleep_mock): res = self.client.create_review_alert( endpoint_id='ddc968ac-43d5-4aa4-9929-1edba7cefc8f', kind='runtime_errors_burst', threshold=3, sensitivity=48, actions='retrain', ) self.assertResourceResponse(res, 'review_alert') @vcr.use_cassette('review_alerts/update_valid.yaml') def test_update_review_alert_valid(self, sleep_mock): res = self.client.update_review_alert(3, threshold=4) self.assertResourceResponse(res, 'review_alert') @vcr.use_cassette('review_alerts/delete.yaml') def test_delete_review_alert_valid(self, sleep_mock): res = self.client.delete_review_alert(3) self.assertResourceResponse(res, 'review_alert') # Review alerts @vcr.use_cassette('review_alert_items/show.yaml') def test_get_review_alert_item(self, sleep_mock): res = self.client.get_review_alert_item(2) self.assertResourceResponse(res, 'review_alert_item') @vcr.use_cassette('review_alert_items/index.yaml') def test_get_review_alert_items(self, sleep_mock): res = self.client.get_review_alert_items() self.assertIndexResponse(res, 'review_alert_item') # Similar trials requests @vcr.use_cassette('similar_trials_requests/show.yaml') def test_get_similar_trials_request(self, sleep_mock): res = self.client.get_similar_trials_request(1) self.assertResourceResponse(res, 'similar_trials_request') @vcr.use_cassette('similar_trials_requests/create_valid.yaml') def test_create_similar_trials_request_valid(self, sleep_mock): res = self.client.create_similar_trials_request( dataset_manifest_id='fab484c53aec74cf', algorithm_name='auger_ml.ensembles.algorithms.VotingAlgorithm', algorithm_params_hash='2DD51EA7809C45B', limit=5) self.assertResourceResponse(res, 'similar_trials_request') # Status @vcr.use_cassette('status/get_status_project_valid.yaml') def test_get_status_project_valid(self, sleep_mock): res = self.client.get_status(object='Project', id=1) self.assertDataResponse(res, {'status': 'undeployed'}) @vcr.use_cassette('status/get_status_invalid.yaml') def test_get_status_invalid(self, sleep_mock): with self.assertRaises(HubApiClient.InvalidParamsError) as context: res = self.client.get_status(object='User', id=1) self.assertInvalidParams(context.exception.metadata(), ['object']) # Tokens @vcr.use_cassette('tokens/create_valid.yaml') def test_create_token_valid(self, sleep_mock): res = self.client.create_token(email='*****@*****.**', password='******') self.assertDataResponse(res, { 'token': string_type, 'confirmation_required': bool }) @vcr.use_cassette('tokens/create_invalid.yaml') def test_create_token_invalid(self, sleep_mock): with self.assertRaises(HubApiClient.FatalApiError) as context: self.client.create_token(email='*****@*****.**', password='******') self.assertUnauthenticatedResponse(context.exception.metadata()) # Trials @vcr.use_cassette('trials/show.yaml') def test_get_trial(self, sleep_mock): res = self.client.get_trial('1231231') self.assertResourceResponse(res, 'trial') @vcr.use_cassette('trials/index.yaml') def test_get_trials(self, sleep_mock): res = self.client.get_trials() self.assertIndexResponse(res, 'trial') @vcr.use_cassette('trials/create.yaml') def test_create_trial(self, sleep_mock): res = self.client.create_trial( id='bb9887bc5b', refit_data_path='s3://some_new_path.csv') self.assertResourceResponse(res, 'trial') assert res['data']['id'] != 'bb9887bc5b' assert res['data']['raw_data'][ 'refit_data_path'] == 's3://some_new_path.csv' @vcr.use_cassette('trials/create.yaml') def test_refit_trial(self, sleep_mock): res = self.client.refit_trial(id='bb9887bc5b', refit_data_path='s3://some_new_path.csv') self.assertResourceResponse(res, 'trial') assert res['data']['id'] != 'bb9887bc5b' assert res['data']['raw_data'][ 'refit_data_path'] == 's3://some_new_path.csv' @vcr.use_cassette('trials/update_one_valid.yaml') def test_update_trial_valid(self, sleep_mock): res = self.client.update_trial('1231231', task_type='classification', evaluation_type='fastest one', score_name='strict one', score_value=99.9, hyperparameter={ 'algorithm_name': 'SVM', 'algorithm_params_hash': 'svm-07.333', 'algorithm_params': { 'x': 1, 'y': 2, } }) self.assertResourceResponse(res, 'trial') @vcr.use_cassette('trials/update_valid.yaml') def test_update_trials_valid(self, sleep_mock): res = self.client.update_trials( experiment_session_id='a2f99b48b6cc5541', trials=[{ 'crossValidationFolds': 5, 'uid': '3D1E99741D37422', 'classification': True, 'score': 0.96, 'score_name': 'accuracy', 'task_type': 'regression', 'algorithm_name': 'sklearn.ensemble.ExtraTreesClassifier', 'algorithm_params_hash': 'etc-55.777', 'algorithm_params': { 'bootstrap': False, 'min_samples_leaf': 11, 'n_estimators': 100, 'max_features': 0.9907161412382496, 'criterion': 'gini', 'min_samples_split': 6 } }]) self.assertIndexResponse(res, 'trial') # Trial searches def trial_history(self): return [{ 'uid': '83D2FE5F2B0A42F', 'score': 0.43478260869565216, 'evaluation_time': 0.40130186080932617, 'algorithm_name': 'sklearn.ensemble.RandomForestClassifier', 'algorithm_params': { 'bootstrap': True, 'max_features': 0.7951475142804721, 'min_samples_leaf': 13, 'min_samples_split': 18, 'n_estimators': 219, 'n_jobs': 1 } }, { 'uid': 'EAD1243A66A7419', 'score': 0.5869565217391305, 'ratio': 1.0, 'evaluation_time': 0.37141990661621094, 'algorithm_name': 'sklearn.ensemble.RandomForestClassifier', 'algorithm_params': { 'bootstrap': True, 'max_features': 0.5384972025139909, 'min_samples_leaf': 8, 'min_samples_split': 12, 'n_estimators': 188, 'n_jobs': 1 } }] @vcr.use_cassette('trial_searches/show.yaml') def test_get_trial_search(self, sleep_mock): res = self.client.get_trial_search('1') self.assertResourceResponse(res, 'trial_search') @vcr.use_cassette('trial_searches/index.yaml') def test_get_trial_searches(self, sleep_mock): res = self.client.get_trial_searches() self.assertIndexResponse(res, 'trial_search') @vcr.use_cassette('trial_searches/create_valid.yaml') def test_create_trial_search_valid(self, sleep_mock): res = self.client.create_trial_search( trials_total_count=125, search_space={ "version": 1, "optimizers_space": { "auger_ml.optimizers.pso_optimizer.PSOOptimizer": { "phig": 0.5, "phip": 0.5, "omega": 0.5 } } }, dataset_metafeatures={ "ClassEntropy": 1.584962500721156, "Dimensionality": 0.02666666666666667, "AutoCorrelation": 0.9865771812080537, "NumberOfClasses": 3, }) self.assertResourceResponse(res, 'trial_search') @vcr.use_cassette('trial_searches/update_valid.yaml') def test_update_trial_search_valid(self, sleep_mock): res = self.client.update_trial_search( id=1, trials_limit=12, trials_history=self.trial_history()) self.assertResourceResponse(res, 'trial_search') @vcr.use_cassette('trial_searches/update_invalid.yaml') def test_update_trial_search_invalid(self, sleep_mock): with self.assertRaises(HubApiClient.InvalidParamsError) as context: res = self.client.update_trial_search( id=1, trials_limit=12, trials_history=self.trial_history()) self.assertInvalidParams(context.exception.metadata(), ['id']) self.assertIn( 'trial search can be continued only when previous iteration is done', str(context.exception)) # Warm start requests @vcr.use_cassette('warm_start_requests/show.yaml') def test_get_warm_start_request(self, sleep_mock): res = self.client.get_warm_start_request(1) self.assertResourceResponse(res, 'warm_start_request') @vcr.use_cassette('warm_start_requests/create_valid.yaml') def test_create_warm_start_request_valid(self, sleep_mock): res = self.client.create_warm_start_request( dataset_manifest_id='100500', score_name='logloss') self.assertResourceResponse(res, 'warm_start_request') # Optimizers srvice # For `SECRET_KEY=strong-secret` def build_hub_client_for_optimizer(self, optimizers_url='http://*****:*****@vcr.use_cassette('optimizers_service/get_next_trials_invalid_token.yaml') def test_get_next_trials_invalid_token(self, sleep_mock): client = self.build_hub_client_for_optimizer(token='wrong') with self.assertRaises(HubApiClient.FatalApiError) as context: client.get_next_trials({'x': 1}) self.assertUnauthenticatedResponse(context.exception.metadata()) @vcr.use_cassette('optimizers_service/get_next_trials_valid.yaml') def test_get_next_trials_valid(self, sleep_mock): client = self.build_hub_client_for_optimizer() with open('tests/fixtures/get_next_trials_payload.json', 'r') as file: res = client.get_next_trials(json.loads(file.read())) @vcr.use_cassette('optimizers_service/get_next_trials_invalid_request.yaml' ) def test_get_next_trials_invalid_request(self, sleep_mock): client = self.build_hub_client_for_optimizer() with self.assertRaises(HubApiClient.InvalidParamsError) as context: res = client.get_next_trials({'x': 'some'}) self.assertInvalidParams(context.exception.metadata(), ['optimizer_name']) # POST /v2/next_trials @vcr.use_cassette('optimizers_service/get_next_trials_v2_valid.yaml') def test_get_next_trials_v2_valid(self, sleep_mock): client = self.build_hub_client_for_optimizer() with open('tests/fixtures/get_next_trials_v2_payload.json', 'r') as file: res = client.get_next_trials_v2(json.loads(file.read())) # POST /fte @vcr.use_cassette('optimizers_service/get_fte_valid.yaml') def test_get_fte_valid(self, sleep_mock): client = self.build_hub_client_for_optimizer() payload = { 'alg_name': "sklearn.ensemble.RandomForestClassifier", 'alg_params': { "bootstrap": True, "min_samples_leaf": 13, "n_estimators": 100, "min_samples_split": 3, "criterion": "gini", "max_features": 0.08361531837907793 }, 'ncols': 100, 'nrows': 10000 } res = client.get_fte(payload) self.assertIsInstance(res['data']['estimated_time'], float) @vcr.use_cassette('optimizers_service/get_fte_invalid.yaml') def test_get_fte_invalid(self, sleep_mock): client = self.build_hub_client_for_optimizer() payload = { 'alg_name': "sklearn.ensemble.RandomForestClassifier", 'ncols': 100, 'nrows': 10000 } with self.assertRaises(HubApiClient.InvalidParamsError) as context: res = client.get_fte(payload) self.assertInvalidParams(context.exception.metadata(), ['alg_params'])