def test_restore_runs(mocker): restore_runs_response = mocker.Mock() mocker.patch.object(ExperimentClient, "_post", return_value=restore_runs_response) response_schema_mock = mocker.patch( "faculty.clients.experiment._RestoreExperimentRunsResponseSchema") filter_schema_mock = mocker.patch( "faculty.clients.experiment._FilterSchema") filter_dump_mock = filter_schema_mock.return_value.dump run_ids = [uuid4(), uuid4()] client = ExperimentClient(mocker.Mock(), mocker.Mock()) response = client.restore_runs(PROJECT_ID, run_ids) assert response == restore_runs_response expected_filter = CompoundFilter( LogicalOperator.OR, [ RunIdFilter(ComparisonOperator.EQUAL_TO, run_ids[0]), RunIdFilter(ComparisonOperator.EQUAL_TO, run_ids[1]), ], ) filter_dump_mock.assert_called_once_with(expected_filter) ExperimentClient._post.assert_called_once_with( "/project/{}/run/restore/query".format(PROJECT_ID), response_schema_mock.return_value, json={"filter": filter_dump_mock.return_value}, )
def test_log_run_data_empty(mocker): mocker.patch.object(ExperimentClient, "_patch_raw") client = ExperimentClient(mocker.Mock(), mocker.Mock()) client.log_run_data(PROJECT_ID, EXPERIMENT_RUN_ID) ExperimentClient._patch_raw.assert_not_called()
def test_log_run_data(mocker): mocker.patch.object(ExperimentClient, "_patch_raw") run_data_schema_mock = mocker.patch( "faculty.clients.experiment._ExperimentRunDataSchema") run_data_dump_mock = run_data_schema_mock.return_value.dump metric = mocker.Mock() param = mocker.Mock() tag = mocker.Mock() client = ExperimentClient(mocker.Mock(), mocker.Mock()) client.log_run_data( PROJECT_ID, EXPERIMENT_RUN_ID, metrics=[metric], params=[param], tags=[tag], ) run_data_schema_mock.assert_called_once_with() run_data_dump_mock.assert_called_once_with({ "metrics": [metric], "params": [param], "tags": [tag] }) ExperimentClient._patch_raw.assert_called_once_with( "/project/{}/run/{}/data".format(PROJECT_ID, EXPERIMENT_RUN_ID), json=run_data_dump_mock.return_value, )
def test_update_run_info(mocker): run = mocker.Mock() mocker.patch.object(ExperimentClient, "_patch", return_value=run) run_schema_mock = mocker.patch( "faculty.clients.experiment._ExperimentRunSchema") run_info_schema_mock = mocker.patch( "faculty.clients.experiment._ExperimentRunInfoSchema") run_info_dump_mock = run_info_schema_mock.return_value.dump status = mocker.Mock() ended_at = mocker.Mock() client = ExperimentClient(mocker.Mock(), mocker.Mock()) returned_run = client.update_run_info(PROJECT_ID, EXPERIMENT_RUN_ID, status, ended_at) assert returned_run == run run_schema_mock.assert_called_once_with() run_info_schema_mock.assert_called_once_with() run_info_dump_mock.assert_called_once_with({ "status": status, "ended_at": ended_at }) ExperimentClient._patch.assert_called_once_with( "/project/{}/run/{}/info".format(PROJECT_ID, EXPERIMENT_RUN_ID), run_schema_mock.return_value, json=run_info_dump_mock.return_value, )
def test_experiment_client_query_runs(mocker): list_response = mocker.Mock() mocker.patch.object(ExperimentClient, "_post", return_value=list_response) response_schema_mock = mocker.patch( "faculty.clients.experiment._ListExperimentRunsResponseSchema") request_schema_mock = mocker.patch( "faculty.clients.experiment._RunQuerySchema") request_dump_mock = request_schema_mock.return_value.dump filter = mocker.Mock() sort = mocker.Mock() client = ExperimentClient(mocker.Mock(), mocker.Mock()) list_result = client.query_runs(PROJECT_ID, filter, sort, start=20, limit=10) assert list_result == list_response request_dump_mock.assert_called_once_with( RunQuery(filter, sort, Page(20, 10))) response_schema_mock.assert_called_once_with() ExperimentClient._post.assert_called_once_with( "/project/{}/run/query".format(PROJECT_ID), response_schema_mock.return_value, json=request_dump_mock.return_value, )
def test_experiment_client_list_runs(mocker): mocker.patch.object(ExperimentClient, "query_runs") client = ExperimentClient(mocker.Mock(), mocker.Mock()) response = client.list_runs( PROJECT_ID, experiment_ids=[123, 456], lifecycle_stage=LifecycleStage.DELETED, start=20, limit=10, ) assert response == ExperimentClient.query_runs.return_value expected_filter = CompoundFilter( LogicalOperator.AND, [ CompoundFilter( LogicalOperator.OR, [ ExperimentIdFilter(ComparisonOperator.EQUAL_TO, 123), ExperimentIdFilter(ComparisonOperator.EQUAL_TO, 456), ], ), DeletedAtFilter(ComparisonOperator.DEFINED, True), ], ) ExperimentClient.query_runs.assert_called_once_with( PROJECT_ID, expected_filter, None, 20, 10)
def test_restore(mocker): mocker.patch.object(ExperimentClient, "_put_raw") client = ExperimentClient(mocker.Mock(), mocker.Mock()) client.restore(PROJECT_ID, EXPERIMENT_ID) ExperimentClient._put_raw.assert_called_once_with( "/project/{}/experiment/{}/restore".format(PROJECT_ID, EXPERIMENT_ID))
def test_experiment_client_list_runs_defaults(mocker): mocker.patch.object(ExperimentClient, "query_runs") client = ExperimentClient(mocker.Mock(), mocker.Mock()) response = client.list_runs(PROJECT_ID) assert response == ExperimentClient.query_runs.return_value ExperimentClient.query_runs.assert_called_once_with( PROJECT_ID, None, None, None, None)
def test_restore_runs_empty_list(mocker): mocker.patch.object(ExperimentClient, "_post") client = ExperimentClient(mocker.Mock(), mocker.Mock()) response = client.restore_runs(PROJECT_ID, run_ids=[]) ExperimentClient._post.assert_not_called() assert len(response.restored_run_ids) == 0 assert len(response.conflicted_run_ids) == 0
def test_experiment_client_create_name_conflict(mocker): error_code = "experiment_name_conflict" exception = Conflict(mocker.Mock(), mocker.Mock(), error_code) mocker.patch.object(ExperimentClient, "_post", side_effect=exception) client = ExperimentClient(mocker.Mock(), mocker.Mock()) with pytest.raises(ExperimentNameConflict, match="name 'experiment name' already exists"): client.create(PROJECT_ID, "experiment name")
def test_experiment_client_update_name_conflict(mocker): error_code = "experiment_name_conflict" exception = Conflict(mocker.Mock(), mocker.Mock(), error_code) mocker.patch.object(ExperimentClient, "_patch_raw", side_effect=exception) client = ExperimentClient(mocker.Mock(), mocker.Mock()) with pytest.raises(ExperimentNameConflict, match="name 'new name' already exists"): client.update(PROJECT_ID, EXPERIMENT_ID, name="new name")
def test_log_run_data_other_conflict(mocker): response_mock = mocker.Mock() exception = Conflict(response_mock, "", "") mocker.patch.object(ExperimentClient, "_patch_raw", side_effect=exception) client = ExperimentClient(mocker.Mock(), mocker.Mock()) with pytest.raises(Conflict): client.log_run_data(PROJECT_ID, EXPERIMENT_RUN_ID, params=[mocker.Mock()])
def test_restore_runs_no_run_ids(mocker): mocker.patch.object(ExperimentClient, "_post") schema_mock = mocker.patch( "faculty.clients.experiment._RestoreExperimentRunsResponseSchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) client.restore_runs(PROJECT_ID) ExperimentClient._post.assert_called_once_with( "/project/{}/run/restore/query".format(PROJECT_ID), schema_mock.return_value, json={}, )
def test_experiment_client_list(mocker): experiment = mocker.Mock() mocker.patch.object(ExperimentClient, "_get", return_value=[experiment]) schema_mock = mocker.patch("faculty.clients.experiment._ExperimentSchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) assert client.list(PROJECT_ID) == [experiment] schema_mock.assert_called_once_with(many=True) ExperimentClient._get.assert_called_once_with( "/project/{}/experiment".format(PROJECT_ID), schema_mock.return_value, params={}, )
def test_experiment_client_get(mocker): experiment = mocker.Mock() mocker.patch.object(ExperimentClient, "_get", return_value=experiment) schema_mock = mocker.patch("faculty.clients.experiment._ExperimentSchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) returned_experiment = client.get(PROJECT_ID, EXPERIMENT_ID) assert returned_experiment == experiment schema_mock.assert_called_once_with() ExperimentClient._get.assert_called_once_with( "/project/{}/experiment/{}".format(PROJECT_ID, EXPERIMENT_ID), schema_mock.return_value, )
def test_log_run_data_param_conflict(mocker): message = "bad params" error_code = "conflicting_params" response_mock = mocker.Mock() response_mock.json.return_value = {"parameterKeys": ["bad-key"]} exception = Conflict(response_mock, message, error_code) mocker.patch.object(ExperimentClient, "_patch_raw", side_effect=exception) client = ExperimentClient(mocker.Mock(), mocker.Mock()) with pytest.raises(ParamConflict, match=message): client.log_run_data(PROJECT_ID, EXPERIMENT_RUN_ID, params=[mocker.Mock()])
def test_experiment_client_list_lifecycle_filter(mocker): experiment = mocker.Mock() mocker.patch.object(ExperimentClient, "_get", return_value=[experiment]) schema_mock = mocker.patch("faculty.clients.experiment._ExperimentSchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) returned_experiments = client.list(PROJECT_ID, lifecycle_stage=LifecycleStage.ACTIVE) assert returned_experiments == [experiment] schema_mock.assert_called_once_with(many=True) ExperimentClient._get.assert_called_once_with( "/project/{}/experiment".format(PROJECT_ID), schema_mock.return_value, params={"lifecycleStage": "active"}, )
def test_experiment_client_update(mocker, name, description): mocker.patch.object(ExperimentClient, "_patch_raw") client = ExperimentClient(mocker.Mock(), mocker.Mock()) client.update(PROJECT_ID, EXPERIMENT_ID, name=name, description=description) ExperimentClient._patch_raw.assert_called_once_with( "/project/{}/experiment/{}".format(PROJECT_ID, EXPERIMENT_ID), json={ "name": name, "description": description }, )
def test_experiment_create_run_experiment_deleted_conflict(mocker): message = "experiment deleted" error_code = "experiment_deleted" response_mock = mocker.Mock() response_mock.json.return_value = {"experimentId": 42} exception = Conflict(response_mock, message, error_code) mocker.patch.object(ExperimentClient, "_post", side_effect=exception) client = ExperimentClient(mocker.Mock(), mocker.Mock()) with pytest.raises(ExperimentDeleted, match=message): client.create_run( PROJECT_ID, EXPERIMENT_ID, name=mocker.Mock(), started_at=mocker.Mock(), parent_run_id=PARENT_RUN_ID, artifact_location=mocker.Mock(), )
def test_experiment_client_create(mocker, description, artifact_location): experiment = mocker.Mock() mocker.patch.object(ExperimentClient, "_post", return_value=experiment) schema_mock = mocker.patch("faculty.clients.experiment._ExperimentSchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) returned_experiment = client.create(PROJECT_ID, "experiment name", description, artifact_location) assert returned_experiment == experiment schema_mock.assert_called_once_with() ExperimentClient._post.assert_called_once_with( "/project/{}/experiment".format(PROJECT_ID), schema_mock.return_value, json={ "name": "experiment name", "description": description, "artifactLocation": artifact_location, }, )
def test_experiment_create_run(mocker): run = mocker.Mock() mocker.patch.object(ExperimentClient, "_post", return_value=run) request_schema_mock = mocker.patch( "faculty.clients.experiment._CreateRunSchema") dump_mock = request_schema_mock.return_value.dump response_schema_mock = mocker.patch( "faculty.clients.experiment._ExperimentRunSchema") run_name = mocker.Mock() started_at = mocker.Mock() artifact_location = mocker.Mock() client = ExperimentClient(mocker.Mock(), mocker.Mock()) returned_run = client.create_run( PROJECT_ID, EXPERIMENT_ID, run_name, started_at, PARENT_RUN_ID, artifact_location=artifact_location, ) assert returned_run == run request_schema_mock.assert_called_once_with() dump_mock.assert_called_once_with({ "name": run_name, "parent_run_id": PARENT_RUN_ID, "started_at": started_at, "artifact_location": artifact_location, "tags": [], }) response_schema_mock.assert_called_once_with() ExperimentClient._post.assert_called_once_with( "/project/{}/experiment/{}/run".format(PROJECT_ID, EXPERIMENT_ID), response_schema_mock.return_value, json=dump_mock.return_value, )
def test_get_metric_history(mocker): key = mocker.Mock() data_point_0 = mocker.Mock() data_point_1 = mocker.Mock() metric_history = mocker.Mock(key=key, history=[data_point_0, data_point_1]) mocker.patch.object(ExperimentClient, "_get", return_value=metric_history) metric_history_schema_mock = mocker.patch( "faculty.clients.experiment._MetricHistorySchema") client = ExperimentClient(mocker.Mock(), mocker.Mock()) metrics = client.get_metric_history(PROJECT_ID, EXPERIMENT_RUN_ID, "metric-key") expected = [ Metric( key=key, step=data_point_0.step, timestamp=data_point_0.timestamp, value=data_point_0.value, ), Metric( key=key, step=data_point_1.step, timestamp=data_point_1.timestamp, value=data_point_1.value, ), ] assert metrics == expected metric_history_schema_mock.assert_called_once_with() ExperimentClient._get.assert_called_once_with( "/project/{}/run/{}/metric/metric-key/history".format( PROJECT_ID, EXPERIMENT_RUN_ID), metric_history_schema_mock.return_value, )