def test_apiclient_search_solutions(search_solutions_request_mock, logger_mock): instance = TA3APIClient(port=9999) dataset = 'test_dataset' expected_value = 'response' # instance mocks instance._get_dataset_doc_path = MagicMock(return_value='dataset-doc-path') instance._build_problem = MagicMock(return_value='build-problem') instance.stub.SearchSolutions = MagicMock(return_value=expected_value) value = instance.search_solutions(dataset) assert value == expected_value search_solutions_request_mock.assert_called_once_with( user_agent='ta3_api_test.py', version='2019.6.11', time_bound_search=1., priority=0., allowed_value_types=[ ValueType.Value('RAW'), ValueType.Value('DATASET_URI'), ValueType.Value('CSV_URI'), ], inputs=[Value(dataset_uri='dataset-doc-path')], problem='build-problem') assert logger_mock.call_count == 2
def _ta3_test(args): local_input = args.input remote_input = '/input' if args.docker else args.input client = TA3APIClient(args.port, local_input, remote_input) print('### Hello ###') client.hello() if args.all: args.dataset = os.listdir(args.input) results = list() for dataset in args.dataset: try: score = _ta3_test_dataset(client, dataset, args.timeout / 60) results.append({'dataset': dataset, 'score': score}) except Exception as ex: results.append({'dataset': dataset, 'score': 'ERROR'}) print('TA3 Error on dataset {}: {}'.format(dataset, ex)) results = pd.DataFrame(results) print( tabulate.tabulate(results[['dataset', 'score']], showindex=False, tablefmt='github', headers=['dataset', 'score']))
def test_apiclient_get_fit_solution_results( get_fit_solution_results_request_mock, logger_mock): instance = TA3APIClient(port=9999) request_id = 'request-id' # 1. no solutions expected_value = [] # mocks get_fit_solution_results_request_mock.return_value = 'request' instance.stub.GetFitSolutionResults = MagicMock( return_value=expected_value) value = instance.get_fit_solution_results(request_id) get_fit_solution_results_request_mock.assert_called_once_with( request_id=request_id) instance.stub.GetFitSolutionResults.assert_called_once_with('request') assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1 # 2. two solutions logger_mock.reset_mock() expected_value = [1, 2] # mocks instance.stub.GetFitSolutionResults = MagicMock( return_value=expected_value) value = instance.get_fit_solution_results(request_id) get_fit_solution_results_request_mock.assert_called_with( request_id=request_id) instance.stub.GetFitSolutionResults.assert_called_once_with('request') assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1 # 3. two solutions but max one result logger_mock.reset_mock() expected_value = [1] # mocks instance.stub.GetFitSolutionResults = MagicMock( return_value=expected_value + [2, 3]) value = instance.get_fit_solution_results(request_id, max_results=1) get_fit_solution_results_request_mock.assert_called_with( request_id=request_id) instance.stub.GetFitSolutionResults.assert_called_once_with('request') assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1
def _ta3_test(args): local_input = args.input remote_input = '/input' if args.docker else args.input client = TA3APIClient(args.port, local_input, remote_input) print('### Hello ###') client.hello() for dataset in args.dataset: _ta3_test_dataset(client, dataset, args.timeout / 60)
def test_apiclient_get_search_solutions_results( get_search_solutions_results_request_mock, logger_mock): instance = TA3APIClient(port=9999) search_id = 'search-id' # 1. no solutions expected_value = [] # instance mocks instance.stub.GetSearchSolutionsResults = MagicMock( return_value=expected_value) value = instance.get_search_solutions_results(search_id) get_search_solutions_results_request_mock.assert_called_once_with( search_id=search_id) assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1 # 2. two solutions logger_mock.reset_mock() expected_value = [1, 2] # instance mocks instance.stub.GetSearchSolutionsResults = MagicMock( return_value=expected_value) value = instance.get_search_solutions_results(search_id) get_search_solutions_results_request_mock.assert_called_with( search_id=search_id) assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1 # 3. two solutions but max one result logger_mock.reset_mock() expected_value = [1] # instance mocks instance.stub.GetSearchSolutionsResults = MagicMock( return_value=expected_value + [2, 3]) value = instance.get_search_solutions_results(search_id, max_results=1) get_search_solutions_results_request_mock.assert_called_with( search_id=search_id) assert value == expected_value assert logger_mock.call_count == len(expected_value) + 1
def test_apiclient_get_dataset_doc_path(): # with default args remote_input = 'input' instance = TA3APIClient(port=9999) dataset = 'test_dataset' doc_path = instance._get_dataset_doc_path(dataset) expected_doc_path = 'file://{}/{}/{}/TRAIN/dataset_TRAIN/datasetDoc.json'.format( abspath('.'), remote_input, dataset) assert doc_path == expected_doc_path # custom args remote_input = 'my-remote-input' instance = TA3APIClient(port=9999, remote_input=remote_input) doc_path = instance._get_dataset_doc_path(dataset) expected_doc_path = 'file://{}/{}/{}/TRAIN/dataset_TRAIN/datasetDoc.json'.format( abspath('.'), remote_input, dataset) assert doc_path == expected_doc_path
def test_apiclient_hello(hello_request_mock, logger_mock): instance = TA3APIClient(port=9999) expected_response = 'response' # mocks hello_request_mock.return_value = 'request' instance.stub.Hello = MagicMock(return_value=expected_response) return_value = instance.hello() assert return_value == expected_response assert logger_mock.call_count == 2 hello_request_mock.assert_called_once_with() instance.stub.Hello.called_once_with('request')
def test_apiclient_list_primitives(list_primitives_request_mock, logger_mock): instance = TA3APIClient(port=9999) expected_response = 'response' # mocks list_primitives_request_mock.return_value = 'request' instance.stub.ListPrimitives = MagicMock(return_value=expected_response) return_value = instance.list_primitives() assert return_value == expected_response assert logger_mock.call_count == 2 list_primitives_request_mock.assert_called_once_with() instance.stub.ListPrimitives.called_once_with('request')
def test_apiclient_update_problem(update_problem_request_mock, logger_mock): instance = TA3APIClient(port=9999) search_id = 'search-id' expected_response = 'response' # mocks update_problem_request_mock.return_value = 'request' instance.stub.UpdateProblem = MagicMock(return_value=expected_response) return_value = instance.update_problem(search_id) assert return_value == expected_response assert logger_mock.call_count == 2 update_problem_request_mock.assert_called_once_with(search_id=search_id) instance.stub.UpdateProblem.called_once_with('request')
def test_apiclient_solution_export(solution_export_request_mock, logger_mock): instance = TA3APIClient(port=9999) solution_id = 'solution-id' rank = 'rank' expected_response = 'response' # mocks solution_export_request_mock.return_value = 'request' instance.stub.SolutionExport = MagicMock(return_value=expected_response) return_value = instance.solution_export(solution_id, rank) assert return_value == expected_response assert logger_mock.call_count == 2 solution_export_request_mock.assert_called_once_with( solution_id=solution_id, rank=rank) instance.stub.SolutionExport.called_once_with('request')
def test_apiclient_describe_solution(describe_solution_request_mock, logger_mock): instance = TA3APIClient(port=9999) solution_id = 'solution-id' expected_response = 'response' # instance mocks instance.stub.DescribeSolution = MagicMock(return_value=expected_response) describe_solution_request_mock.return_value = 'request' return_value = instance.describe_solution(solution_id) assert return_value == expected_response describe_solution_request_mock.assert_called_once_with( solution_id=solution_id) instance.stub.DescribeSolution.called_once_with('request') assert logger_mock.call_count == 2
def _ta3_test(args): local_input = args.input remote_input = '/input' if args.docker else args.input client = TA3APIClient(args.port, local_input, remote_input) print('### Hello ###') client.hello() if args.all: args.dataset = os.listdir(args.input) for dataset in args.dataset: try: _ta3_test_dataset(client, dataset, args.timeout / 60) except Exception as e: print( 'An error occurred trying to process the dataset {}, produced by {}' .format(dataset, e))
def test_apiclient_stop_search_solutions(stop_search_solutions_request_mock, logger_mock): instance = TA3APIClient(port=9999) search_id = 'search-id' expected_response = 'response' # instance mocks instance.stub.StopSearchSolutions = MagicMock( return_value=expected_response) stop_search_solutions_request_mock.return_value = 'request' return_value = instance.stop_search_solutions(search_id) assert return_value == expected_response stop_search_solutions_request_mock.assert_called_once_with( search_id=search_id) instance.stub.StopSearchSolutions.called_once_with('request') assert logger_mock.call_count == 2
def test_apiclient_score_solution(score_solution_request_mock, logger_mock): instance = TA3APIClient(port=9999) solution_id = 'solution-id' dataset = 'test-dataset' expected_response = 'response' # mocks score_solution_request_mock.return_value = 'request' problem_mock = MagicMock() problem_mock.problem.performance_metrics = 'metrics' instance._build_problem = MagicMock(return_value=problem_mock) instance._get_dataset_doc_path = MagicMock(return_value='dataset-doc-path') instance.stub.ScoreSolution = MagicMock(return_value=expected_response) return_value = instance.score_solution(solution_id, dataset) assert return_value == expected_response assert logger_mock.call_count == 2 assert score_solution_request_mock.called instance.stub.ScoreSolution.called_once_with('request')
def test_apiclient_fit_solution(fit_solution_request_mock, logger_mock): instance = TA3APIClient(port=9999) solution_id = 'solution-id' dataset = 'test-dataset' expected_response = 'response' # mocks fit_solution_request_mock.return_value = 'request' instance._get_dataset_doc_path = MagicMock(return_value='dataset-doc-path') instance.stub.FitSolution = MagicMock(return_value=expected_response) return_value = instance.fit_solution(solution_id, dataset) assert return_value == expected_response assert logger_mock.call_count == 2 fit_solution_request_mock.assert_called_once_with( solution_id=solution_id, inputs=[Value(dataset_uri='dataset-doc-path')], expose_outputs=['outputs.0'], expose_value_types=[ValueType.Value('CSV_URI')]) instance.stub.FitSolution.called_once_with('request')
def test_apiclient_get_score_solution(get_score_solution_results_request_mock, logger_mock): instance = TA3APIClient(port=9999) request_id = 'request-id' # 1. no scores scores = [] # mocks get_score_solution_results_request_mock.return_value = 'request' instance.stub.GetScoreSolutionResults = MagicMock(return_value=scores) value = instance.get_score_solution_results(request_id) get_score_solution_results_request_mock.assert_called_once_with( request_id=request_id) instance.stub.GetScoreSolutionResults.assert_called_once_with('request') assert value is None assert logger_mock.call_count == len(scores) + 1 # 2. three solutions logger_mock.reset_mock() scores = [1, 2, 3] # mocks instance.stub.GetScoreSolutionResults = MagicMock(return_value=scores) value = instance.get_score_solution_results(request_id) get_score_solution_results_request_mock.assert_called_with( request_id=request_id) instance.stub.GetScoreSolutionResults.assert_called_once_with('request') assert value == scores[-1] # the last score assert logger_mock.call_count == len(scores) + 1