def test_update_job_libraries( job_list, job_update_response_list_old, job_update_response_list_new, host, ): for job in job_update_response_list_old: responses.add( responses.GET, host + '/api/2.0/jobs/get?job_id={}'.format(job['job_id']), status=200, json=job, ) responses.add_callback( responses.POST, host + '/api/2.0/jobs/reset', callback=request_callback, ) update_job_libraries( logger, job_list, FileNameMatch('test_library-1.2.3.egg'), 'dbfs:/FileStore/jars/some_library_uri', '', host, ) assert len(responses.calls) == 2 assert (json.loads( responses.calls[1].response.text) == job_update_response_list_new[0])
def test_update_databricks_with_jar_only_upload( load_mock, caplog, prod_folder, host, cfg, ): with mock.patch('stork.update_databricks_library.CFG_FILE', cfg): update_databricks( logger, path='some/path/to/test-library-1.0.3.jar', token='', folder=prod_folder, update_jobs=False, cleanup=False, ) out = caplog.record_tuples[0][2] expected_out = 'new library test-library-1.0.3 loaded to Databricks' assert strip_whitespace(out) == strip_whitespace(expected_out) load_mock.assert_called_with( 'some/path/to/test-library-1.0.3.jar', FileNameMatch('test-library-1.0.3.jar'), prod_folder, '', host, )
def test_filename_match_egg_snapshot_branch(): match = FileNameMatch('new_library-1.0.0-SNAPSHOT-my-branch.egg') assert match.library_name == 'new_library' assert match.version == '1.0.0-SNAPSHOT-my-branch' assert match.major_version == '1' assert match.minor_version == '0.0' assert match.suffix == 'egg' assert match.lib_type == 'python-egg'
def test_filename_match_jar_snapshot_branch(): match = FileNameMatch('new_library-1.0.0-SNAPSHOT-my-branch.jar') assert match.library_name == 'new_library' assert match.version == '1.0.0-SNAPSHOT-my-branch' assert match.major_version == '1' assert match.minor_version == '0.0' assert match.suffix == 'jar' assert match.lib_type == 'java-jar'
def test_filename_match_egg(): match = FileNameMatch('new_library-1.0.0.egg') assert match.library_name == 'new_library' assert match.version == '1.0.0' assert match.major_version == '1' assert match.minor_version == '0.0' assert match.suffix == 'egg' assert match.lib_type == 'python-egg'
def test_filename_match_jar(): match = FileNameMatch('new_library-1.0.0.jar') assert match.library_name == 'new_library' assert match.version == '1.0.0' assert match.major_version == '1' assert match.minor_version == '0.0' assert match.suffix == 'jar' assert match.lib_type == 'java-jar'
def id_nums(): id_nums = { 'test-library-1.0.3': { 'name_match': FileNameMatch('test-library-1.0.3.egg'), 'id_num': '8', }, 'test-library-1.0.2': { 'name_match': FileNameMatch('test-library-1.0.2.egg'), 'id_num': '7', }, 'test-library-1.0.1': { 'name_match': FileNameMatch('test-library-1.0.1.egg'), 'id_num': '6', }, 'test-library-plus-stuff-0.0.0': { 'name_match': FileNameMatch('test-library-plus-stuff-0.0.0.egg'), 'id_num': '4', }, 'test-library-0.0.0': { 'name_match': FileNameMatch('test-library-0.0.0.egg'), 'id_num': '3', }, 'awesome_library_a-0.10.1': { 'name_match': FileNameMatch('awesome_library_a-0.10.1.egg'), 'id_num': '2', }, 'awesome_library_b-4.2.3': { 'name_match': FileNameMatch('awesome_library_b-4.2.3.egg'), 'id_num': '1', }, } return id_nums
def id_nums(): id_nums = { 'awesome_library_b-4.2.3': { 'name_match': FileNameMatch('awesome_library_b-4.2.3.egg'), 'id_num': 1, }, 'awesome_library_a-0.10.1': { 'name_match': FileNameMatch('awesome_library_a-0.10.1.egg'), 'id_num': 2, }, 'test-library-0.0.0': { 'name_match': FileNameMatch('test-library-0.0.0.egg'), 'id_num': 3, }, 'test-library-plus-stuff-0.0.0': { 'name_match': FileNameMatch('test-library-plus-stuff-0.0.0.egg'), 'id_num': 4, }, 'test-library-1.0.1': { 'name_match': FileNameMatch('test-library-1.0.1.egg'), 'id_num': 5, }, 'test-library-1.0.2': { 'name_match': FileNameMatch('test-library-1.0.2.egg'), 'id_num': 6, }, 'test-library-1.0.3': { 'name_match': FileNameMatch('test-library-1.0.3.egg'), 'id_num': 7, } } return id_nums
def test_update_databricks_update_jobs( delete_mock, update_mock, lib_mock, job_mock, load_mock, library_mapping, id_nums, job_list, caplog, prod_folder, host, cfg, ): path = 'some/path/to/test-library-1.0.3-py3.6.egg' delete_mock.return_value = ['test-library-1.0.1', 'test-library-1.0.2'] job_mock.return_value = job_list lib_mock.return_value = (library_mapping, id_nums) with mock.patch('stork.update_databricks_library.CFG_FILE', cfg): update_databricks( logger, path=path, token='', folder=prod_folder, update_jobs=True, cleanup=True, ) out = [r[2] for r in caplog.record_tuples] expected_out = [ 'new library test-library-1.0.3 loaded to Databricks', 'current major version of library used by jobs: job_3', 'updated jobs: job_3', 'removed old versions: test-library-1.0.1, test-library-1.0.2', ] match = FileNameMatch('test-library-1.0.3-py3.6.egg') assert out == expected_out load_mock.assert_called_with(path, match, prod_folder, '', host) job_mock.assert_called_with(logger, match, library_mapping, '', host) lib_mock.assert_called_with(logger, prod_folder, '', host) update_mock.assert_called_with( logger, job_list, match, 'dbfs:/FileStore/jars/47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg', '', host, ) delete_mock.assert_called_with( logger, match, id_nums=id_nums, token='', prod_folder=prod_folder, host=host, )
def test_update_databricks_already_exists( load_mock, caplog, prod_folder, host, cfg, ): responses.add( responses.GET, 'https://test-api', status=500, content_type='text/plain', json={ 'error_code': 'http 500', 'message': ( 'NameConflictException: ' 'Node named "test-library" already exists' ) }, ) res = requests.get('https://test-api') load_mock.side_effect = APIError(res) with mock.patch('stork.update_databricks_library.CFG_FILE', cfg): update_databricks( logger, path='some/path/to/test-library-1.0.1-py3.6.egg', token='', folder='/other/folder', update_jobs=False, cleanup=False, ) out = caplog.record_tuples[0][2] expected_out = ( 'This version (1.0.1) already exists: ' 'if a change has been made please update your version number. ' 'Note this error can also occur if you are uploading a jar ' 'and an egg already exists with the same name and version, ' 'or vice versa. In this case you will need to choose a ' 'different library name or a different folder for either the ' 'egg or the jar.' ) assert strip_whitespace(out) == strip_whitespace(expected_out) load_mock.assert_called_with( 'some/path/to/test-library-1.0.1-py3.6.egg', FileNameMatch('test-library-1.0.1-py3.6.egg'), '/other/folder', '', host, )
def test_load_library_jar(host, prod_folder): filename = 'test-library-1.0.3.jar' responses.add( responses.POST, host + '/api/1.2/libraries/upload', status=200, ) with mock.patch('builtins.open', mock.mock_open(read_data='jar file contents')): load_library( filename=filename, match=FileNameMatch(filename), folder=prod_folder, token='', host=host, )
def test_get_job_list(library_mapping, job_list, job_list_response, host): responses.add( responses.GET, host + '/api/2.0/jobs/list', status=200, json=job_list_response, ) match = FileNameMatch('test-library-1.1.2.egg') job_list_actual = get_job_list( logger, match=match, library_mapping=library_mapping, token='', host=host, ) assert len(responses.calls) == 1 assert job_list_actual == job_list
def library_mapping(): library_mapping = { '47fb08a7-test-library_1_0_2_py3_6-e5f8c.egg': FileNameMatch('test-library-1.0.2.egg'), '47fb08a7-test-library_1_0_1_py3_6-e5f8c.egg': FileNameMatch('test-library-1.0.1.egg'), '47fb08a7-test-library_0_0_0_py3_6-e5f8c.egg': FileNameMatch('test-library-0.0.0.egg'), '01832402-test-library-plus-stuff_0_0_0_py3_6-e5f8c.egg': FileNameMatch('test-library-plus-stuff-0.0.0.egg'), '996c949b-awesome_library_a_0_10_1_py3_6-266f.egg': FileNameMatch('awesome_library_a-0.10.1.egg'), '47fb08a7-awesome_library_b_4_2_3_py3_6-e5f8c.egg': FileNameMatch('awesome_library_b-4.2.3.egg'), '47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg': FileNameMatch('test-library-1.0.3.egg'), } return library_mapping
def test_load_library_APIError(host, prod_folder): filename = 'test-library-1.0.3-py3.6.egg' responses.add( responses.POST, host + '/api/1.2/libraries/upload', status=401, ) with pytest.raises(APIError) as err: with mock.patch('builtins.open', mock.mock_open(read_data='egg file contents')): load_library( filename=filename, match=FileNameMatch(filename), folder=prod_folder, token='', host=host, ) assert err.code == 'http 401'
def test_update_databricks_wrong_folder(load_mock, caplog, host, cfg): with mock.patch('stork.update_databricks_library.CFG_FILE', cfg): update_databricks( logger, path='some/path/to/test-library-1.0.3-py3.6.egg', token='', folder='/other/folder', update_jobs=True, cleanup=True, ) out = caplog.record_tuples[0][2] expected_out = 'new library test-library-1.0.3 loaded to Databricks' assert strip_whitespace(out) == strip_whitespace(expected_out) load_mock.assert_called_with( 'some/path/to/test-library-1.0.3-py3.6.egg', FileNameMatch('test-library-1.0.3-py3.6.egg'), '/other/folder', '', host, )
def test_delete_old_versions(id_nums, host, prod_folder): for i in range(2): responses.add_callback( responses.POST, host + '/api/1.2/libraries/delete', callback=request_callback, ) actual_deleted_libraries = delete_old_versions( logger, FileNameMatch('test-library-1.0.3-SNAPSHOT.egg'), id_nums, token='', prod_folder=prod_folder, host=host, ) assert len(responses.calls) == 2 actual_responses = [res.response.text for res in responses.calls] assert set(actual_responses) == {'libraryId=6', 'libraryId=7'} assert (set(actual_deleted_libraries) == { 'test-library-1.0.1.egg', 'test-library-1.0.2.egg' })
def test_filename_match_wrong_file_type(): with pytest.raises(FileNameError) as err: FileNameMatch('test-library-1.0.3.zip') assert err.filename == 'test-library-1.0.3.zip'
def test_filename_match_garbage_version(): with pytest.raises(FileNameError) as err: FileNameMatch('test-library-1.0.3-askjdhfa.egg') assert err.filename == 'test-library-1.0.3-askjdhfa.egg'
def test_filename_match_equal(): match_1 = FileNameMatch('test-library-1.0.3.egg') match_2 = FileNameMatch('test-library-1.0.3-py3.5.egg') assert match_1 == match_2
def test_filename_match_not_equal(): match_1 = FileNameMatch('test-library-1.0.3.egg') match_2 = FileNameMatch('test-library-1.0.3-SNAPSHOT.egg') assert match_1 != match_2
def test_filename_match_should_not_replace_snapshot(): match_1 = FileNameMatch('test-library-1.1.3.egg') match_2 = FileNameMatch('test-library-0.0.3-SNAPSHOT.egg') assert not match_1.replace_version(match_2, mock.MagicMock())
def test_filename_match_should_replace(): match_1 = FileNameMatch('test-library-1.1.3.egg') match_2 = FileNameMatch('test-library-1.0.3.egg') assert match_1.replace_version(match_2, mock.MagicMock())