def test_update_job_libraries(
job_list,
job_update_response_list_old,
job_update_response_list_new,
host,
):
for job in job_update_response_list_old:
responses.add(
responses.GET,
host + '/api/2.0/jobs/get?job_id={}'.format(job['job_id']),
status=200,
json=job,
)
responses.add_callback(
responses.POST,
host + '/api/2.0/jobs/reset',
callback=request_callback,
)
update_job_libraries(
logger,
job_list,
FileNameMatch('test_library-1.2.3.egg'),
'dbfs:/FileStore/jars/some_library_uri',
'',
host,
)
assert len(responses.calls) == 2
assert (json.loads(
responses.calls[1].response.text) == job_update_response_list_new[0])
def test_update_databricks_with_jar_only_upload(
load_mock,
caplog,
prod_folder,
host,
cfg,
):
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path='some/path/to/test-library-1.0.3.jar',
token='',
folder=prod_folder,
update_jobs=False,
cleanup=False,
)
out = caplog.record_tuples[0][2]
expected_out = 'new library test-library-1.0.3 loaded to Databricks'
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.3.jar',
FileNameMatch('test-library-1.0.3.jar'),
prod_folder,
'',
host,
)
def test_filename_match_jar_snapshot_branch():
match = FileNameMatch('new_library-1.0.0-SNAPSHOT-my-branch.jar')
assert match.library_name == 'new_library'
assert match.version == '1.0.0-SNAPSHOT-my-branch'
assert match.major_version == '1'
assert match.minor_version == '0.0'
assert match.suffix == 'jar'
assert match.lib_type == 'java-jar'
def test_filename_match_jar():
match = FileNameMatch('new_library-1.0.0.jar')
assert match.library_name == 'new_library'
assert match.version == '1.0.0'
assert match.major_version == '1'
assert match.minor_version == '0.0'
assert match.suffix == 'jar'
assert match.lib_type == 'java-jar'
def test_filename_match_egg_snapshot_branch():
match = FileNameMatch('new_library-1.0.0-SNAPSHOT-my-branch.egg')
assert match.library_name == 'new_library'
assert match.version == '1.0.0-SNAPSHOT-my-branch'
assert match.major_version == '1'
assert match.minor_version == '0.0'
assert match.suffix == 'egg'
assert match.lib_type == 'python-egg'
def test_filename_match_egg():
match = FileNameMatch('new_library-1.0.0.egg')
assert match.library_name == 'new_library'
assert match.version == '1.0.0'
assert match.major_version == '1'
assert match.minor_version == '0.0'
assert match.suffix == 'egg'
assert match.lib_type == 'python-egg'
def id_nums():
id_nums = {
'test-library-1.0.3': {
'name_match': FileNameMatch('test-library-1.0.3.egg'),
'id_num': '8',
},
'test-library-1.0.2': {
'name_match': FileNameMatch('test-library-1.0.2.egg'),
'id_num': '7',
},
'test-library-1.0.1': {
'name_match': FileNameMatch('test-library-1.0.1.egg'),
'id_num': '6',
},
'test-library-plus-stuff-0.0.0': {
'name_match': FileNameMatch('test-library-plus-stuff-0.0.0.egg'),
'id_num': '4',
},
'test-library-0.0.0': {
'name_match': FileNameMatch('test-library-0.0.0.egg'),
'id_num': '3',
},
'awesome_library_a-0.10.1': {
'name_match': FileNameMatch('awesome_library_a-0.10.1.egg'),
'id_num': '2',
},
'awesome_library_b-4.2.3': {
'name_match': FileNameMatch('awesome_library_b-4.2.3.egg'),
'id_num': '1',
},
}
return id_nums
def test_update_databricks_update_jobs(
delete_mock,
update_mock,
lib_mock,
job_mock,
load_mock,
library_mapping,
id_nums,
job_list,
caplog,
prod_folder,
host,
cfg,
):
path = 'some/path/to/test-library-1.0.3-py3.6.egg'
delete_mock.return_value = ['test-library-1.0.1', 'test-library-1.0.2']
job_mock.return_value = job_list
lib_mock.return_value = (library_mapping, id_nums)
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path=path,
token='',
folder=prod_folder,
update_jobs=True,
cleanup=True,
)
out = [r[2] for r in caplog.record_tuples]
expected_out = [
'new library test-library-1.0.3 loaded to Databricks',
'current major version of library used by jobs: job_3',
'updated jobs: job_3',
'removed old versions: test-library-1.0.1, test-library-1.0.2',
]
match = FileNameMatch('test-library-1.0.3-py3.6.egg')
assert out == expected_out
load_mock.assert_called_with(path, match, prod_folder, '', host)
job_mock.assert_called_with(logger, match, library_mapping, '', host)
lib_mock.assert_called_with(logger, prod_folder, '', host)
update_mock.assert_called_with(
logger,
job_list,
match,
'dbfs:/FileStore/jars/47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg',
'',
host,
)
delete_mock.assert_called_with(
logger,
match,
id_nums=id_nums,
token='',
prod_folder=prod_folder,
host=host,
)
def test_update_databricks_update_jobs_no_cleanup(
update_mock,
lib_mock,
job_mock,
load_mock,
library_mapping,
id_nums,
job_list,
caplog,
prod_folder,
host,
):
path = 'some/path/to/test-library-1.0.3-py3.6.egg'
job_mock.return_value = job_list
lib_mock.return_value = (library_mapping, id_nums)
update_databricks(
logger,
path=path,
token='',
folder=prod_folder,
update_jobs=True,
cleanup=False,
)
out = [r[2] for r in caplog.record_tuples]
expected_out = [
'new library test-library-1.0.3 loaded to Databricks',
'current major version of library used by jobs: job_3',
'updated jobs: job_3',
]
assert out == expected_out
match = FileNameMatch('test-library-1.0.3-py3.6.egg')
load_mock.assert_called_with(
path,
match,
prod_folder,
'',
host,
)
job_mock.assert_called_with(logger, match, library_mapping, '', host)
lib_mock.assert_called_with(logger, prod_folder, '', host)
update_mock.assert_called_with(
logger,
job_list,
match,
'dbfs:/FileStore/jars/47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg',
'',
host,
)
def test_update_databricks_already_exists(
load_mock,
caplog,
prod_folder,
host,
cfg,
):
responses.add(
responses.GET,
'https://test-api',
status=500,
content_type='text/plain',
json={
'error_code':
'http 500',
'message': ('NameConflictException: '
'Node named "test-library" already exists')
},
)
res = requests.get('https://test-api')
load_mock.side_effect = APIError(res)
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path='some/path/to/test-library-1.0.1-py3.6.egg',
token='',
folder='/other/folder',
update_jobs=False,
cleanup=False,
)
out = caplog.record_tuples[0][2]
expected_out = (
'This version (1.0.1) already exists: '
'if a change has been made please update your version number. '
'Note this error can also occur if you are uploading a jar '
'and an egg already exists with the same name and version, '
'or vice versa. In this case you will need to choose a '
'different library name or a different folder for either the '
'egg or the jar.')
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.1-py3.6.egg',
FileNameMatch('test-library-1.0.1-py3.6.egg'),
'/other/folder',
'',
host,
)
def test_load_library_jar(host, prod_folder):
filename = 'test-library-1.0.3.jar'
responses.add(
responses.POST,
host + '/api/1.2/libraries/upload',
status=200,
)
with mock.patch('builtins.open',
mock.mock_open(read_data='jar file contents')):
load_library(
filename=filename,
match=FileNameMatch(filename),
folder=prod_folder,
token='',
host=host,
)
def test_update_databricks_wrong_folder(load_mock, caplog, host):
update_databricks(
logger,
path='some/path/to/test-library-1.0.3-py3.6.egg',
token='',
folder='/other/folder',
update_jobs=True,
cleanup=True,
)
out = caplog.record_tuples[0][2]
expected_out = 'new library test-library-1.0.3 loaded to Databricks'
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.3-py3.6.egg',
FileNameMatch('test-library-1.0.3-py3.6.egg'),
'/other/folder',
'',
host,
)
def test_get_job_list(library_mapping, job_list, job_list_response, host):
responses.add(
responses.GET,
host + '/api/2.0/jobs/list',
status=200,
json=job_list_response,
)
match = FileNameMatch('test-library-1.1.2.egg')
job_list_actual = get_job_list(
logger,
match=match,
library_mapping=library_mapping,
token='',
host=host,
)
assert len(responses.calls) == 1
assert job_list_actual == job_list
def library_mapping():
library_mapping = {
'47fb08a7-test-library_1_0_2_py3_6-e5f8c.egg':
FileNameMatch('test-library-1.0.2.egg'),
'47fb08a7-test-library_1_0_1_py3_6-e5f8c.egg':
FileNameMatch('test-library-1.0.1.egg'),
'01832402-test-library-plus-stuff_0_0_0_py3_6-e5f8c.egg':
FileNameMatch('test-library-plus-stuff-0.0.0.egg'),
'47fb08a7-test-library_0_0_0_py3_6-e5f8c.egg':
FileNameMatch('test-library-0.0.0.egg'),
'996c949b-awesome_library_a_0_10_1_py3_6-266f.egg':
FileNameMatch('awesome_library_a-0.10.1.egg'),
'47fb08a7-awesome_library_b_4_2_3_py3_6-e5f8c.egg':
FileNameMatch('awesome_library_b-4.2.3.egg'),
'47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg':
FileNameMatch('test-library-1.0.3.egg'),
}
return library_mapping
def test_load_library_APIError(host, prod_folder):
filename = 'test-library-1.0.3-py3.6.egg'
responses.add(
responses.POST,
host + '/api/1.2/libraries/upload',
status=401,
)
with pytest.raises(APIError) as err:
with mock.patch('builtins.open',
mock.mock_open(read_data='egg file contents')):
load_library(
filename=filename,
match=FileNameMatch(filename),
folder=prod_folder,
token='',
host=host,
)
assert err.code == 'http 401'
def test_delete_old_versions(id_nums, host, prod_folder):
for i in range(2):
responses.add_callback(
responses.POST,
host + '/api/1.2/libraries/delete',
callback=request_callback,
)
actual_deleted_libraries = delete_old_versions(
logger,
FileNameMatch('test-library-1.0.3-SNAPSHOT.egg'),
id_nums,
token='',
prod_folder=prod_folder,
host=host,
)
assert len(responses.calls) == 2
actual_responses = [res.response.text for res in responses.calls]
assert set(actual_responses) == {'libraryId=6', 'libraryId=7'}
assert (set(actual_deleted_libraries) == {
'test-library-1.0.1.egg', 'test-library-1.0.2.egg'
})
def test_filename_match_wrong_file_type():
with pytest.raises(FileNameError) as err:
FileNameMatch('test-library-1.0.3.zip')
assert err.filename == 'test-library-1.0.3.zip'
def test_filename_match_equal():
match_1 = FileNameMatch('test-library-1.0.3.egg')
match_2 = FileNameMatch('test-library-1.0.3-py3.5.egg')
assert match_1 == match_2
def test_filename_match_not_equal():
match_1 = FileNameMatch('test-library-1.0.3.egg')
match_2 = FileNameMatch('test-library-1.0.3-SNAPSHOT.egg')
assert match_1 != match_2
def test_filename_match_should_replace():
match_1 = FileNameMatch('test-library-1.1.3.egg')
match_2 = FileNameMatch('test-library-1.0.3.egg')
assert match_1.replace_version(match_2, mock.MagicMock())
def test_filename_match_should_not_replace_snapshot():
match_1 = FileNameMatch('test-library-1.1.3.egg')
match_2 = FileNameMatch('test-library-0.0.3-SNAPSHOT.egg')
assert not match_1.replace_version(match_2, mock.MagicMock())
def test_filename_match_garbage_version():
with pytest.raises(FileNameError) as err:
FileNameMatch('test-library-1.0.3-askjdhfa.egg')
assert err.filename == 'test-library-1.0.3-askjdhfa.egg'
