def test_update_databricks_with_jar_only_upload(
load_mock,
caplog,
prod_folder,
host,
cfg,
):
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path='some/path/to/test-library-1.0.3.jar',
token='',
folder=prod_folder,
update_jobs=False,
cleanup=False,
)
out = caplog.record_tuples[0][2]
expected_out = 'new library test-library-1.0.3 loaded to Databricks'
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.3.jar',
FileNameMatch('test-library-1.0.3.jar'),
prod_folder,
'',
host,
)
def test_update_databricks_update_jobs(
delete_mock,
update_mock,
lib_mock,
job_mock,
load_mock,
library_mapping,
id_nums,
job_list,
caplog,
prod_folder,
host,
cfg,
):
path = 'some/path/to/test-library-1.0.3-py3.6.egg'
delete_mock.return_value = ['test-library-1.0.1', 'test-library-1.0.2']
job_mock.return_value = job_list
lib_mock.return_value = (library_mapping, id_nums)
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path=path,
token='',
folder=prod_folder,
update_jobs=True,
cleanup=True,
)
out = [r[2] for r in caplog.record_tuples]
expected_out = [
'new library test-library-1.0.3 loaded to Databricks',
'current major version of library used by jobs: job_3',
'updated jobs: job_3',
'removed old versions: test-library-1.0.1, test-library-1.0.2',
]
match = FileNameMatch('test-library-1.0.3-py3.6.egg')
assert out == expected_out
load_mock.assert_called_with(path, match, prod_folder, '', host)
job_mock.assert_called_with(logger, match, library_mapping, '', host)
lib_mock.assert_called_with(logger, prod_folder, '', host)
update_mock.assert_called_with(
logger,
job_list,
match,
'dbfs:/FileStore/jars/47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg',
'',
host,
)
delete_mock.assert_called_with(
logger,
match,
id_nums=id_nums,
token='',
prod_folder=prod_folder,
host=host,
)
def test_update_databricks_update_jobs_no_cleanup(
update_mock,
lib_mock,
job_mock,
load_mock,
library_mapping,
id_nums,
job_list,
caplog,
prod_folder,
host,
):
path = 'some/path/to/test-library-1.0.3-py3.6.egg'
job_mock.return_value = job_list
lib_mock.return_value = (library_mapping, id_nums)
update_databricks(
logger,
path=path,
token='',
folder=prod_folder,
update_jobs=True,
cleanup=False,
)
out = [r[2] for r in caplog.record_tuples]
expected_out = [
'new library test-library-1.0.3 loaded to Databricks',
'current major version of library used by jobs: job_3',
'updated jobs: job_3',
]
assert out == expected_out
match = FileNameMatch('test-library-1.0.3-py3.6.egg')
load_mock.assert_called_with(
path,
match,
prod_folder,
'',
host,
)
job_mock.assert_called_with(logger, match, library_mapping, '', host)
lib_mock.assert_called_with(logger, prod_folder, '', host)
update_mock.assert_called_with(
logger,
job_list,
match,
'dbfs:/FileStore/jars/47fb08a7-test-library_1_0_3_py3_6-e5f8c.egg',
'',
host,
)
def test_update_databricks_filename_not_match(
load_mock,
prod_folder,
host,
):
with pytest.raises(FileNameError) as err:
update_databricks(
logger,
path='some/path/to/test-library-1.0.3.zip',
token='',
folder=prod_folder,
update_jobs=False,
cleanup=False,
)
assert err.filename == 'test-library-1.0.3.zip'
def test_update_databricks_already_exists(
load_mock,
caplog,
prod_folder,
host,
cfg,
):
responses.add(
responses.GET,
'https://test-api',
status=500,
content_type='text/plain',
json={
'error_code':
'http 500',
'message': ('NameConflictException: '
'Node named "test-library" already exists')
},
)
res = requests.get('https://test-api')
load_mock.side_effect = APIError(res)
with mock.patch('apparate.update_databricks_library.CFG_FILE', cfg):
update_databricks(
logger,
path='some/path/to/test-library-1.0.1-py3.6.egg',
token='',
folder='/other/folder',
update_jobs=False,
cleanup=False,
)
out = caplog.record_tuples[0][2]
expected_out = (
'This version (1.0.1) already exists: '
'if a change has been made please update your version number. '
'Note this error can also occur if you are uploading a jar '
'and an egg already exists with the same name and version, '
'or vice versa. In this case you will need to choose a '
'different library name or a different folder for either the '
'egg or the jar.')
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.1-py3.6.egg',
FileNameMatch('test-library-1.0.1-py3.6.egg'),
'/other/folder',
'',
host,
)
def test_update_databricks_wrong_folder(load_mock, caplog, host):
update_databricks(
logger,
path='some/path/to/test-library-1.0.3-py3.6.egg',
token='',
folder='/other/folder',
update_jobs=True,
cleanup=True,
)
out = caplog.record_tuples[0][2]
expected_out = 'new library test-library-1.0.3 loaded to Databricks'
assert strip_whitespace(out) == strip_whitespace(expected_out)
load_mock.assert_called_with(
'some/path/to/test-library-1.0.3-py3.6.egg',
FileNameMatch('test-library-1.0.3-py3.6.egg'),
'/other/folder',
'',
host,
)