def test_ons_postcode_directory_new_data_available(mocker):
# Mocking
download_file_mock = request_mock(mocker, download_url_response_dict)
get_data_files_html_mock = inspect_online_selenium_mock(
mocker, selenium_url_response_dict)
s3_storage_mock = storage_mock(['ONSPD_MAY_2020_UK.zip'])
# Run method to test
datasource = ONSPostcodeDirectory(s3_storage_mock)
datasource.update()
# Assert the correct html page is accessed to scan for new files
assert get_data_files_html_mock.inspected_urls == [
'https://geoportal.statistics.gov.uk/search?sort=-created&tags=ons%20postcode%20directory',
'https://geoportal.statistics.gov.uk/datasets/ons-postcode-directory-august-2020',
]
# Assert correct file is downloaded
download_file_mock.assert_called_once_with(
'https://www.arcgis.com/sharing/rest/content/items/a644dd04d18f4592b7d36705f93270d8/data'
)
# Assert correct file is written to s3 bucket
s3_storage_mock.write_file.assert_called_once_with('ONSPD_AUG_2020_UK.zip',
b'some data')
def test_hmrc_new_data_available(mocker):
# Mocking
download_file_mock = request_mock(mocker, {'*': b'some data'})
s3_storage_mock = storage_mock(s3_keys_returned=['source/hmrc/exporters/exporters1702.zip'])
get_data_files_html_mock = inspect_online_mock(mocker, html_string)
# Run method to test
exporters = HMRCExporters(s3_storage_mock)
exporters.update()
# Assert the correct html page is accessed twice to scan for new files
get_data_files_html_mock.assert_has_calls([call(exporters.download_url, context=ANY)])
# Assert correct files are downloaded
download_file_mock.assert_has_calls(
[
call(
'https://www.uktradeinfo.com/Statistics/Documents'
'/Data%20Downloads/exporters1703.zip'
)
],
any_order=True,
)
# Assert correct files are written to s3 bucket
s3_storage_mock.write_file.assert_has_calls(
[call.write_file('exporters1703.zip', b'some data')]
)
def test_ch_accounts_no_new_data(mocker):
# Mocking
download_file_mock = request_mock(mocker, {'*': b'some data'})
s3_storage_mock = storage_mock(['Accounts_Bulk_Data-2017-06-20.zip'])
get_data_files_html_mock = inspect_online_mock(mocker, accounts_html_string)
# Run method to test
datasource = CompaniesHouseAccounts(s3_storage_mock)
datasource.update()
# Assert the correct html page is accessed to scan for new files
get_data_files_html_mock.assert_called_once_with(datasource.download_url, context=ANY)
# Assert no file is downloaded
download_file_mock.assert_not_called()
# Assert no new file is written to s3 bucket
s3_storage_mock.write_file.assert_not_called()
def test_hmrc_no_new_data(mocker):
# Mocking
download_file_mock = request_mock(mocker, {'*': b'some data'})
s3_storage_mock = storage_mock(s3_keys_returned=['source/hmrc/exporters/exporters1703.zip'])
get_data_files_html_mock = inspect_online_mock(mocker, html_string)
# Run method to test
exporters = HMRCExporters(s3_storage_mock)
exporters.update()
# Assert the correct html page is accessed to scan for new files
get_data_files_html_mock.assert_called_with(exporters.download_url, context=ANY)
# Assert no file is downloaded
download_file_mock.assert_not_called()
# Assert no new file is written to s3 bucket
s3_storage_mock.write_file.assert_not_called()
def test_ons_postcode_no_new_data(mocker):
# Mocking
download_file_mock = request_mock(mocker, download_url_response_dict)
get_data_files_html_mock = inspect_online_selenium_mock(
mocker, selenium_url_response_dict)
s3_storage_mock = storage_mock(['ONSPD_AUG_2020_UK.zip'])
# Run method to test
datasource = ONSPostcodeDirectory(s3_storage_mock)
datasource.update()
# Assert the correct html page is accessed to scan for new files
assert get_data_files_html_mock.inspected_urls == [
'https://geoportal.statistics.gov.uk/search?sort=-created&tags=ons%20postcode%20directory',
]
# Assert no file is downloaded
download_file_mock.assert_not_called()
# Assert no new file is written to s3 bucket
s3_storage_mock.write_file.assert_not_called()
def test_ch_accounts_new_data_available(mocker):
# Mocking
download_file_mock = request_mock(mocker, {'*': b'some data'})
s3_storage_mock = storage_mock(['Accounts_Bulk_Data-2017-06-17.zip'])
get_data_files_html_mock = inspect_online_mock(mocker, accounts_html_string)
# Run method to test
datasource = CompaniesHouseAccounts(s3_storage_mock)
datasource.update()
# Assert the correct html page is accessed to scan for new files
get_data_files_html_mock.assert_called_once_with(datasource.download_url, context=ANY)
# Assert correct file is downloaded
download_file_mock.assert_has_calls(
[call('http://download.companieshouse.gov.uk/Accounts_Bulk_Data-2017-06-20.zip')]
)
# Assert correct file is written to s3 bucket
s3_storage_mock.write_file.assert_called_once_with(
'Accounts_Bulk_Data-2017-06-20.zip', b'some data'
)