def test_download_url_to_file__via_googledrive(self, mocker):
dummy_temp_file = 'dummy_file'
mock_temp_file = mocker.patch.object(URL,
"create_temp_file",
return_value=dummy_temp_file)
mock_download_url = mocker.patch.object(URLDownload, "download")
mock_download_googledrive = mocker.patch.object(
URLDownloadGoogleDrive, "download")
mock_move = mocker.patch("shutil.move")
url_metadata = {
"url": 'http://dummy_url.html',
"method": 'googledrive'
}
filename = 'dummy_filename.zip'
verbose = True
URL().download_url_to_file(url_metadata=url_metadata,
filename=filename,
verbose=verbose)
mock_temp_file.assert_called_once_with(filename)
assert not mock_download_url.called
mock_download_googledrive.assert_called_once_with(
url_metadata['url'], filename=dummy_temp_file)
mock_move.assert_called_once_with(dummy_temp_file, filename)
def test_download_url(self, mocker):
dummy_metadata = {'md5hash': 'dummy_hash'}
dummy_download_dir = os.path.join('some', 'path', 'to', 'data')
dummy_filename = os.path.join(dummy_download_dir, 'file1.zip')
mock_get_metadata = mocker.patch.object(
URL,
"get_url_metadata_and_dir_paths",
return_value=(dummy_metadata, dummy_download_dir, dummy_filename))
mock_exists = mocker.patch("os.path.exists", return_value=False)
mock_create_dir = mocker.patch("os.makedirs")
mock_download = mocker.patch.object(URL, "download_url_to_file")
mock_md5_checksum = mocker.patch.object(URL, "md5_checksum")
url = 'http://url1.zip'
save_dir = os.path.join('path', 'to', 'data', 'dir')
verbose = True
filename = URL().download_url(url=url,
save_dir=save_dir,
verbose=verbose)
mock_get_metadata.assert_called_once_with(url, save_dir)
mock_exists.assert_called_once_with(dummy_download_dir)
mock_create_dir.assert_called_once_with(dummy_download_dir)
mock_download.assert_called_once_with(dummy_metadata, dummy_filename,
verbose)
mock_md5_checksum.assert_called_once_with(dummy_filename,
dummy_metadata['md5hash'])
assert filename == dummy_filename
def test_get_value_from_key__key_doesnt_exist(self, mocker):
value = URL().get_value_from_key(input={
"url": 'http://dummy_url.html',
"md5hash": 'dummy_hash'
},
key='filename',
default='default')
assert value == 'default'
def test_md5_checksum__raises_error(self, mocker):
dummy_hash = 'a5s6dea9s8rtqw1s1g45jk4s4dfg49'
mock_get_hash = mocker.patch.object(URL,
"get_file_hash",
return_value=dummy_hash)
with pytest.raises(MD5HashNotEqual):
filename = os.path.join('some', 'path', 'to', 'filename1.zip')
md5hash = '87897asd98f74asd4fas6d4as8v46t'
URL().md5_checksum(filename=filename, md5hash=md5hash)
def test_md5_checksum(self, mocker):
dummy_hash = 'a5s6dea9s8rtqw1s1g45jk4s4dfg49'
mock_get_hash = mocker.patch.object(URL,
"get_file_hash",
return_value=dummy_hash)
filename = os.path.join('some', 'path', 'to', 'filename1.zip')
md5hash = 'a5s6dea9s8rtqw1s1g45jk4s4dfg49'
URL().md5_checksum(filename=filename, md5hash=md5hash)
mock_get_hash.assert_called_once_with(filename)
def test_parse_url_metadata__string(self, mocker):
url = 'http://url1.zip'
url_metadata = URL().parse_url_metadata(url)
assert url_metadata == {
"url": url,
"md5hash": None,
"filename": 'url1.zip',
"extract_dir": '',
"method": 'requests'
}
def test_download_url_to_file__raises_exception(self, mocker):
dummy_temp_file = 'dummy_file'
mock_temp_file = mocker.patch.object(URL,
"create_temp_file",
return_value=dummy_temp_file)
with pytest.raises(InvalidURLDownloadSource):
URL().download_url_to_file(url_metadata={
"url": 'http://dummy_url.html',
"method": 'invalid_method'
},
filename='dummy_filename.zip',
verbose=True)
def test_parse_url_metadata__googledrive(self, mocker):
url = {
"url": '134728318290',
"save_name": 'file_gdrive.zip',
"extract_dir": os.path.join('some', 'extract', 'path'),
"source": 'googledrive'
}
url_metadata = URL().parse_url_metadata(url)
assert url_metadata == {
"url": url['url'],
"md5hash": None,
"filename": url['save_name'],
"extract_dir": url['extract_dir'],
"method": 'googledrive'
}
def test_parse_url_metadata__dict(self, mocker):
url = {
"url": 'http://url1.zip',
"md5hash": '123456798',
"save_name": 'file_a87is9.zip',
"extract_dir": os.path.join('some', 'extract', 'path')
}
url_metadata = URL().parse_url_metadata(url)
assert url_metadata == {
"url": url['url'],
"md5hash": url['md5hash'],
"filename": url['save_name'],
"extract_dir": url['extract_dir'],
"method": 'requests'
}
def test_get_url_filename(self, mocker):
dummy_extract_dir = os.path.join('some', 'dir', 'to', 'extract')
dummy_filename = 'filename1.zip'
dummy_metadata = {
"extract_dir": dummy_extract_dir,
"filename": dummy_filename
}
mock_parse_url = mocker.patch.object(URL,
"parse_url_metadata",
return_value=dummy_metadata)
url = 'http://filename1.zip'
filename = URL.get_url_filename(url=url)
mock_parse_url.assert_called_once_with(url)
assert filename == dummy_filename
def test_exists_url_file(self, mocker, file_exists):
dummy_metadata = {'dummy': 'data'}
dummy_dir = os.path.join('some', 'path', 'to', 'data')
dummy_filename = os.path.join('some', 'path', 'to', 'data',
'file1.zip')
mock_get_metadata = mocker.patch.object(
URL,
"get_url_metadata_and_dir_paths",
return_value=(dummy_metadata, dummy_dir, dummy_filename))
mock_exists = mocker.patch("os.path.exists", return_value=file_exists)
url = 'http://url1.zip'
save_dir = os.path.join('path', 'to', 'data', 'dir')
result = URL().exists_url_file(url, save_dir)
mock_get_metadata.assert_called_once_with(url, save_dir)
mock_exists.assert_called_once_with(dummy_filename)
assert result == file_exists
def test_create_temp_file(self, mocker):
dummy_file = mocker.MagicMock()
dummy_filename = "filename.tmp"
mock_create_file = mocker.patch("tempfile.mkstemp",
return_value=(dummy_file,
dummy_filename))
mock_close = mocker.patch("os.close")
mock_unlink = mocker.patch("os.unlink")
filename_dir = os.path.join('some', 'path', 'to', 'data')
filename = os.path.join(filename_dir, 'filename1.zip')
tmpfile = URL().create_temp_file(filename)
assert tmpfile == dummy_filename
mock_create_file.assert_called_once_with(".tmp",
prefix=filename,
dir=filename_dir)
mock_close.assert_called_once_with(dummy_file)
mock_unlink.assert_called_once_with(dummy_filename)
def test_download__url_not_exists(self, mocker):
dummy_filename = os.path.join('some', 'path', 'to', 'data',
'file1.zip')
mock_exists_file = mocker.patch.object(URL,
"exists_url_file",
return_value=False)
mock_get_metadata = mocker.patch.object(
URL, "get_url_metadata_and_dir_paths")
mock_download_url = mocker.patch.object(URL,
"download_url",
return_value=dummy_filename)
url = 'http://url1.zip'
save_dir = os.path.join('path', 'to', 'data', 'dir')
filename = URL.download(url=url, save_dir=save_dir, verbose=True)
mock_exists_file.assert_called_once_with(url, save_dir)
assert not mock_get_metadata.called
mock_download_url.assert_called_once_with(url, save_dir, True)
assert filename == dummy_filename
def test_get_url_metadata_and_dir_paths(self, mocker):
dummy_extract_dir = os.path.join('some', 'dir', 'to', 'extract')
dummy_filename = 'filename1.zip'
dummy_metadata = {
"extract_dir": dummy_extract_dir,
"filename": dummy_filename
}
mock_parse_url = mocker.patch.object(URL,
"parse_url_metadata",
return_value=dummy_metadata)
url = 'http://url1.zip'
save_dir = os.path.join('path', 'to', 'data', 'dir')
url_metadata, download_dir, filename = URL(
).get_url_metadata_and_dir_paths(url, save_dir)
mock_parse_url.assert_called_once_with(url)
assert url_metadata == dummy_metadata
assert download_dir == os.path.join(save_dir, dummy_extract_dir)
assert filename == os.path.join(save_dir,
dummy_metadata["extract_dir"],
dummy_metadata["filename"])
def test_get_value_from_key__string_input(self, mocker):
input = 'http://dummy_url'
key = 'url'
value = URL().get_value_from_key(input, key, default='default_val')
assert value == 'default_val'
def test_parse_url_metadata__invalid_url_type(self, mocker):
with pytest.raises(AssertionError):
url = ['http://url1.zip']
url_metadata = URL().parse_url_metadata(url)
def test_get_value_from_key__key_exists(self, mocker):
dictionary = {"url": 'http://dummy_url.html', "md5hash": 'dummy_hash'}
key = 'url'
value = URL().get_value_from_key(dictionary, key)
assert value == dictionary[key]