def test_stream_download(mocker):
"""Test stream download"""
mock_req = mocker.patch.object(utils.helpers, "requests")
mock_stream = mocker.MagicMock()
mock_stream.headers = {"content-length": "1000"}
mock_req.get.return_value = mock_stream
tqdm_mock = mocker.patch.object(utils.helpers, "tqdm")
utils.stream_download("https://someurl.com/file.ext")
expect_args = {
"unit_scale": True,
"unit_divisor": 1024,
"smoothing": 0.1,
"bar_format": mocker.ANY,
}
tqdm_mock.assert_called_once_with(total=1000, unit="B", **expect_args)
def test_stream_download(mocker):
"""Test stream download"""
mock_req = mocker.patch.object(utils.helpers, 'requests')
mock_stream = mocker.MagicMock()
mock_stream.headers = {'content-length': '1000'}
mock_req.get.return_value = mock_stream
tqdm_mock = mocker.patch.object(utils.helpers, 'tqdm')
utils.stream_download("https://someurl.com/file.ext")
expect_args = {
'unit_scale': True,
'unit_divisor': 1024,
'smoothing': 0.1,
'bar_format': mocker.ANY
}
tqdm_mock.assert_called_once_with(total=1000, unit='B', **expect_args)
def _fetch_package(self, url):
"""Fetch and stub package at url
Args:
url (str): URL to fetch
Returns:
Path: path to package
"""
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = Path(tmp_dir)
file_name = utils.get_url_filename(url)
_file_name = "".join(self.log.iter_formatted(f"$B[{file_name}]"))
content = utils.stream_download(
url, desc=f"{self.log.get_service()} {_file_name}")
pkg_path = utils.extract_tarbytes(content, tmp_path)
ignore = ['setup.py', '__version__', 'test_']
pkg_init = next(pkg_path.rglob("__init__.py"), None)
py_files = [f for f in pkg_path.rglob(
"*.py") if not any(i in f.name for i in ignore)]
stubs = [utils.generate_stub(f) for f in py_files]
if pkg_init:
data_path = self.pkg_data / pkg_init.parent.name
shutil.copytree(pkg_init.parent, data_path)
return data_path
for file, stub in stubs:
shutil.copy2(file, (self.pkg_data / file.name))
shutil.copy2(stub, (self.pkg_data / stub.name))
def fetch(self) -> bytes:
"""Fetch package contents into memory.
Returns:
bytes: Package archive contents.
"""
self.log.debug(f"fetching package: {self.file_name}")
desc = self.format_desc(self.file_name)
content = utils.stream_download(self.source_url, desc=desc)
return content
def ready(self):
"""Retrieves and unpacks source
Prepares remote stub resource by downloading and
unpacking it into a temporary directory.
This directory is removed on exit of the superclass
context manager
Returns:
callable: StubSource.ready parent method
"""
tmp_dir = tempfile.mkdtemp()
tmp_path = Path(tmp_dir)
filename = utils.get_url_filename(self.location).split(".tar.gz")[0]
outpath = tmp_path / filename
_file_name = "".join(self.log.iter_formatted(f"$B[{filename}]"))
content = utils.stream_download(
self.location, desc=f"{self.log.get_service()} {_file_name}")
source_path = self._unpack_archive(content, outpath)
teardown = partial(shutil.rmtree, tmp_path)
return super().ready(path=source_path, teardown=teardown)
