def test_stream_download(mocker): """Test stream download""" mock_req = mocker.patch.object(utils.helpers, "requests") mock_stream = mocker.MagicMock() mock_stream.headers = {"content-length": "1000"} mock_req.get.return_value = mock_stream tqdm_mock = mocker.patch.object(utils.helpers, "tqdm") utils.stream_download("https://someurl.com/file.ext") expect_args = { "unit_scale": True, "unit_divisor": 1024, "smoothing": 0.1, "bar_format": mocker.ANY, } tqdm_mock.assert_called_once_with(total=1000, unit="B", **expect_args)
def test_stream_download(mocker): """Test stream download""" mock_req = mocker.patch.object(utils.helpers, 'requests') mock_stream = mocker.MagicMock() mock_stream.headers = {'content-length': '1000'} mock_req.get.return_value = mock_stream tqdm_mock = mocker.patch.object(utils.helpers, 'tqdm') utils.stream_download("https://someurl.com/file.ext") expect_args = { 'unit_scale': True, 'unit_divisor': 1024, 'smoothing': 0.1, 'bar_format': mocker.ANY } tqdm_mock.assert_called_once_with(total=1000, unit='B', **expect_args)
def _fetch_package(self, url): """Fetch and stub package at url Args: url (str): URL to fetch Returns: Path: path to package """ with tempfile.TemporaryDirectory() as tmp_dir: tmp_path = Path(tmp_dir) file_name = utils.get_url_filename(url) _file_name = "".join(self.log.iter_formatted(f"$B[{file_name}]")) content = utils.stream_download( url, desc=f"{self.log.get_service()} {_file_name}") pkg_path = utils.extract_tarbytes(content, tmp_path) ignore = ['setup.py', '__version__', 'test_'] pkg_init = next(pkg_path.rglob("__init__.py"), None) py_files = [f for f in pkg_path.rglob( "*.py") if not any(i in f.name for i in ignore)] stubs = [utils.generate_stub(f) for f in py_files] if pkg_init: data_path = self.pkg_data / pkg_init.parent.name shutil.copytree(pkg_init.parent, data_path) return data_path for file, stub in stubs: shutil.copy2(file, (self.pkg_data / file.name)) shutil.copy2(stub, (self.pkg_data / stub.name))
def fetch(self) -> bytes: """Fetch package contents into memory. Returns: bytes: Package archive contents. """ self.log.debug(f"fetching package: {self.file_name}") desc = self.format_desc(self.file_name) content = utils.stream_download(self.source_url, desc=desc) return content
def ready(self): """Retrieves and unpacks source Prepares remote stub resource by downloading and unpacking it into a temporary directory. This directory is removed on exit of the superclass context manager Returns: callable: StubSource.ready parent method """ tmp_dir = tempfile.mkdtemp() tmp_path = Path(tmp_dir) filename = utils.get_url_filename(self.location).split(".tar.gz")[0] outpath = tmp_path / filename _file_name = "".join(self.log.iter_formatted(f"$B[{filename}]")) content = utils.stream_download( self.location, desc=f"{self.log.get_service()} {_file_name}") source_path = self._unpack_archive(content, outpath) teardown = partial(shutil.rmtree, tmp_path) return super().ready(path=source_path, teardown=teardown)