def extract(self, path: pathlib.Path, data_dir: pathlib.Path, file_list_file: pathlib.Path) -> None:
"""Attempt to extract the lzma compressed file. Save metadata about the file in the downloaded dataset in
``file_list_file``.
:param path: Path to the lzma file.
:param data_dir: Path to the data dir to extract the data file to.
:file_list_file: Path to the file that stores metadata of the file in the downloaded dataset.
:raises lzma.LZMAError: The lzma file was unable to be read.
:raises OSError: The extracted file is empty.
"""
try:
mylzma = lzma.open(path)
extracted_file_name = path.stem
with mylzma, open(data_dir / extracted_file_name, 'wb') as f_out:
shutil.copyfileobj(mylzma, f_out)
except lzma.LZMAError as e:
raise lzma.LZMAError(f'Failed to uncompress lzma file "{path}"\ncaused by:\n{e}')
extracted_fp = data_dir / extracted_file_name
FileListFileContents = Dict[str, Union[int, str]]
contents: FileListFileContents = {}
metadata: Dict[str, Union[str, FileListFileContents]] = {}
metadata['type'] = 'xz'
contents['filename'] = extracted_file_name
contents['size'] = extracted_fp.stat().st_size
if contents['size'] == 0:
raise OSError(f'The extracted file {extracted_file_name} is empty.')
metadata['contents'] = contents
with open(file_list_file, mode='w') as f:
json.dump(metadata, f, indent=2)
def unprocess(self, data):
super(DeenPluginLzma, self).unprocess(data)
if not lzma:
self.log.error('lzma module not found')
return data
results = []
while True:
decomp = lzma.LZMADecompressor(lzma.FORMAT_AUTO, None, None)
try:
res = decomp.decompress(data)
except lzma.LZMAError as e:
self.log.error(e)
self.log.debug(e, exc_info=True)
if results:
break
else:
self.error = e
return
results.append(res)
data = decomp.unused_data
if not data:
break
if not decomp.eof:
ex = lzma.LZMAError('Compressed data ended before the end-of-stream marker was reached')
self.error = ex
self.log.error(self.error)
self.log.debug(self.error, exc_info=True)
data = b''.join(results)
return data
def test_decompress_pkg_index_gz_failure(self):
"""
Test decompression for Packages.gz file failure handling.
:return:
"""
zdcmp = MagicMock(side_effect=zlib.error("Firewall currently too hot"))
xdcmp = MagicMock(side_effect=lzma.LZMAError(""))
with patch("spacewalk.common.repo.zlib.decompress", zdcmp) as m_zlib, \
patch("spacewalk.common.repo.lzma.decompress", xdcmp) as m_lzma:
with pytest.raises(GeneralRepoException) as exc:
DpkgRepo("http://dummy_url").decompress_pkg_index()
assert not xdcmp.called
assert zdcmp.called
assert "hot" in str(exc.value)
def decompress_lzma(data: bytes) -> bytes:
results = []
len(data)
while True:
decomp = lzma.LZMADecompressor(lzma.FORMAT_AUTO, None, None)
try:
res = decomp.decompress(data)
except lzma.LZMAError:
if results:
break # Leftover data is not a valid LZMA/XZ stream; ignore it.
else:
raise # Error on the first iteration; bail out.
results.append(res)
data = decomp.unused_data
if not data:
break
if not decomp.eof:
raise lzma.LZMAError("Compressed data ended before the end-of-stream marker was reached")
return b"".join(results)