class Archive:
"""Compatibility layer for ZipFile/Info and TarFile/Info"""
def __init__(self, filename):
self._filename = filename
if filename.endswith("tar.gz"):
self._obj = tarfile.open(filename, "r:gz")
elif filename.endswith("zip"):
self._obj = ZipFile(filename)
else:
raise ValueError(f"{filename} doesn't seem to be a zip or tar.gz")
def __iter__(self):
if hasattr(self._obj, "infolist"):
return iter(self._obj.infolist())
return iter(self._obj)
def get_name(self, zip_or_tar_info):
if hasattr(zip_or_tar_info, "filename"):
return zip_or_tar_info.filename
return zip_or_tar_info.name
def get_content(self, zip_or_tar_info):
if hasattr(self._obj, "extractfile"):
content = self._obj.extractfile(zip_or_tar_info)
if content is None:
msg = f"Invalid {zip_or_tar_info.name} in {self._filename}"
raise ValueError(msg)
return str(content.read(), "utf-8")
return str(self._obj.read(zip_or_tar_info), "utf-8")
def dependencies_check(self, archive):
""" Check package dependencies
look for 'package_name'.egg-info/requires.txt
or
requirements.txt
"""
if archive.endswith(".zip"):
arch = ZipFile("repo/{}".format(archive)).namelist()
else:
arch = tarfile.open("repo/{}".format(archive))
unpack = self.__unpack(arch)
# Check if there is no dependencies. If none, return None
if unpack is None:
return
# Dependencies list
deps = []
if not archive.endswith(".zip"):
a_file = arch.extractfile(member=unpack)
else:
zip_file = ZipFile("repo/{}".format(archive))
# Remove [python>= version] version checks
a_file = zip_file.read(unpack)
a_file = re.sub(r'\[.*\]', '', a_file.decode('ascii')).split()
for dep in a_file:
if hasattr(dep, 'decode'):
dep = dep.decode('ascii')
if re.match(r"^\w", dep):
deps.append(dep)
download = DownloadPackage(dep, quiet=True, dependencies=True)
dep = download()
if dep is not None:
deps.extend(dep)
return deps
def handle_file(file_path, module_filter, only_detect, is_temp_file=False):
# todo modular archive handling
# todo PE overlay extraction
# todo PE resources extraction
# todo Installer extraction
if is_zipfile(file_path):
# extract each file and handle it
# todo consider adding archive password support
try:
z = ZipFile(file_path)
for n in z.namelist():
data = z.read(n)
new_path = write_file_to_temp_file(data)
for p, r in handle_file(new_path, module_filter, only_detect, is_temp_file=True):
result_path = ""
if is_temp_file:
result_path = n
else:
result_path = file_path + "," + n
if p is not None:
result_path += "," + p
yield result_path, r
remove(new_path)
except KeyboardInterrupt:
raise
except:
pass
elif tarfile.is_tarfile(file_path):
try:
with tarfile.open(file_path, 'r') as z:
for member in z.getmembers():
try:
data = z.extractfile(member).read()
n = member.name
new_path = write_file_to_temp_file(data)
for p, r in handle_file(new_path, module_filter, only_detect, is_temp_file=True):
result_path = ""
if is_temp_file:
result_path = n
else:
result_path = file_path + "," + n
if p is not None:
result_path += "," + p
yield result_path, r
remove(new_path)
except KeyboardInterrupt:
raise
except:
pass
except KeyboardInterrupt:
raise
except:
pass
elif is_rarfile(file_path):
try:
z = RarFile(file_path)
for n in z.namelist():
data = z.read(n)
new_path = write_file_to_temp_file(data)
for p, r in handle_file(new_path, module_filter, only_detect, is_temp_file=True):
result_path = ""
if is_temp_file:
result_path = n
else:
result_path = file_path + "," + n
if p is not None:
result_path += "," + p
yield result_path, r
remove(new_path)
except KeyboardInterrupt:
raise
except:
pass
else:
# assume we are dealing with a normal file
# todo Convert file handling to use file paths
if getsize(file_path) < 1024 * 1024 * 1024:
with open(file_path, mode='rb') as file_handle:
file_content = file_handle.read()
r = scan_file_data(file_content, module_filter, only_detect)
if r is not None:
if is_temp_file:
yield None, r
else:
yield file_path, r