def diff(feeder1, feeder2):
tmpdir = get_temporary_directory().name
fifo1_path = os.path.join(tmpdir, 'fifo1')
fifo2_path = os.path.join(tmpdir, 'fifo2')
with FIFOFeeder(feeder1, fifo1_path) as fifo1, \
FIFOFeeder(feeder2, fifo2_path) as fifo2:
return run_diff(fifo1_path, fifo2_path, fifo1.end_nl_q, fifo2.end_nl_q)
def ensure_unpacked(self):
if hasattr(self, '_members'):
return
self._members = collections.OrderedDict()
self._temp_dir = get_temporary_directory().name
logger.debug("Extracting %s to %s", self.source.path, self._temp_dir)
output = subprocess.check_output(
(
'unsquashfs',
'-n',
'-f',
'-no',
'-li',
'-d',
'.',
self.source.path,
),
stderr=subprocess.PIPE,
cwd=self._temp_dir,
)
output = iter(output.decode('utf-8').rstrip('\n').split('\n'))
# Skip headers
for _ in iter(functools.partial(next, output), ''):
pass
for line in output:
if not line:
continue
try:
cls = self.MEMBER_CLASS[line[0]]
except KeyError:
logger.debug("Unknown squashfs entry: %s", line)
continue
try:
kwargs = cls.parse(line)
except SquashfsInvalidLineFormat:
continue
# Pop to avoid duplicating member name in the key and the value
member_name = kwargs.pop('member_name')
self._members[member_name] = (cls, kwargs)
logger.debug(
"Extracted %d entries from %s to %s",
len(self._members),
self.source.path,
self._temp_dir,
)
def path(self):
if self._path is None:
logger.debug("Unpacking %s from %s", self._name,
self.container.source.name)
assert self._temp_dir is None
self._temp_dir = get_temporary_directory()
with profile('container_extract', self.container):
self._path = self.container.extract(self._name,
self._temp_dir.name)
return self._path
def compare_rpm_headers(path1, path2):
# compare headers
with get_temporary_directory() as rpmdb_dir:
rpm.addMacro("_dbpath", rpmdb_dir)
ts = rpm.TransactionSet()
ts.setVSFlags(-1)
header1 = get_rpm_header(path1, ts)
header2 = get_rpm_header(path2, ts)
return Difference.from_text(header1,
header2,
path1,
path2,
source="header")
def open_archive(self):
self._members = []
self._unpacked = os.path.join(
get_temporary_directory().name,
os.path.basename(self.source.name),
)
self._andmanifest = None
self._andmanifest_orig = None
logger.debug("Extracting %s to %s", self.source.name, self._unpacked)
subprocess.check_call((
'apktool', 'd', '-k', '-m', '-o', self._unpacked, self.source.path,
), shell=False, stderr=None, stdout=subprocess.PIPE)
for root, _, files in os.walk(self._unpacked):
current_dir = []
for filename in files:
abspath = os.path.join(root, filename)
# apktool.yml is a file created by apktool and containing
# metadata information. Rename it to clarify and always make it
# appear at the beginning of the directory listing for
# reproducibility.
if filename == 'apktool.yml':
abspath = filter_apk_metadata(
abspath,
os.path.basename(self.source.name),
)
relpath = abspath[len(self._unpacked) + 1:]
current_dir.insert(0, relpath)
continue
relpath = abspath[len(self._unpacked)+1:]
if filename == 'AndroidManifest.xml':
containing_dir = root[len(self._unpacked)+1:]
if containing_dir == 'original':
self._andmanifest_orig = relpath
if containing_dir == '':
self._andmanifest = relpath
continue
current_dir.append(relpath)
self._members.extend(current_dir)
return self
def recognizes(cls, file):
if binwalk is None:
return False
if not super().recognizes(file):
return False
# RPM files are .cpio, but let's always leave it to the RPM comparator.
# This avoids the RpmFile > BinWalkFile > CpioFile > RpmFile circular
# ordering dependency.
if file.container and isinstance(file.container.source, RpmFile):
return False
# Don't recurse; binwalk has already found everything
if isinstance(file.container, cls.CONTAINER_CLASS):
return False
unpacked = get_temporary_directory(prefix='binwalk')
logger.debug("Extracting %s to %s", file.path, unpacked.name)
binwalk.scan(
file.path,
dd='cpio:cpio',
carve=True,
quiet=True,
signature=True,
directory=unpacked.name,
)
members = {
'{} file embedded at offset {}'.format(
os.path.splitext(x)[1],
os.path.basename(os.path.splitext(x)[0]),
): x
for x in glob.glob(os.path.join(unpacked.name, '*/*'))
}
logger.debug("Found %d embedded member(s)", len(members))
if not members:
unpacked.cleanup()
return False
file._members = members
file._unpacked = unpacked
return True
def open_archive(self):
self._members = []
self._unpacked = get_temporary_directory()
logger.debug(
"Extracting Android boot image to %s", self._unpacked.name
)
subprocess.check_call(
['abootimg', '-x', os.path.abspath(self.source.path)],
cwd=self._unpacked.name,
stdout=subprocess.PIPE,
)
self._members = sorted(os.listdir(self._unpacked.name))
return self
def ensure_unpacked(self):
if hasattr(self, '_members'):
return
tmpdir = get_temporary_directory().name
self._members = collections.OrderedDict()
logger.debug("Extracting %s to %s", self.source.path, tmpdir)
with libarchive.file_reader(self.source.path) as archive:
for idx, entry in enumerate(archive):
# Always skip directories
if entry.isdir:
continue
# Save extracting excluded files
if any_excluded(entry.pathname):
continue
# Keep directory sizes small. could be improved but should be
# good enough for "ordinary" large archives.
dst = os.path.join(tmpdir, str(idx // 4096), str(idx % 4096))
root, ext = os.path.splitext(entry.pathname)
dst += ext
# Maintain a mapping of archive path to the extracted path,
# avoiding the need to sanitise filenames.
self._members[entry.pathname] = dst
logger.debug("Extracting %s to %s", entry.pathname, dst)
os.makedirs(os.path.dirname(dst), exist_ok=True)
try:
with open(dst, 'wb') as f:
for block in entry.get_blocks():
f.write(block)
except Exception as exc:
raise ContainerExtractionError(entry.pathname, exc)
logger.debug(
"Extracted %d entries from %s to %s",
len(self._members),
self.source.path,
tmpdir,
)
def fn(self):
if not hasattr(fn, '_temp_dir'):
fn._temp_dir = get_temporary_directory('binwalk').name
return fn._temp_dir
