def test_get_all_files_from(self, tmpdir):
"""Test get_all_files_from()."""
test_dir = Path(str(tmpdir)).resolve()
def touch_file(path):
try:
path.parent.mkdir(parents=True)
except OSError as e:
if e.errno != errno.EEXIST:
# was created in previous iteration
pass
path.touch()
py_files = {
str(test_dir / 'test_path/file.py'),
str(test_dir / 'test_path/some.py')
}
test_files = {str(test_dir / 'test_path/test')}
hidden_files = {str(test_dir / 'test_path/.hidden')}
git_files = {str(test_dir / 'test_path/.git/object')}
all_files = set(
itertools.chain(py_files, test_files, hidden_files, git_files))
for f in all_files:
touch_file(Path(f))
test_dir = str(test_dir)
assert set(get_all_files_from(test_dir)) == all_files
assert set(get_all_files_from(test_dir, path_filter=skip_git_files)) == \
set(itertools.chain(py_files, test_files, hidden_files))
assert set(get_all_files_from(test_dir, file_filter=lambda x: x.endswith('.py'))) == \
py_files
assert set(get_all_files_from(test_dir, path_filter=hidden_path_filter)) == \
set(itertools.chain(py_files, test_files))
def execute(self, arguments):
"""Task code.
:param arguments: dictionary with task arguments
:return: {}, results
"""
self._strict_assert(arguments.get('ecosystem'))
self._strict_assert(arguments.get('name'))
self._strict_assert(arguments.get('version'))
cache_path = ObjectCache.get_from_dict(arguments).get_source_tarball()
results = []
for path in get_all_files_from(cache_path, path_filter=skip_git_files):
self.log.debug("path = %s", path)
bw = TimedCommand(['binwalk', '-B', path])
status, output, error = bw.run(timeout=60)
self.log.debug("status = %s, error = %s", status, error)
self.log.debug("output = %s", output)
parsed_binwalk = self.parse_binwalk(output)
results.append({
"path": os.path.relpath(path, cache_path),
"output": parsed_binwalk,
})
return {'summary': [], 'status': 'success', 'details': results}
def execute(self, arguments):
self._strict_assert(arguments.get('ecosystem'))
self._strict_assert(arguments.get('name'))
self._strict_assert(arguments.get('version'))
results = []
cache_path = ObjectCache.get_from_dict(arguments).get_extracted_source_tarball()
def worker(path):
mime = TimedCommand.get_command_output(['file', path, '-b', '-i']).pop()
self.log.debug("%s mime = %s", path, mime)
typ = TimedCommand.get_command_output(['file', path, '-b'])
self.log.debug("%s filetype = %s", path, typ)
linguist = None
if 'charset=binary' not in mime:
linguist = self._parse_linguist(
TimedCommand.get_command_output(['linguist', path])
)
self.log.debug("%s linguist output = %s", path, linguist)
results.append({
"type": typ,
"output": linguist,
"path": os.path.relpath(path, cache_path),
})
with ThreadPool(target=worker) as tp:
for path in get_all_files_from(cache_path, path_filter=skip_git_files):
tp.add_task(path)
return {'summary': [], 'status': 'success', 'details': results}
def test_get_all_files_from(self, tmpdir):
"""Test get_all_files_from()."""
test_dir = os.path.abspath(str(tmpdir))
def touch_file(path):
abspath = os.path.join(test_dir, path)
abs_dir_path = os.path.dirname(abspath)
try:
os.makedirs(abs_dir_path)
except OSError as e:
if e.errno != errno.EEXIST:
# was created in previous iteration
pass
with open(abspath, "w") as fd:
fd.write("banana")
py_files = {
os.path.join(test_dir, 'test_path/file.py'),
os.path.join(test_dir, 'test_path/some.py'),
}
test_files = {
os.path.join(test_dir, 'test_path/test'),
}
hidden_files = {
os.path.join(test_dir, 'test_path/.hidden'),
}
git_files = {
os.path.join(test_dir, 'test_path/.git/object'),
}
all_files = set(
itertools.chain(py_files, test_files, hidden_files, git_files))
for f in all_files:
touch_file(f)
assert set(get_all_files_from(test_dir)) == all_files
assert set(get_all_files_from(test_dir, path_filter=skip_git_files)) == \
set(itertools.chain(py_files, test_files, hidden_files))
assert set(get_all_files_from(test_dir, file_filter=lambda x: x.endswith('.py'))) == \
py_files
assert set(get_all_files_from(test_dir, path_filter=hidden_path_filter)) == \
set(itertools.chain(py_files, test_files))
