def test_get_all_files_from(self, tmpdir): """Test get_all_files_from().""" test_dir = Path(str(tmpdir)).resolve() def touch_file(path): try: path.parent.mkdir(parents=True) except OSError as e: if e.errno != errno.EEXIST: # was created in previous iteration pass path.touch() py_files = { str(test_dir / 'test_path/file.py'), str(test_dir / 'test_path/some.py') } test_files = {str(test_dir / 'test_path/test')} hidden_files = {str(test_dir / 'test_path/.hidden')} git_files = {str(test_dir / 'test_path/.git/object')} all_files = set( itertools.chain(py_files, test_files, hidden_files, git_files)) for f in all_files: touch_file(Path(f)) test_dir = str(test_dir) assert set(get_all_files_from(test_dir)) == all_files assert set(get_all_files_from(test_dir, path_filter=skip_git_files)) == \ set(itertools.chain(py_files, test_files, hidden_files)) assert set(get_all_files_from(test_dir, file_filter=lambda x: x.endswith('.py'))) == \ py_files assert set(get_all_files_from(test_dir, path_filter=hidden_path_filter)) == \ set(itertools.chain(py_files, test_files))
def execute(self, arguments): """Task code. :param arguments: dictionary with task arguments :return: {}, results """ self._strict_assert(arguments.get('ecosystem')) self._strict_assert(arguments.get('name')) self._strict_assert(arguments.get('version')) cache_path = ObjectCache.get_from_dict(arguments).get_source_tarball() results = [] for path in get_all_files_from(cache_path, path_filter=skip_git_files): self.log.debug("path = %s", path) bw = TimedCommand(['binwalk', '-B', path]) status, output, error = bw.run(timeout=60) self.log.debug("status = %s, error = %s", status, error) self.log.debug("output = %s", output) parsed_binwalk = self.parse_binwalk(output) results.append({ "path": os.path.relpath(path, cache_path), "output": parsed_binwalk, }) return {'summary': [], 'status': 'success', 'details': results}
def execute(self, arguments): self._strict_assert(arguments.get('ecosystem')) self._strict_assert(arguments.get('name')) self._strict_assert(arguments.get('version')) results = [] cache_path = ObjectCache.get_from_dict(arguments).get_extracted_source_tarball() def worker(path): mime = TimedCommand.get_command_output(['file', path, '-b', '-i']).pop() self.log.debug("%s mime = %s", path, mime) typ = TimedCommand.get_command_output(['file', path, '-b']) self.log.debug("%s filetype = %s", path, typ) linguist = None if 'charset=binary' not in mime: linguist = self._parse_linguist( TimedCommand.get_command_output(['linguist', path]) ) self.log.debug("%s linguist output = %s", path, linguist) results.append({ "type": typ, "output": linguist, "path": os.path.relpath(path, cache_path), }) with ThreadPool(target=worker) as tp: for path in get_all_files_from(cache_path, path_filter=skip_git_files): tp.add_task(path) return {'summary': [], 'status': 'success', 'details': results}
def test_get_all_files_from(self, tmpdir): """Test get_all_files_from().""" test_dir = os.path.abspath(str(tmpdir)) def touch_file(path): abspath = os.path.join(test_dir, path) abs_dir_path = os.path.dirname(abspath) try: os.makedirs(abs_dir_path) except OSError as e: if e.errno != errno.EEXIST: # was created in previous iteration pass with open(abspath, "w") as fd: fd.write("banana") py_files = { os.path.join(test_dir, 'test_path/file.py'), os.path.join(test_dir, 'test_path/some.py'), } test_files = { os.path.join(test_dir, 'test_path/test'), } hidden_files = { os.path.join(test_dir, 'test_path/.hidden'), } git_files = { os.path.join(test_dir, 'test_path/.git/object'), } all_files = set( itertools.chain(py_files, test_files, hidden_files, git_files)) for f in all_files: touch_file(f) assert set(get_all_files_from(test_dir)) == all_files assert set(get_all_files_from(test_dir, path_filter=skip_git_files)) == \ set(itertools.chain(py_files, test_files, hidden_files)) assert set(get_all_files_from(test_dir, file_filter=lambda x: x.endswith('.py'))) == \ py_files assert set(get_all_files_from(test_dir, path_filter=hidden_path_filter)) == \ set(itertools.chain(py_files, test_files))