def glob(cls, *parts, **kwargs): """Perform a glob with validation.""" if parts: if len(parts) == 1: p = parts[0] else: p = cls.globjoin(*parts) if not cls.absolute: p = cls.globjoin(cls.tempdir, p) else: p = cls.tempdir res = glob.glob(p, **kwargs) print("RESULTS: ", res) if res: cls.assert_equal({type(r) for r in res}, {str}) cls.assert_count_equal(glob.iglob(p, **kwargs), res) if 'root_dir' in kwargs and kwargs['root_dir'] is not None: kwargs['root_dir'] = os.fsencode(kwargs['root_dir']) bres = [os.fsencode(x) for x in res] cls.assert_count_equal(glob.glob(os.fsencode(p), **kwargs), bres) cls.assert_count_equal(glob.iglob(os.fsencode(p), **kwargs), bres) if bres: cls.assert_equal({type(r) for r in bres}, {bytes}) return res
def nglob(cls, *parts, **kwargs): """Perform a glob with validation.""" if parts: if len(parts) == 1: p = parts[0] else: p = cls.globjoin(*parts) if not cls.absolute: p = cls.globjoin(cls.tempdir, p) else: p = cls.tempdir p = '!' + p if not cls.just_negative: if not cls.absolute: p = [cls.globjoin(cls.tempdir, '**'), p] else: p = ['**', p] else: p = [p] res = glob.glob(p, **kwargs) print("RESULTS: ", res) if res: cls.assert_equal({type(r) for r in res}, {str}) cls.assert_count_equal(glob.iglob(p, **kwargs), res) bres = [os.fsencode(x) for x in res] cls.assert_count_equal(glob.glob([os.fsencode(x) for x in p], **kwargs), bres) cls.assert_count_equal(glob.iglob([os.fsencode(x) for x in p], **kwargs), bres) if bres: cls.assert_equal({type(r) for r in bres}, {bytes}) return res
def test_integrity(self): """Test glob integrity, or better put, test the path structure comes out sane.""" orig = [ pathlib.Path(x) for x in glob.iglob('docs/**/*.md', flags=glob.GLOBSTAR) ] results = list( pathlib.Path('docs').glob('**/*.md', flags=glob.GLOBSTAR)) self.assertEqual(orig, results) orig = [ pathlib.Path(x) for x in glob.iglob('**/*.md', flags=glob.GLOBSTAR) ] results = list(pathlib.Path('').glob('**/*.md', flags=glob.GLOBSTAR)) self.assertEqual(orig, results)
def _walk_src(self, targets, flags, pipeline): """Walk source and parse files.""" for target in targets: patterns = glob.globsplit(target, flags=flags) for f in glob.iglob(patterns, flags=flags): if not os.path.isdir(f): self.log('', 2) self.log('> Processing: %s' % f, 1) if pipeline: try: yield pipeline[0]._run_first(f) except Exception as e: err = self.get_error(e) yield [filters.SourceText('', f, '', '', err)] else: try: if self.default_encoding: encoding = filters.PYTHON_ENCODING_NAMES.get( self.default_encoding, self.default_encoding).lower() encoding = codecs.lookup(encoding).name else: encoding = self.default_encoding yield [filters.SourceText('', f, encoding, 'file')] except Exception as e: err = self.get_error(e) yield [filters.SourceText('', f, '', '', err)]
def _walk_src(self, targets, flags, pipeline): """Walk source and parse files.""" found_something = False for target in targets: # Glob using `S` for patterns wit `|` and `O` to exclude directories. for f in glob.iglob(target, flags=flags | glob.S | glob.O): found_something = True self.log('', 2) self.log('> Processing: %s' % f, 1) if pipeline: try: yield pipeline[0]._run_first(f) except Exception as e: err = self.get_error(e) yield [filters.SourceText('', f, '', '', err)] else: try: if self.default_encoding: encoding = filters.PYTHON_ENCODING_NAMES.get( self.default_encoding, self.default_encoding).lower() encoding = codecs.lookup(encoding).name else: encoding = self.default_encoding yield [filters.SourceText('', f, encoding, 'file')] except Exception as e: err = self.get_error(e) yield [filters.SourceText('', f, '', '', err)] if not found_something: raise RuntimeError( 'None of the source targets from the configuration match any files:\n{}' .format('\n'.join('- {}'.format(target) for target in targets)))
def __init__(self, options, **kwargs): super().__init__(options, **kwargs) self.non_words = set() for target in self.config.get("wordlists", []): for match in glob.iglob(target, flags=glob.N | glob.B | glob.G | glob.S | glob.O): self.non_words.update(get_custom_wordlist(match))
def hash_files(verbose, debug): """Hash the file list.""" found = [] h = hashlib.new('md5') for pattern in FILES_PATTERNS: for f in glob.iglob(glob.globsplit(pattern, flags=FLAGS), flags=FLAGS): name = f.replace('\\', '/') found.append(name) if verbose: print('FILES:') for f in sorted(found): if verbose: print(f) h.update(f.encode('ascii')) with open(f, 'rb') as f: h.update(f.read().replace(b'\r\n', b'\n')) result = h.hexdigest() print('HASH: ', result) return result
def hash_files(verbose, debug): """Hash the file list.""" found = [] h = hashlib.new('md5') for pattern in FILES_PATTERNS: for f in glob.iglob(pattern, flags=FLAGS): name = f.replace('\\', '/') found.append(name) if verbose: print('FILES:') for f in sorted(found): if verbose: print(f) h.update(f.encode('ascii')) with open(f, 'rb') as f: h.update(f.read().replace(b'\r\n', b'\n')) result = h.hexdigest() print('HASH: ', result) return result
def _generate_include_paths(includes_node: etree.ElementBase, root_path: str, zip_mode: bool = False) -> typing.Generator: for include_node in filter(is_include_node, includes_node): attr_no_recurse: bool = include_node.get( XmlAttributeName.NO_RECURSE) == 'True' attr_path: str = include_node.get(XmlAttributeName.PATH).strip() search_path: str = include_node.text if not search_path: PackageManager.log.error( f'Include path at line {include_node.sourceline} in project file is empty' ) sys.exit(1) if not zip_mode and startswith(search_path, os.pardir): PackageManager.log.error( f'Include paths cannot start with "{os.pardir}"') sys.exit(1) if startswith(search_path, os.curdir): search_path = search_path.replace(os.curdir, root_path, 1) # fix invalid pattern with leading separator if not zip_mode and startswith(search_path, (os.path.sep, os.path.altsep)): search_path = '**' + search_path if '\\' in search_path: search_path = search_path.replace('\\', '/') # populate files list using glob patterns or relative paths if '*' in search_path: for include_path in glob.iglob( search_path, root_dir=root_path, flags=PackageManager.DEFAULT_GLFLAGS | glob.GLOBSTAR if not attr_no_recurse else 0x0): yield os.path.join(root_path, include_path), attr_path elif not os.path.isabs(search_path): test_path = os.path.normpath( os.path.join(root_path, search_path)) if os.path.isfile(test_path): yield test_path, attr_path elif os.path.isdir(test_path): yield from PackageManager._match( test_path, '*.*', user_path=attr_path, no_recurse=attr_no_recurse) else: for include_path in glob.iglob( search_path, root_dir=root_path, flags=PackageManager.DEFAULT_GLFLAGS | glob.GLOBSTAR if not attr_no_recurse else 0x0): yield os.path.join(root_path, include_path), attr_path # populate files list using absolute paths else: if not zip_mode and root_path not in search_path: PackageManager.log.error( f'Cannot include path outside RootDir: "{search_path}"' ) sys.exit(1) search_path = os.path.abspath(os.path.normpath(search_path)) if os.path.isfile(search_path): yield search_path, attr_path else: yield from PackageManager._match( search_path, '*.*', user_path=attr_path, no_recurse=attr_no_recurse) for match_node in filter(is_match_node, includes_node): attr_in: str = match_node.get(XmlAttributeName.IN).strip() attr_no_recurse: bool = match_node.get( XmlAttributeName.NO_RECURSE) == 'True' attr_exclude: str = match_node.get( XmlAttributeName.EXCLUDE).strip() attr_path: str = match_node.get(XmlAttributeName.PATH).strip() in_path: str = os.path.normpath(attr_in) if in_path == os.pardir or startswith(in_path, os.pardir): in_path = in_path.replace( os.pardir, os.path.normpath(os.path.join(root_path, os.pardir)), 1) elif in_path == os.curdir or startswith(in_path, os.curdir): in_path = in_path.replace(os.curdir, root_path, 1) if not os.path.isabs(in_path): in_path = os.path.join(root_path, in_path) elif zip_mode and root_path not in in_path: PackageManager.log.error( f'Cannot match path outside RootDir: "{in_path}"') sys.exit(1) if not os.path.isdir(in_path): PackageManager.log.error( f'Cannot match path that does not exist or is not a directory: "{in_path}"' ) sys.exit(1) match_text: str = match_node.text if startswith(match_text, '.'): PackageManager.log.error( f'Match pattern at line {match_node.sourceline} in project file is not a valid wildcard pattern' ) sys.exit(1) yield from PackageManager._match(in_path, match_text, exclude_pattern=attr_exclude, user_path=attr_path, no_recurse=attr_no_recurse)
def test_limit_iglob(self): """Test expansion limit of `iglob`.""" with self.assertRaises(_wcparse.PatternLimitException): list(glob.iglob('{1..11}', flags=glob.BRACE, limit=10))