def glob(cls, *parts, **kwargs):
"""Perform a glob with validation."""
if parts:
if len(parts) == 1:
p = parts[0]
else:
p = cls.globjoin(*parts)
if not cls.absolute:
p = cls.globjoin(cls.tempdir, p)
else:
p = cls.tempdir
res = glob.glob(p, **kwargs)
print("RESULTS: ", res)
if res:
cls.assert_equal({type(r) for r in res}, {str})
cls.assert_count_equal(glob.iglob(p, **kwargs), res)
if 'root_dir' in kwargs and kwargs['root_dir'] is not None:
kwargs['root_dir'] = os.fsencode(kwargs['root_dir'])
bres = [os.fsencode(x) for x in res]
cls.assert_count_equal(glob.glob(os.fsencode(p), **kwargs), bres)
cls.assert_count_equal(glob.iglob(os.fsencode(p), **kwargs), bres)
if bres:
cls.assert_equal({type(r) for r in bres}, {bytes})
return res
def nglob(cls, *parts, **kwargs):
"""Perform a glob with validation."""
if parts:
if len(parts) == 1:
p = parts[0]
else:
p = cls.globjoin(*parts)
if not cls.absolute:
p = cls.globjoin(cls.tempdir, p)
else:
p = cls.tempdir
p = '!' + p
if not cls.just_negative:
if not cls.absolute:
p = [cls.globjoin(cls.tempdir, '**'), p]
else:
p = ['**', p]
else:
p = [p]
res = glob.glob(p, **kwargs)
print("RESULTS: ", res)
if res:
cls.assert_equal({type(r) for r in res}, {str})
cls.assert_count_equal(glob.iglob(p, **kwargs), res)
bres = [os.fsencode(x) for x in res]
cls.assert_count_equal(glob.glob([os.fsencode(x) for x in p], **kwargs), bres)
cls.assert_count_equal(glob.iglob([os.fsencode(x) for x in p], **kwargs), bres)
if bres:
cls.assert_equal({type(r) for r in bres}, {bytes})
return res
def test_integrity(self):
"""Test glob integrity, or better put, test the path structure comes out sane."""
orig = [
pathlib.Path(x)
for x in glob.iglob('docs/**/*.md', flags=glob.GLOBSTAR)
]
results = list(
pathlib.Path('docs').glob('**/*.md', flags=glob.GLOBSTAR))
self.assertEqual(orig, results)
orig = [
pathlib.Path(x) for x in glob.iglob('**/*.md', flags=glob.GLOBSTAR)
]
results = list(pathlib.Path('').glob('**/*.md', flags=glob.GLOBSTAR))
self.assertEqual(orig, results)
def _walk_src(self, targets, flags, pipeline):
"""Walk source and parse files."""
for target in targets:
patterns = glob.globsplit(target, flags=flags)
for f in glob.iglob(patterns, flags=flags):
if not os.path.isdir(f):
self.log('', 2)
self.log('> Processing: %s' % f, 1)
if pipeline:
try:
yield pipeline[0]._run_first(f)
except Exception as e:
err = self.get_error(e)
yield [filters.SourceText('', f, '', '', err)]
else:
try:
if self.default_encoding:
encoding = filters.PYTHON_ENCODING_NAMES.get(
self.default_encoding,
self.default_encoding).lower()
encoding = codecs.lookup(encoding).name
else:
encoding = self.default_encoding
yield [filters.SourceText('', f, encoding, 'file')]
except Exception as e:
err = self.get_error(e)
yield [filters.SourceText('', f, '', '', err)]
def _walk_src(self, targets, flags, pipeline):
"""Walk source and parse files."""
found_something = False
for target in targets:
# Glob using `S` for patterns wit `|` and `O` to exclude directories.
for f in glob.iglob(target, flags=flags | glob.S | glob.O):
found_something = True
self.log('', 2)
self.log('> Processing: %s' % f, 1)
if pipeline:
try:
yield pipeline[0]._run_first(f)
except Exception as e:
err = self.get_error(e)
yield [filters.SourceText('', f, '', '', err)]
else:
try:
if self.default_encoding:
encoding = filters.PYTHON_ENCODING_NAMES.get(
self.default_encoding,
self.default_encoding).lower()
encoding = codecs.lookup(encoding).name
else:
encoding = self.default_encoding
yield [filters.SourceText('', f, encoding, 'file')]
except Exception as e:
err = self.get_error(e)
yield [filters.SourceText('', f, '', '', err)]
if not found_something:
raise RuntimeError(
'None of the source targets from the configuration match any files:\n{}'
.format('\n'.join('- {}'.format(target)
for target in targets)))
def __init__(self, options, **kwargs):
super().__init__(options, **kwargs)
self.non_words = set()
for target in self.config.get("wordlists", []):
for match in glob.iglob(target,
flags=glob.N | glob.B | glob.G | glob.S
| glob.O):
self.non_words.update(get_custom_wordlist(match))
def hash_files(verbose, debug):
"""Hash the file list."""
found = []
h = hashlib.new('md5')
for pattern in FILES_PATTERNS:
for f in glob.iglob(glob.globsplit(pattern, flags=FLAGS), flags=FLAGS):
name = f.replace('\\', '/')
found.append(name)
if verbose:
print('FILES:')
for f in sorted(found):
if verbose:
print(f)
h.update(f.encode('ascii'))
with open(f, 'rb') as f:
h.update(f.read().replace(b'\r\n', b'\n'))
result = h.hexdigest()
print('HASH: ', result)
return result
def hash_files(verbose, debug):
"""Hash the file list."""
found = []
h = hashlib.new('md5')
for pattern in FILES_PATTERNS:
for f in glob.iglob(pattern, flags=FLAGS):
name = f.replace('\\', '/')
found.append(name)
if verbose:
print('FILES:')
for f in sorted(found):
if verbose:
print(f)
h.update(f.encode('ascii'))
with open(f, 'rb') as f:
h.update(f.read().replace(b'\r\n', b'\n'))
result = h.hexdigest()
print('HASH: ', result)
return result
def _generate_include_paths(includes_node: etree.ElementBase,
root_path: str,
zip_mode: bool = False) -> typing.Generator:
for include_node in filter(is_include_node, includes_node):
attr_no_recurse: bool = include_node.get(
XmlAttributeName.NO_RECURSE) == 'True'
attr_path: str = include_node.get(XmlAttributeName.PATH).strip()
search_path: str = include_node.text
if not search_path:
PackageManager.log.error(
f'Include path at line {include_node.sourceline} in project file is empty'
)
sys.exit(1)
if not zip_mode and startswith(search_path, os.pardir):
PackageManager.log.error(
f'Include paths cannot start with "{os.pardir}"')
sys.exit(1)
if startswith(search_path, os.curdir):
search_path = search_path.replace(os.curdir, root_path, 1)
# fix invalid pattern with leading separator
if not zip_mode and startswith(search_path,
(os.path.sep, os.path.altsep)):
search_path = '**' + search_path
if '\\' in search_path:
search_path = search_path.replace('\\', '/')
# populate files list using glob patterns or relative paths
if '*' in search_path:
for include_path in glob.iglob(
search_path,
root_dir=root_path,
flags=PackageManager.DEFAULT_GLFLAGS
| glob.GLOBSTAR if not attr_no_recurse else 0x0):
yield os.path.join(root_path, include_path), attr_path
elif not os.path.isabs(search_path):
test_path = os.path.normpath(
os.path.join(root_path, search_path))
if os.path.isfile(test_path):
yield test_path, attr_path
elif os.path.isdir(test_path):
yield from PackageManager._match(
test_path,
'*.*',
user_path=attr_path,
no_recurse=attr_no_recurse)
else:
for include_path in glob.iglob(
search_path,
root_dir=root_path,
flags=PackageManager.DEFAULT_GLFLAGS
| glob.GLOBSTAR if not attr_no_recurse else 0x0):
yield os.path.join(root_path, include_path), attr_path
# populate files list using absolute paths
else:
if not zip_mode and root_path not in search_path:
PackageManager.log.error(
f'Cannot include path outside RootDir: "{search_path}"'
)
sys.exit(1)
search_path = os.path.abspath(os.path.normpath(search_path))
if os.path.isfile(search_path):
yield search_path, attr_path
else:
yield from PackageManager._match(
search_path,
'*.*',
user_path=attr_path,
no_recurse=attr_no_recurse)
for match_node in filter(is_match_node, includes_node):
attr_in: str = match_node.get(XmlAttributeName.IN).strip()
attr_no_recurse: bool = match_node.get(
XmlAttributeName.NO_RECURSE) == 'True'
attr_exclude: str = match_node.get(
XmlAttributeName.EXCLUDE).strip()
attr_path: str = match_node.get(XmlAttributeName.PATH).strip()
in_path: str = os.path.normpath(attr_in)
if in_path == os.pardir or startswith(in_path, os.pardir):
in_path = in_path.replace(
os.pardir,
os.path.normpath(os.path.join(root_path, os.pardir)), 1)
elif in_path == os.curdir or startswith(in_path, os.curdir):
in_path = in_path.replace(os.curdir, root_path, 1)
if not os.path.isabs(in_path):
in_path = os.path.join(root_path, in_path)
elif zip_mode and root_path not in in_path:
PackageManager.log.error(
f'Cannot match path outside RootDir: "{in_path}"')
sys.exit(1)
if not os.path.isdir(in_path):
PackageManager.log.error(
f'Cannot match path that does not exist or is not a directory: "{in_path}"'
)
sys.exit(1)
match_text: str = match_node.text
if startswith(match_text, '.'):
PackageManager.log.error(
f'Match pattern at line {match_node.sourceline} in project file is not a valid wildcard pattern'
)
sys.exit(1)
yield from PackageManager._match(in_path,
match_text,
exclude_pattern=attr_exclude,
user_path=attr_path,
no_recurse=attr_no_recurse)
def test_limit_iglob(self):
"""Test expansion limit of `iglob`."""
with self.assertRaises(_wcparse.PatternLimitException):
list(glob.iglob('{1..11}', flags=glob.BRACE, limit=10))
