def test_recursive_hidden_bytes(self):
"""Test non-recursive search with byte strings."""
walker = wcmatch.WcMatch(os.fsencode(self.tempdir), b'*.txt', None,
True, True, self.default_flags)
self.crawl_files(walker)
self.assertEqual(self.skipped, 4)
self.assertEqual(sorted(self.files),
self.norm_list([b'.hidden/a.txt', b'a.txt']))
def test_recursive_hidden_folder_exclude_inverse(self):
"""Test non-recursive search with inverse."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', '*|-.hidden', True,
True, self.default_flags)
self.crawl_files(walker)
self.assertEqual(self.skipped, 4)
self.assertEqual(sorted(self.files),
self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_non_recursive_inverse(self):
"""Test non-recursive inverse search."""
walker = wcmatch.WcMatch(self.tempdir, '*.*|-*.file', None, False,
False, self.default_flags)
self.crawl_files(walker)
self.assertEqual(self.skipped, 2)
self.assertEqual(sorted(self.files),
self.norm_list(['a.txt', 'c.txt.bak']))
def test_recursive_hidden(self):
"""Test non-recursive search."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
self.crawl_files(walker)
self.assertEqual(self.skipped, 4)
self.assertEqual(sorted(self.files),
self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_full_path_exclude(self):
"""Test full path exclude."""
walker = wcmatch.WcMatch(
self.tempdir, '*.txt', '**/.hidden', True, True,
self.default_flags | wcmatch.DIRPATHNAME | wcmatch.GLOBSTAR)
self.crawl_files(walker)
self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_full_file(self):
"""Test full file."""
walker = wcmatch.WcMatch(
self.tempdir, '**/*.txt|-**/.hidden/*', None, True, True,
self.default_flags | wcmatch.FILEPATHNAME | wcmatch.GLOBSTAR)
self.crawl_files(walker)
self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_skip_override(self):
"""Test `on_skip` override."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
walker.on_skip = lambda base, name: '<SKIPPED>'
self.crawl_files(walker)
self.assertEqual(len(self.skip_records), 4)
def test_match_base_filepath(self):
"""Test `MATCHBASE` with filepath."""
walker = wcmatch.WcMatch(self.tempdir,
'*.txt',
flags=self.default_flags | wcmatch.RECURSIVE
| wcmatch.HIDDEN | wcmatch.FILEPATHNAME
| wcmatch.MATCHBASE)
self.crawl_files(walker)
self.assertEqual(sorted(self.files),
self.norm_list(['a.txt', '.hidden/a.txt']))
def test_avoid_symlinks(self):
"""Test avoiding symlinks."""
walker = wcmatch.WcMatch(self.tempdir,
'*.txt',
flags=self.default_flags | wcmatch.RECURSIVE
| wcmatch.HIDDEN)
self.crawl_files(walker)
self.assertEqual(sorted(self.files),
self.norm_list(['a.txt', '.hidden/a.txt']))
def test_symlinks(self):
"""Test symlinks."""
walker = wcmatch.WcMatch(
self.tempdir, '*.txt', None, self.default_flags | wcmatch.RECURSIVE
| wcmatch.HIDDEN | wcmatch.SYMLINKS)
self.crawl_files(walker)
self.assertEqual(
sorted(self.files),
self.norm_list(['a.txt', '.hidden/a.txt', 'sym1/a.txt']))
def test_error_override(self):
"""Test `on_eror` override."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
walker.on_validate_file = lambda base, name: self.force_err()
walker.on_error = lambda base, name: '<ERROR>'
self.crawl_files(walker)
self.assertEqual(len(self.error_records), 2)
def test_empty_string_file(self):
"""Test when file pattern is an empty string."""
walker = wcmatch.WcMatch(self.tempdir, '', None, True, True,
self.default_flags)
self.crawl_files(walker)
self.assertEqual(
sorted(self.files),
self.norm_list([
'a.txt', '.hidden/a.txt', '.hidden/b.file', 'b.file',
'.hidden_file', 'c.txt.bak'
]))
def list_(ctx, target):
targets = [
os.path.dirname(x)
for x in sorted(
wcmatch.WcMatch(
f"{target}", "BUILD.yaml", GLOB_EXCLUDES, flags=wcmatch.RECURSIVE
).match()
)
]
logger.info(f"Found {len(targets)} target(s):")
for t in targets:
logger.info(t)
def test_abort_early(self):
"""Test aborting early."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt*', None, True, True,
self.default_flags)
walker.kill()
records = 0
for f in walker.imatch():
records += 1
self.assertTrue(records == 1 or walker.get_skipped() == 1)
def get_python_filenames(abs_path: str) -> list:
""":arg
path_arg: absolute directory of the code base
return a list of paths that are organic to the code base
"""
# TODO: try not to hardcode this deny list 'site-packages | node_modules'
paths = wcmatch.WcMatch(abs_path,
"*.py",
"site-packages|node_modules",
flags=wcmatch.RECURSIVE).match()
return paths
def test_recursive_hidden_folder_exclude(self):
"""Test non-recursive search."""
walker = wcmatch.WcMatch(self.tempdir,
'*.txt',
exclude_pattern='.hidden',
flags=self.default_flags | wcmatch.RECURSIVE
| wcmatch.HIDDEN)
self.crawl_files(walker)
self.assertEqual(self.skipped, 3)
self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def find_script_paths_from_folder(
root_dir: str,
*,
no_recurse: bool,
matcher: wcmatch.WcMatch = None) -> Generator:
"""Yields existing script paths starting from absolute folder path"""
if not matcher:
user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0
matcher = wcmatch.WcMatch(root_dir,
'*.psc',
flags=wcmatch.IGNORECASE | user_flags)
for script_path in matcher.imatch():
yield script_path
def test_errors(self):
"""Test errors."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
walker.on_validate_file = lambda base, name: self.force_err()
self.crawl_files(walker)
self.assertEqual(sorted(self.files), self.norm_list([]))
self.errors = []
self.skipped = 0
self.files = []
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
walker.on_validate_directory = lambda base, name: self.force_err()
self.crawl_files(walker)
self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_abort_early(self):
"""Test aborting early."""
walker = wcmatch.WcMatch(self.tempdir,
'*.txt*',
flags=self.default_flags | wcmatch.RECURSIVE
| wcmatch.HIDDEN)
walker.kill()
records = 0
for f in walker.imatch():
records += 1
self.assertTrue(records == 0 or walker.get_skipped() == 0)
def _match(root_dir: str,
file_pattern: str,
*,
exclude_pattern: str = '',
user_path: str = '',
no_recurse: bool = False) -> typing.Generator:
user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0
matcher = wcmatch.WcMatch(root_dir,
file_pattern,
exclude_pattern=exclude_pattern,
flags=PackageManager.DEFAULT_WCFLAGS
| user_flags)
matcher.on_reset()
matcher._skipped = 0
for file_path in matcher._walk():
yield file_path, user_path
def load_dict(name: str,
root: str,
dictobj: dict,
debug: bool = False) -> list:
"""Filter directory based on given cleanlist. Returns paths of given files. This function utilizes
wcmatch to identify files.
:param name: what to call this event in the debug logs.
:type name: str
:param root: directory to start the file search in.
:type root: str
:param dictobj: unprocessed dictionary to load.
:type dictobj: dict
:param debug: send debug messages
:type debug: bool
:returns list: paths of files found through the cleanlist."""
file_search = ""
exclusions = ""
f, e = 0, 0
for key in dictobj.keys():
if dictobj[key] is False:
if f == 0: # wcmatch requires no '|' on the first entry.
file_search += key
else:
file_search += "|" + key
f += 1
elif dictobj[key] is True:
if e == 0:
exclusions += key
else:
exclusions += "|" + key
e += 1
message("{} file_search: {}".format(name, file_search), "debug")
message("{} exclusions: {}".format(name, exclusions), "debug")
message("{} is searching for files in directory: {}".format(name, root),
"debug")
pl = wcmatch.WcMatch(root,
file_search,
exclusions,
flags=wcmatch.RECURSIVE | wcmatch.GLOBSTAR
| wcmatch.PATHNAME).match()
return pl
def test_abort(self):
"""Test aborting."""
walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True,
self.default_flags)
records = 0
for f in walker.imatch():
records += 1
walker.kill()
self.assertEqual(records, 1)
# Reset our test tracker along with the walker object
self.errors = []
self.skipped = 0
self.files = []
records = 0
walker.reset()
self.crawl_files(walker)
self.assertEqual(sorted(self.files),
self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_limit_wcmatch(self):
"""Test expansion limit of `globmatch`."""
with self.assertRaises(_wcparse.PatternLimitException):
wcmatch.WcMatch('.', '{1..11}', flags=wcmatch.BRACE, limit=10)
def _get_script_paths_from_folders_node(self) -> typing.Generator:
"""Returns script paths from the Folders element array"""
for folder_node in filter(is_folder_node, self.folders_node):
self.try_fix_namespace_path(folder_node)
attr_no_recurse: bool = folder_node.get(
XmlAttributeName.NO_RECURSE) == 'True'
folder_path: str = folder_node.text
# handle . and .. in path
if folder_path == os.pardir or startswith(folder_path, os.pardir):
folder_path = folder_path.replace(
os.pardir,
os.path.normpath(os.path.join(self.project_path,
os.pardir)), 1)
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
if folder_path == os.curdir or startswith(folder_path, os.curdir):
folder_path = folder_path.replace(os.curdir, self.project_path,
1)
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
if startswith(folder_path, self.remote_schemas, ignorecase=True):
local_path = self._get_remote_path(folder_node)
PapyrusProject.log.info(
f'Adding import path from remote: "{local_path}"...')
self.import_paths.insert(0, local_path)
PapyrusProject.log.info(
f'Adding folder path from remote: "{local_path}"...')
yield from PathHelper.find_script_paths_from_folder(
local_path, no_recurse=attr_no_recurse)
continue
folder_path = os.path.normpath(folder_path)
# try to add absolute path
if os.path.isabs(folder_path) and os.path.isdir(folder_path):
yield from PathHelper.find_script_paths_from_folder(
folder_path, no_recurse=attr_no_recurse)
continue
# try to add project-relative folder path
test_path = os.path.join(self.project_path, folder_path)
if os.path.isdir(test_path):
# count scripts to avoid issue where an errant `test_path` may exist and contain no sources
# this can be a problem if that folder contains sources but user error is hard to fix
test_passed = False
user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0
matcher = wcmatch.WcMatch(test_path,
'*.psc',
flags=wcmatch.IGNORECASE
| user_flags)
for _ in matcher.imatch():
test_passed = True
break
if test_passed:
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse=attr_no_recurse, matcher=matcher)
continue
# try to add import-relative folder path
for import_path in self.import_paths:
test_path = os.path.join(import_path, folder_path)
if os.path.isdir(test_path):
yield from PathHelper.find_script_paths_from_folder(
test_path, no_recurse=attr_no_recurse)
