def test_recursive_hidden_bytes(self): """Test non-recursive search with byte strings.""" walker = wcmatch.WcMatch(os.fsencode(self.tempdir), b'*.txt', None, True, True, self.default_flags) self.crawl_files(walker) self.assertEqual(self.skipped, 4) self.assertEqual(sorted(self.files), self.norm_list([b'.hidden/a.txt', b'a.txt']))
def test_recursive_hidden_folder_exclude_inverse(self): """Test non-recursive search with inverse.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', '*|-.hidden', True, True, self.default_flags) self.crawl_files(walker) self.assertEqual(self.skipped, 4) self.assertEqual(sorted(self.files), self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_non_recursive_inverse(self): """Test non-recursive inverse search.""" walker = wcmatch.WcMatch(self.tempdir, '*.*|-*.file', None, False, False, self.default_flags) self.crawl_files(walker) self.assertEqual(self.skipped, 2) self.assertEqual(sorted(self.files), self.norm_list(['a.txt', 'c.txt.bak']))
def test_recursive_hidden(self): """Test non-recursive search.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) self.crawl_files(walker) self.assertEqual(self.skipped, 4) self.assertEqual(sorted(self.files), self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_full_path_exclude(self): """Test full path exclude.""" walker = wcmatch.WcMatch( self.tempdir, '*.txt', '**/.hidden', True, True, self.default_flags | wcmatch.DIRPATHNAME | wcmatch.GLOBSTAR) self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_full_file(self): """Test full file.""" walker = wcmatch.WcMatch( self.tempdir, '**/*.txt|-**/.hidden/*', None, True, True, self.default_flags | wcmatch.FILEPATHNAME | wcmatch.GLOBSTAR) self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_skip_override(self): """Test `on_skip` override.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) walker.on_skip = lambda base, name: '<SKIPPED>' self.crawl_files(walker) self.assertEqual(len(self.skip_records), 4)
def test_match_base_filepath(self): """Test `MATCHBASE` with filepath.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', flags=self.default_flags | wcmatch.RECURSIVE | wcmatch.HIDDEN | wcmatch.FILEPATHNAME | wcmatch.MATCHBASE) self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['a.txt', '.hidden/a.txt']))
def test_avoid_symlinks(self): """Test avoiding symlinks.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', flags=self.default_flags | wcmatch.RECURSIVE | wcmatch.HIDDEN) self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['a.txt', '.hidden/a.txt']))
def test_symlinks(self): """Test symlinks.""" walker = wcmatch.WcMatch( self.tempdir, '*.txt', None, self.default_flags | wcmatch.RECURSIVE | wcmatch.HIDDEN | wcmatch.SYMLINKS) self.crawl_files(walker) self.assertEqual( sorted(self.files), self.norm_list(['a.txt', '.hidden/a.txt', 'sym1/a.txt']))
def test_error_override(self): """Test `on_eror` override.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) walker.on_validate_file = lambda base, name: self.force_err() walker.on_error = lambda base, name: '<ERROR>' self.crawl_files(walker) self.assertEqual(len(self.error_records), 2)
def test_empty_string_file(self): """Test when file pattern is an empty string.""" walker = wcmatch.WcMatch(self.tempdir, '', None, True, True, self.default_flags) self.crawl_files(walker) self.assertEqual( sorted(self.files), self.norm_list([ 'a.txt', '.hidden/a.txt', '.hidden/b.file', 'b.file', '.hidden_file', 'c.txt.bak' ]))
def list_(ctx, target): targets = [ os.path.dirname(x) for x in sorted( wcmatch.WcMatch( f"{target}", "BUILD.yaml", GLOB_EXCLUDES, flags=wcmatch.RECURSIVE ).match() ) ] logger.info(f"Found {len(targets)} target(s):") for t in targets: logger.info(t)
def test_abort_early(self): """Test aborting early.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt*', None, True, True, self.default_flags) walker.kill() records = 0 for f in walker.imatch(): records += 1 self.assertTrue(records == 1 or walker.get_skipped() == 1)
def get_python_filenames(abs_path: str) -> list: """:arg path_arg: absolute directory of the code base return a list of paths that are organic to the code base """ # TODO: try not to hardcode this deny list 'site-packages | node_modules' paths = wcmatch.WcMatch(abs_path, "*.py", "site-packages|node_modules", flags=wcmatch.RECURSIVE).match() return paths
def test_recursive_hidden_folder_exclude(self): """Test non-recursive search.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', exclude_pattern='.hidden', flags=self.default_flags | wcmatch.RECURSIVE | wcmatch.HIDDEN) self.crawl_files(walker) self.assertEqual(self.skipped, 3) self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def find_script_paths_from_folder( root_dir: str, *, no_recurse: bool, matcher: wcmatch.WcMatch = None) -> Generator: """Yields existing script paths starting from absolute folder path""" if not matcher: user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0 matcher = wcmatch.WcMatch(root_dir, '*.psc', flags=wcmatch.IGNORECASE | user_flags) for script_path in matcher.imatch(): yield script_path
def test_errors(self): """Test errors.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) walker.on_validate_file = lambda base, name: self.force_err() self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list([])) self.errors = [] self.skipped = 0 self.files = [] walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) walker.on_validate_directory = lambda base, name: self.force_err() self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['a.txt']))
def test_abort_early(self): """Test aborting early.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt*', flags=self.default_flags | wcmatch.RECURSIVE | wcmatch.HIDDEN) walker.kill() records = 0 for f in walker.imatch(): records += 1 self.assertTrue(records == 0 or walker.get_skipped() == 0)
def _match(root_dir: str, file_pattern: str, *, exclude_pattern: str = '', user_path: str = '', no_recurse: bool = False) -> typing.Generator: user_flags = wcmatch.RECURSIVE if not no_recurse else 0x0 matcher = wcmatch.WcMatch(root_dir, file_pattern, exclude_pattern=exclude_pattern, flags=PackageManager.DEFAULT_WCFLAGS | user_flags) matcher.on_reset() matcher._skipped = 0 for file_path in matcher._walk(): yield file_path, user_path
def load_dict(name: str, root: str, dictobj: dict, debug: bool = False) -> list: """Filter directory based on given cleanlist. Returns paths of given files. This function utilizes wcmatch to identify files. :param name: what to call this event in the debug logs. :type name: str :param root: directory to start the file search in. :type root: str :param dictobj: unprocessed dictionary to load. :type dictobj: dict :param debug: send debug messages :type debug: bool :returns list: paths of files found through the cleanlist.""" file_search = "" exclusions = "" f, e = 0, 0 for key in dictobj.keys(): if dictobj[key] is False: if f == 0: # wcmatch requires no '|' on the first entry. file_search += key else: file_search += "|" + key f += 1 elif dictobj[key] is True: if e == 0: exclusions += key else: exclusions += "|" + key e += 1 message("{} file_search: {}".format(name, file_search), "debug") message("{} exclusions: {}".format(name, exclusions), "debug") message("{} is searching for files in directory: {}".format(name, root), "debug") pl = wcmatch.WcMatch(root, file_search, exclusions, flags=wcmatch.RECURSIVE | wcmatch.GLOBSTAR | wcmatch.PATHNAME).match() return pl
def test_abort(self): """Test aborting.""" walker = wcmatch.WcMatch(self.tempdir, '*.txt', None, True, True, self.default_flags) records = 0 for f in walker.imatch(): records += 1 walker.kill() self.assertEqual(records, 1) # Reset our test tracker along with the walker object self.errors = [] self.skipped = 0 self.files = [] records = 0 walker.reset() self.crawl_files(walker) self.assertEqual(sorted(self.files), self.norm_list(['.hidden/a.txt', 'a.txt']))
def test_limit_wcmatch(self): """Test expansion limit of `globmatch`.""" with self.assertRaises(_wcparse.PatternLimitException): wcmatch.WcMatch('.', '{1..11}', flags=wcmatch.BRACE, limit=10)
def _get_script_paths_from_folders_node(self) -> typing.Generator: """Returns script paths from the Folders element array""" for folder_node in filter(is_folder_node, self.folders_node): self.try_fix_namespace_path(folder_node) attr_no_recurse: bool = folder_node.get( XmlAttributeName.NO_RECURSE) == 'True' folder_path: str = folder_node.text # handle . and .. in path if folder_path == os.pardir or startswith(folder_path, os.pardir): folder_path = folder_path.replace( os.pardir, os.path.normpath(os.path.join(self.project_path, os.pardir)), 1) yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue if folder_path == os.curdir or startswith(folder_path, os.curdir): folder_path = folder_path.replace(os.curdir, self.project_path, 1) yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue if startswith(folder_path, self.remote_schemas, ignorecase=True): local_path = self._get_remote_path(folder_node) PapyrusProject.log.info( f'Adding import path from remote: "{local_path}"...') self.import_paths.insert(0, local_path) PapyrusProject.log.info( f'Adding folder path from remote: "{local_path}"...') yield from PathHelper.find_script_paths_from_folder( local_path, no_recurse=attr_no_recurse) continue folder_path = os.path.normpath(folder_path) # try to add absolute path if os.path.isabs(folder_path) and os.path.isdir(folder_path): yield from PathHelper.find_script_paths_from_folder( folder_path, no_recurse=attr_no_recurse) continue # try to add project-relative folder path test_path = os.path.join(self.project_path, folder_path) if os.path.isdir(test_path): # count scripts to avoid issue where an errant `test_path` may exist and contain no sources # this can be a problem if that folder contains sources but user error is hard to fix test_passed = False user_flags = wcmatch.RECURSIVE if not attr_no_recurse else 0x0 matcher = wcmatch.WcMatch(test_path, '*.psc', flags=wcmatch.IGNORECASE | user_flags) for _ in matcher.imatch(): test_passed = True break if test_passed: yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse=attr_no_recurse, matcher=matcher) continue # try to add import-relative folder path for import_path in self.import_paths: test_path = os.path.join(import_path, folder_path) if os.path.isdir(test_path): yield from PathHelper.find_script_paths_from_folder( test_path, no_recurse=attr_no_recurse)