def test_glob_integrity(self): """`globmatch` must match what glob globs.""" # Number of slashes is inconsequential # Glob really looks at what is in between. Multiple slashes are the same as one separator. # UNC mounts are special cases and it matters there. self.assertTrue( all( [ glob.globmatch( x, '**/../*.{md,py}', flags=self.flags ) for x in glob.glob('**/../*.{md,py}', flags=self.flags) ] ) ) self.assertTrue( all( [ glob.globmatch( x, './**/./../*.py', flags=self.flags ) for x in glob.glob('./**/./../*.py', flags=self.flags) ] ) ) self.assertTrue( all( [ glob.globmatch( x, './///**///./../*.py', flags=self.flags ) for x in glob.glob('./**/.//////..////*.py', flags=self.flags) ] ) )
def test_glob_parsing_nix(self, mock__iscase_sensitive): """Test wildcard parsing.""" mock__iscase_sensitive.return_value = True _wcparse._compile.cache_clear() self.assertTrue( glob.globmatch('some/name/with/named/file/test.py', '**/named/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na[/]med/file/test.py', '**/na[/]med/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na[/]med\\/file/test.py', '**/na[/]med\\/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na\\med/file/test.py', r'**/na[\\]med/file/*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some/name/with/na[\\/]med\\/file/test.py', r'**/na[\/]med\/file/*.py', flags=self.flags | glob.R))
def check_connection( self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """ This method checks two things: - That the credentials provided in config are valid for access. - That the path pattern(s) provided in config are valid to be matched against. :param logger: an instance of AirbyteLogger to use :param config: The user-provided configuration as specified by the source's spec. This usually contains information required to check connection e.g. tokens, secrets and keys etc. :return: A tuple of (boolean, error). If boolean is true, then the connection check is successful and we can connect to the underlying data source using the provided configuration. Otherwise, the input config cannot be used to connect to the underlying data source, and the "error" object should describe what went wrong. The error object will be cast to string to display the problem to the user. """ try: for file_info in self.stream_class(**config).filepath_iterator(): # TODO: will need to split config.get("path_pattern") up by stream once supporting multiple streams # test that matching on the pattern doesn't error globmatch(file_info.key, config.get("path_pattern"), flags=GLOBSTAR | SPLIT) # just need first file here to test connection and valid patterns return True, None except Exception as e: logger.error(format_exc()) return False, e logger.warn("Found 0 files (but connection is valid).") return True, None
def test_unfinished_ext(self): """Test unfinished ext.""" flags = self.flags flags ^= glob.NEGATE for x in ['!', '?', '+', '*', '@']: self.assertTrue(glob.globmatch(x + '(a|B', x + '(a|B', flags=flags)) self.assertFalse(glob.globmatch(x + '(a|B', 'B', flags=flags))
def __init__(self, net_provider: Union[Type[PytorchNN], Callable[[], PytorchNN]], criterion_provider: Union[Type[nn.modules.loss._Loss], Callable[[], nn.modules.loss._Loss]], optimizer_provider: Union[Type[t.optim.Optimizer], Callable[[Iterable], t.optim.Optimizer]], record_best_weights: bool = False, cuda: bool = False, **kwargs): self.net_provider = net_provider self.criterion_provider = criterion_provider self.optimizer_provider = optimizer_provider self._cuda = cuda self.net = to_device(net_provider(**kwargs), cuda) self.criterion = to_device( call_callable_dynamic_args(criterion_provider, module=self.net, params=self.net.named_parameters()), cuda) self.optimizer = optimizer_provider(self.net.parameters()) self.log_once = LogOnce().log self.best_weights = None self.best_loss = float('inf') self.last_loss = float('inf') self.record_best_weights = record_best_weights # initialization self.optimizer.zero_grad() if hasattr(self.net, "init_weights"): self.net.apply(self.net.init_weights) # penalty self.l1_penalty_tensors = {t.zeros(1): 1.} self.l2_penalty_tensors = {t.zeros(1): 1.} def param_dict(module): return { t[0].replace('.', '/'): t[1] for t in module.named_parameters() } if hasattr(self.net, "L1") and len(self.net.L1()) > 0: self.l1_penalty_tensors = \ {tensor: penalty for param, tensor in param_dict(self.net).items() for path, penalty in self.net.L1().items() if glob.globmatch(param, path, flags=glob.GLOBSTAR)} if hasattr(self.net, "L2") and len(self.net.L2()) > 0: self.l2_penalty_tensors = \ {tensor: penalty for param, tensor in param_dict(self.net).items() for path, penalty in self.net.L2().items() if glob.globmatch(param, path, flags=glob.GLOBSTAR)}
def test_windows_drives(self): """Test windows drives.""" if util.is_case_sensitive(): return self.assertTrue( glob.globmatch('//?/c:/somepath/to/match/file.txt', '//?/c:/**/*.txt', flags=self.flags)) self.assertTrue( glob.globmatch('c:/somepath/to/match/file.txt', 'c:/**/*.txt', flags=self.flags))
def apply(self): """Sync labels.""" add_labels = {} for file in self._get_changed_files(): for label in self.labels: names = label['labels'] lows = [n.lower() for n in names] match = False for pattern in label['patterns']: if glob.globmatch(file, pattern, flags=self.flags): match = True break if match: for index, low in enumerate(lows): if low not in add_labels: add_labels[low] = names[index] break remove_labels = {} for label in self.labels: names = label['labels'] lows = [n.lower() for n in names] for index, low in enumerate(lows): if low not in add_labels and low not in remove_labels: remove_labels[low] = names[index] self._update_issue_labels(add_labels, remove_labels)
def file_found_processor(file: str): specs = config.data.get('permissions.specs') if specs: for spec, mode in specs.items(): if globmatch(file, spec, flags=GLOBSTAR): return (), {"file": file, "mode": mode} return None
def detect_syntax(self, view): """Detect the syntax.""" self.plugins = {} if view.is_scratch( ) or not view.file_name(): # buffer has never been saved return self.reset_cache_variables(view) self.load_syntaxes() if not self.syntaxes: return found = False for syntax in self.syntaxes: # stop on the first syntax that matches if self.syntax_matches(syntax): found = True self.set_syntax(syntax.get("syntax")) break # If pattern matches extension trim pattern, try again after trimming the extension if not found: for ext_trim in self.ext_trim: try: match = False pattern = ext_trim.get('file_path') if pattern: match = re.match(pattern, self.norm_file_name) is not None if not match: pattern = ext_trim.get('globmatch') case = glob.C if ext_trim.get('case', False) else glob.I match = glob.globmatch(self.file_name, pattern, flags=GLOB_FLAGS | case) except Exception: if self.reraise_exceptions: raise else: match = False if match: parts = os.path.splitext(self.orig_file_name) # If there is no extension, then there is nothing to do if parts[1]: self.file_name = parts[0] self.norm_file_name = self.file_name.replace( '\\', '/') if self.is_win else self.file_name for syntax in self.syntaxes: if self.syntax_matches(syntax): self.set_syntax( syntax.get("syntax", syntax.get("name"))) break break self.plugins = {}
def get_zip_content(self, filename): """Get zip content.""" with zipfile.ZipFile(filename, 'r') as z: self.determine_file_type(z) for item in z.infolist(): if glob.globmatch(item.filename, glob.globsplit(self.filepattern, flags=self.FLAGS), flags=self.FLAGS): yield z.read(item.filename), item.filename
def matches(self, path: Optional[str]) -> bool: if self.type == RestType.GLOB: return path is not None and glob.globmatch( path, self.pattern, flags=glob.GLOBSTAR) elif self.type == RestType.REGEX: return path is not None and re.search( self.pattern, PurePath(path).as_posix()) is not None else: return True
def glob_matches(self, rule): """Perform glob matches.""" result = False pattern = rule.get('globmatch') if pattern: case = glob.C if rule.get('case', False) else glob.I result = glob.globmatch(self.file_name, pattern, flags=GLOB_FLAGS | case)
def pattern_matched_filepath_iterator(self, file_infos: Iterable[FileInfo]) -> Iterator[FileInfo]: """ iterates through iterable file_infos and yields only those file_infos that match user-provided path patterns :param file_infos: filepath_iterator(), this is a param rather than method reference in order to unit test this :yield: FileInfo object to use in StorageFile(), if matching on user-provided path patterns """ for file_info in file_infos: if globmatch(file_info.key, self._path_pattern, flags=GLOBSTAR | SPLIT): yield file_info
def _get_src_and_dst(path, repo_path, sources, dst_root, used_sources): is_wildcard = False matched_pattern = None if not sources: source = Path(".") else: source = None for s in sources.keys(): try: Path(path).relative_to(s) except ValueError: if glob.globmatch(path, str(s), flags=glob.GLOBSTAR): is_wildcard = True source = Path(path) used_sources.add(s) matched_pattern = str(s) break else: source = Path(s) used_sources.add(source) break if not source: return if is_wildcard: # Search to see if a parent of the path matches the pattern and return it while glob.globmatch(str(source.parent), matched_pattern, flags=glob.GLOBSTAR) and source != source.parent: source = source.parent src = repo_path / path source_name = source.name relative_path = Path(path).relative_to(source) if src.is_dir() and is_wildcard: sources[source] = None used_sources.add(source) dst = dst_root / source_name / relative_path return path, src, dst
def test_glob_parsing_win(self, mock__iscase_sensitive, mock_platform): """Test windows style glob parsing.""" mock_platform.return_value = "windows" mock__iscase_sensitive.return_value = False _wcparse._compile.cache_clear() self.assertTrue( glob.globmatch('some/name/with/named/file/test.py', '**/named/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na[/]med/file/test.py', '**/na[/]med/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na[/]med\\/file/test.py', '**/na[/]med\\/file/*.py', flags=self.flags)) self.assertTrue( glob.globmatch('some/name/with/na[\\]med/file/test.py', r'**/na[\\]med/file/*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some\\name\\with\\na[\\]med\\file\\test.py', r'**/na[\\]med/file/*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some\\name\\with\\na[\\]med\\file*.py', r'**\\na[\\]med\\file\*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some\\name\\with\\na[\\]med\\file\\test.py', r'**\\na[\\]m\ed\\file\\*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some\\name\\with\\na[\\]med\\\\file\\test.py', r'**\\na[\\]m\ed\\/file\\*.py', flags=self.flags | glob.R)) self.assertTrue( glob.globmatch('some\\name\\with\\na[\\\\]med\\\\file\\test.py', r'**\\na[\/]m\ed\/file\\*.py', flags=self.flags | glob.R))
def _get_src_and_dst(self, path, repo_path, sources, dst_root, used_sources): is_wildcard = False if not sources: source = Path('.') else: source = None for s in sources.keys(): try: Path(path).relative_to(s) except ValueError: if glob.globmatch(path, str(s), flags=glob.GLOBSTAR): is_wildcard = True source = path used_sources.add(s) break else: source = s used_sources.add(source) break if not source: return src = repo_path / path source_name = Path(source).name relative_path = Path(path).relative_to(source) if src.is_dir() and is_wildcard: sources[source] = None used_sources.add(source) if not dst_root.exists(): # Destination will be a file or directory if len(sources) == 1 and not is_wildcard: dst = dst_root / relative_path else: # Treat destination as a directory dst = dst_root / source_name / relative_path elif dst_root.is_dir(): dst = dst_root / source_name / relative_path else: # Destination is an existing file if src.is_dir(): raise errors.ParameterError( 'Cannot copy multiple files or directories to a file') # Later we need to check if we are copying multiple files dst = dst_root return (path, src, dst)
def glob_match(path: Union[str, Path], patterns: Union[str, Sequence[str]]) -> bool: """ Test if *path* matches any of the glob *patterns*. Besides basic glob features like single-part wildcards (``*``), character classes (``[…]``), and brace expansion (``{…, …}``), several advanced globbing features are also supported: multi-part wildcards (``**``), extended globbing (``@(…)``, ``+(…)``, etc.), basename matching for patterns containing only a single path part, and negation (``!…``). Implemented with with :func:`wcmatch.glob.globmatch`. """ return globmatch(path, patterns, flags=GLOBSTAR | BRACE | EXTGLOB | MATCHBASE | NEGATE)
def __call__(self, view: sublime.View) -> bool: """Does this filter match the view? An empty filter matches any view.""" if self.language: syntax = view.syntax() if not syntax or basescope2languageid( syntax.scope) != self.language: return False if self.scheme: uri = view.settings().get("lsp_uri") if isinstance( uri, str) and urllib.parse.urlparse(uri).scheme != self.scheme: return False if self.pattern: if not globmatch(view.file_name() or "", self.pattern, flags=GLOBSTAR | BRACE): return False return True
def get_labels(rules, files, flags): """Sync labels.""" add_labels = {} for file in files: for label in rules: try: names = label['labels'] lows = [n.lower() for n in names] except Exception: traceback.print_exc(file=sys.stdout) continue match = False for pattern in label['patterns']: try: match = glob.globmatch(file, pattern, flags=flags) except Exception: traceback.print_exc(file=sys.stdout) match = False if match: break if match: for index, low in enumerate(lows): if low not in add_labels: add_labels[low] = names[index] remove_labels = {} for label in rules: try: names = label['labels'] lows = [n.lower() for n in names] except Exception: traceback.print_exc(file=sys.stdout) continue for index, low in enumerate(lows): if low not in add_labels and low not in remove_labels: remove_labels[low] = names[index] return add_labels, remove_labels
def _get_src_and_dst(self, path, repo_path, sources, dst_root, used_sources): is_wildcard = False if not sources: source = Path('.') else: source = None for s in sources.keys(): try: Path(path).relative_to(s) except ValueError: if glob.globmatch(path, str(s), flags=glob.GLOBSTAR): is_wildcard = True source = path used_sources.add(s) break else: source = s used_sources.add(source) break if not source: return src = repo_path / path source_name = Path(source).name relative_path = Path(path).relative_to(source) if src.is_dir() and is_wildcard: sources[source] = None used_sources.add(source) dst = dst_root / source_name / relative_path return (path, src, dst)
def sane_globmatch(path, matchers): if len(matchers) == 0: return False return globmatch(path, matchers, flags=GLOBSTAR)
class TestGlobMatch: """ Tests that are performed against globmatch. Each case entry is a list of 4 parameters. * Pattern * Filename * Expected result (boolean of whether pattern matched filename) * Flags The default flags are NEGATE | GLOBSTAR | EXTGLOB | BRACE. Any flags passed through via entry are XORed. So if any of the default flags are passed via an entry, they will be disabled. All other flags will enable the feature. """ cases = [ ['*.!(js|css)', 'bar.min.js', True, glob.N], ['!*.+(js|css)', 'bar.min.js', False, glob.N], ['*.+(js|css)', 'bar.min.js', True, glob.N], ['*.!(j)', 'a-integration-test.js', True, glob.N], ['!(*-integration-test.js)', 'a-integration-test.js', False, glob.N], ['*-!(integration-)test.js', 'a-integration-test.js', True, glob.N], ['*-!(integration)-test.js', 'a-integration-test.js', False, glob.N], ['*!(-integration)-test.js', 'a-integration-test.js', True, glob.N], ['*!(-integration-)test.js', 'a-integration-test.js', True, glob.N], ['*!(integration)-test.js', 'a-integration-test.js', True, glob.N], ['*!(integration-test).js', 'a-integration-test.js', True, glob.N], ['*-!(integration-test).js', 'a-integration-test.js', True, glob.N], ['*-!(integration-test.js)', 'a-integration-test.js', True, glob.N], ['*-!(integra)tion-test.js', 'a-integration-test.js', False, glob.N], ['*-integr!(ation)-test.js', 'a-integration-test.js', False, glob.N], ['*-integr!(ation-t)est.js', 'a-integration-test.js', False, glob.N], ['*-i!(ntegration-)test.js', 'a-integration-test.js', False, glob.N], ['*i!(ntegration-)test.js', 'a-integration-test.js', True, glob.N], ['*te!(gration-te)st.js', 'a-integration-test.js', True, glob.N], ['*-!(integration)?test.js', 'a-integration-test.js', False, glob.N], ['*?!(integration)?test.js', 'a-integration-test.js', True, glob.N], ['foo-integration-test.js', 'foo-integration-test.js', True, glob.N], ['!(*-integration-test.js)', 'foo-integration-test.js', False, glob.N], ['*.!(js).js', 'foo.jszzz.js', True, glob.N], ['*.!(js)', 'asd.jss', True, glob.N], ['*.!(js).!(xy)', 'asd.jss.xyz', True, glob.N], ['*.!(js).!(xy)', 'asd.jss.xy', False, glob.N], ['*.!(js).!(xy)', 'asd.js.xyz', False, glob.N], ['*.!(js).!(xy)', 'asd.js.xy', False, glob.N], ['*.!(js).!(xy)', 'asd.sjs.zxy', True, glob.N], ['*.!(js).!(xy)', 'asd..xyz', True, glob.N], ['*.!(js).!(xy)', 'asd..xy', False, glob.N], ['*.!(js|x).!(xy)', 'asd..xy', False, glob.N], ['*.!(js)', 'foo.js.js', True, glob.N], ['*(*.json|!(*.js))', 'testjson.json', True, glob.N], ['+(*.json|!(*.js))', 'testjson.json', True, glob.N], ['@(*.json|!(*.js))', 'testjson.json', True, glob.N], ['?(*.json|!(*.js))', 'testjson.json', True, glob.N], ['*(*.json|!(*.js))', 'foojs.js', False, glob.N], # XXX bash 4.3 disagrees! ['+(*.json|!(*.js))', 'foojs.js', False, glob.N], # XXX bash 4.3 disagrees! ['@(*.json|!(*.js))', 'foojs.js', False, glob.N], ['?(*.json|!(*.js))', 'foojs.js', False, glob.N], ['*(*.json|!(*.js))', 'other.bar', True, glob.N], ['+(*.json|!(*.js))', 'other.bar', True, glob.N], ['@(*.json|!(*.js))', 'other.bar', True, glob.N], ['?(*.json|!(*.js))', 'other.bar', True, glob.N] ] @classmethod def setup_class(cls): """Setup default flag options.""" # The tests we scraped were written with this assumed. cls.flags = glob.NEGATE | glob.GLOBSTAR | glob.EXTGLOB | glob.BRACE @classmethod def evaluate(cls, case): """Evaluate case.""" pattern = case[0] filename = case[1] goal = case[2] flags = cls.flags if len(case) > 3: flags ^= case[3] print("PATTERN: ", pattern) print("FILE: ", filename) print("GOAL: ", goal) print("FLAGS: ", bin(flags)) assert glob.globmatch( filename, pattern, flags=flags) == goal, "Expression did not evaluate as %s" % goal