def test_glob_integrity(self):
"""`globmatch` must match what glob globs."""
# Number of slashes is inconsequential
# Glob really looks at what is in between. Multiple slashes are the same as one separator.
# UNC mounts are special cases and it matters there.
self.assertTrue(
all(
[
glob.globmatch(
x, '**/../*.{md,py}', flags=self.flags
) for x in glob.glob('**/../*.{md,py}', flags=self.flags)
]
)
)
self.assertTrue(
all(
[
glob.globmatch(
x, './**/./../*.py', flags=self.flags
) for x in glob.glob('./**/./../*.py', flags=self.flags)
]
)
)
self.assertTrue(
all(
[
glob.globmatch(
x, './///**///./../*.py', flags=self.flags
) for x in glob.glob('./**/.//////..////*.py', flags=self.flags)
]
)
)
def test_glob_parsing_nix(self, mock__iscase_sensitive):
"""Test wildcard parsing."""
mock__iscase_sensitive.return_value = True
_wcparse._compile.cache_clear()
self.assertTrue(
glob.globmatch('some/name/with/named/file/test.py',
'**/named/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na[/]med/file/test.py',
'**/na[/]med/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na[/]med\\/file/test.py',
'**/na[/]med\\/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na\\med/file/test.py',
r'**/na[\\]med/file/*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some/name/with/na[\\/]med\\/file/test.py',
r'**/na[\/]med\/file/*.py',
flags=self.flags | glob.R))
def check_connection(
self, logger: AirbyteLogger,
config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
"""
This method checks two things:
- That the credentials provided in config are valid for access.
- That the path pattern(s) provided in config are valid to be matched against.
:param logger: an instance of AirbyteLogger to use
:param config: The user-provided configuration as specified by the source's spec.
This usually contains information required to check connection e.g. tokens, secrets and keys etc.
:return: A tuple of (boolean, error). If boolean is true, then the connection check is successful and we can connect to the underlying data
source using the provided configuration.
Otherwise, the input config cannot be used to connect to the underlying data source, and the "error" object should describe what went wrong.
The error object will be cast to string to display the problem to the user.
"""
try:
for file_info in self.stream_class(**config).filepath_iterator():
# TODO: will need to split config.get("path_pattern") up by stream once supporting multiple streams
# test that matching on the pattern doesn't error
globmatch(file_info.key,
config.get("path_pattern"),
flags=GLOBSTAR | SPLIT)
# just need first file here to test connection and valid patterns
return True, None
except Exception as e:
logger.error(format_exc())
return False, e
logger.warn("Found 0 files (but connection is valid).")
return True, None
def test_unfinished_ext(self):
"""Test unfinished ext."""
flags = self.flags
flags ^= glob.NEGATE
for x in ['!', '?', '+', '*', '@']:
self.assertTrue(glob.globmatch(x + '(a|B', x + '(a|B', flags=flags))
self.assertFalse(glob.globmatch(x + '(a|B', 'B', flags=flags))
def __init__(self,
net_provider: Union[Type[PytorchNN], Callable[[], PytorchNN]],
criterion_provider: Union[Type[nn.modules.loss._Loss],
Callable[[],
nn.modules.loss._Loss]],
optimizer_provider: Union[Type[t.optim.Optimizer],
Callable[[Iterable],
t.optim.Optimizer]],
record_best_weights: bool = False,
cuda: bool = False,
**kwargs):
self.net_provider = net_provider
self.criterion_provider = criterion_provider
self.optimizer_provider = optimizer_provider
self._cuda = cuda
self.net = to_device(net_provider(**kwargs), cuda)
self.criterion = to_device(
call_callable_dynamic_args(criterion_provider,
module=self.net,
params=self.net.named_parameters()),
cuda)
self.optimizer = optimizer_provider(self.net.parameters())
self.log_once = LogOnce().log
self.best_weights = None
self.best_loss = float('inf')
self.last_loss = float('inf')
self.record_best_weights = record_best_weights
# initialization
self.optimizer.zero_grad()
if hasattr(self.net, "init_weights"):
self.net.apply(self.net.init_weights)
# penalty
self.l1_penalty_tensors = {t.zeros(1): 1.}
self.l2_penalty_tensors = {t.zeros(1): 1.}
def param_dict(module):
return {
t[0].replace('.', '/'): t[1]
for t in module.named_parameters()
}
if hasattr(self.net, "L1") and len(self.net.L1()) > 0:
self.l1_penalty_tensors = \
{tensor: penalty for param, tensor in param_dict(self.net).items()
for path, penalty in self.net.L1().items()
if glob.globmatch(param, path, flags=glob.GLOBSTAR)}
if hasattr(self.net, "L2") and len(self.net.L2()) > 0:
self.l2_penalty_tensors = \
{tensor: penalty for param, tensor in param_dict(self.net).items()
for path, penalty in self.net.L2().items()
if glob.globmatch(param, path, flags=glob.GLOBSTAR)}
def test_windows_drives(self):
"""Test windows drives."""
if util.is_case_sensitive():
return
self.assertTrue(
glob.globmatch('//?/c:/somepath/to/match/file.txt',
'//?/c:/**/*.txt',
flags=self.flags))
self.assertTrue(
glob.globmatch('c:/somepath/to/match/file.txt',
'c:/**/*.txt',
flags=self.flags))
def apply(self):
"""Sync labels."""
add_labels = {}
for file in self._get_changed_files():
for label in self.labels:
names = label['labels']
lows = [n.lower() for n in names]
match = False
for pattern in label['patterns']:
if glob.globmatch(file, pattern, flags=self.flags):
match = True
break
if match:
for index, low in enumerate(lows):
if low not in add_labels:
add_labels[low] = names[index]
break
remove_labels = {}
for label in self.labels:
names = label['labels']
lows = [n.lower() for n in names]
for index, low in enumerate(lows):
if low not in add_labels and low not in remove_labels:
remove_labels[low] = names[index]
self._update_issue_labels(add_labels, remove_labels)
def file_found_processor(file: str):
specs = config.data.get('permissions.specs')
if specs:
for spec, mode in specs.items():
if globmatch(file, spec, flags=GLOBSTAR):
return (), {"file": file, "mode": mode}
return None
def detect_syntax(self, view):
"""Detect the syntax."""
self.plugins = {}
if view.is_scratch(
) or not view.file_name(): # buffer has never been saved
return
self.reset_cache_variables(view)
self.load_syntaxes()
if not self.syntaxes:
return
found = False
for syntax in self.syntaxes:
# stop on the first syntax that matches
if self.syntax_matches(syntax):
found = True
self.set_syntax(syntax.get("syntax"))
break
# If pattern matches extension trim pattern, try again after trimming the extension
if not found:
for ext_trim in self.ext_trim:
try:
match = False
pattern = ext_trim.get('file_path')
if pattern:
match = re.match(pattern,
self.norm_file_name) is not None
if not match:
pattern = ext_trim.get('globmatch')
case = glob.C if ext_trim.get('case',
False) else glob.I
match = glob.globmatch(self.file_name,
pattern,
flags=GLOB_FLAGS | case)
except Exception:
if self.reraise_exceptions:
raise
else:
match = False
if match:
parts = os.path.splitext(self.orig_file_name)
# If there is no extension, then there is nothing to do
if parts[1]:
self.file_name = parts[0]
self.norm_file_name = self.file_name.replace(
'\\', '/') if self.is_win else self.file_name
for syntax in self.syntaxes:
if self.syntax_matches(syntax):
self.set_syntax(
syntax.get("syntax", syntax.get("name")))
break
break
self.plugins = {}
def get_zip_content(self, filename):
"""Get zip content."""
with zipfile.ZipFile(filename, 'r') as z:
self.determine_file_type(z)
for item in z.infolist():
if glob.globmatch(item.filename, glob.globsplit(self.filepattern, flags=self.FLAGS), flags=self.FLAGS):
yield z.read(item.filename), item.filename
def matches(self, path: Optional[str]) -> bool:
if self.type == RestType.GLOB:
return path is not None and glob.globmatch(
path, self.pattern, flags=glob.GLOBSTAR)
elif self.type == RestType.REGEX:
return path is not None and re.search(
self.pattern,
PurePath(path).as_posix()) is not None
else:
return True
def glob_matches(self, rule):
"""Perform glob matches."""
result = False
pattern = rule.get('globmatch')
if pattern:
case = glob.C if rule.get('case', False) else glob.I
result = glob.globmatch(self.file_name,
pattern,
flags=GLOB_FLAGS | case)
def pattern_matched_filepath_iterator(self, file_infos: Iterable[FileInfo]) -> Iterator[FileInfo]:
"""
iterates through iterable file_infos and yields only those file_infos that match user-provided path patterns
:param file_infos: filepath_iterator(), this is a param rather than method reference in order to unit test this
:yield: FileInfo object to use in StorageFile(), if matching on user-provided path patterns
"""
for file_info in file_infos:
if globmatch(file_info.key, self._path_pattern, flags=GLOBSTAR | SPLIT):
yield file_info
def _get_src_and_dst(path, repo_path, sources, dst_root, used_sources):
is_wildcard = False
matched_pattern = None
if not sources:
source = Path(".")
else:
source = None
for s in sources.keys():
try:
Path(path).relative_to(s)
except ValueError:
if glob.globmatch(path, str(s), flags=glob.GLOBSTAR):
is_wildcard = True
source = Path(path)
used_sources.add(s)
matched_pattern = str(s)
break
else:
source = Path(s)
used_sources.add(source)
break
if not source:
return
if is_wildcard:
# Search to see if a parent of the path matches the pattern and return it
while glob.globmatch(str(source.parent), matched_pattern, flags=glob.GLOBSTAR) and source != source.parent:
source = source.parent
src = repo_path / path
source_name = source.name
relative_path = Path(path).relative_to(source)
if src.is_dir() and is_wildcard:
sources[source] = None
used_sources.add(source)
dst = dst_root / source_name / relative_path
return path, src, dst
def test_glob_parsing_win(self, mock__iscase_sensitive, mock_platform):
"""Test windows style glob parsing."""
mock_platform.return_value = "windows"
mock__iscase_sensitive.return_value = False
_wcparse._compile.cache_clear()
self.assertTrue(
glob.globmatch('some/name/with/named/file/test.py',
'**/named/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na[/]med/file/test.py',
'**/na[/]med/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na[/]med\\/file/test.py',
'**/na[/]med\\/file/*.py',
flags=self.flags))
self.assertTrue(
glob.globmatch('some/name/with/na[\\]med/file/test.py',
r'**/na[\\]med/file/*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some\\name\\with\\na[\\]med\\file\\test.py',
r'**/na[\\]med/file/*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some\\name\\with\\na[\\]med\\file*.py',
r'**\\na[\\]med\\file\*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some\\name\\with\\na[\\]med\\file\\test.py',
r'**\\na[\\]m\ed\\file\\*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some\\name\\with\\na[\\]med\\\\file\\test.py',
r'**\\na[\\]m\ed\\/file\\*.py',
flags=self.flags | glob.R))
self.assertTrue(
glob.globmatch('some\\name\\with\\na[\\\\]med\\\\file\\test.py',
r'**\\na[\/]m\ed\/file\\*.py',
flags=self.flags | glob.R))
def _get_src_and_dst(self, path, repo_path, sources, dst_root,
used_sources):
is_wildcard = False
if not sources:
source = Path('.')
else:
source = None
for s in sources.keys():
try:
Path(path).relative_to(s)
except ValueError:
if glob.globmatch(path, str(s), flags=glob.GLOBSTAR):
is_wildcard = True
source = path
used_sources.add(s)
break
else:
source = s
used_sources.add(source)
break
if not source:
return
src = repo_path / path
source_name = Path(source).name
relative_path = Path(path).relative_to(source)
if src.is_dir() and is_wildcard:
sources[source] = None
used_sources.add(source)
if not dst_root.exists(): # Destination will be a file or directory
if len(sources) == 1 and not is_wildcard:
dst = dst_root / relative_path
else: # Treat destination as a directory
dst = dst_root / source_name / relative_path
elif dst_root.is_dir():
dst = dst_root / source_name / relative_path
else: # Destination is an existing file
if src.is_dir():
raise errors.ParameterError(
'Cannot copy multiple files or directories to a file')
# Later we need to check if we are copying multiple files
dst = dst_root
return (path, src, dst)
def glob_match(path: Union[str, Path], patterns: Union[str,
Sequence[str]]) -> bool:
"""
Test if *path* matches any of the glob *patterns*.
Besides basic glob features like single-part wildcards (``*``), character
classes (``[…]``), and brace expansion (``{…, …}``), several advanced
globbing features are also supported: multi-part wildcards (``**``),
extended globbing (``@(…)``, ``+(…)``, etc.), basename matching for
patterns containing only a single path part, and negation (``!…``).
Implemented with with :func:`wcmatch.glob.globmatch`.
"""
return globmatch(path,
patterns,
flags=GLOBSTAR | BRACE | EXTGLOB | MATCHBASE | NEGATE)
def __call__(self, view: sublime.View) -> bool:
"""Does this filter match the view? An empty filter matches any view."""
if self.language:
syntax = view.syntax()
if not syntax or basescope2languageid(
syntax.scope) != self.language:
return False
if self.scheme:
uri = view.settings().get("lsp_uri")
if isinstance(
uri,
str) and urllib.parse.urlparse(uri).scheme != self.scheme:
return False
if self.pattern:
if not globmatch(view.file_name() or "",
self.pattern,
flags=GLOBSTAR | BRACE):
return False
return True
def get_labels(rules, files, flags):
"""Sync labels."""
add_labels = {}
for file in files:
for label in rules:
try:
names = label['labels']
lows = [n.lower() for n in names]
except Exception:
traceback.print_exc(file=sys.stdout)
continue
match = False
for pattern in label['patterns']:
try:
match = glob.globmatch(file, pattern, flags=flags)
except Exception:
traceback.print_exc(file=sys.stdout)
match = False
if match:
break
if match:
for index, low in enumerate(lows):
if low not in add_labels:
add_labels[low] = names[index]
remove_labels = {}
for label in rules:
try:
names = label['labels']
lows = [n.lower() for n in names]
except Exception:
traceback.print_exc(file=sys.stdout)
continue
for index, low in enumerate(lows):
if low not in add_labels and low not in remove_labels:
remove_labels[low] = names[index]
return add_labels, remove_labels
def _get_src_and_dst(self, path, repo_path, sources, dst_root,
used_sources):
is_wildcard = False
if not sources:
source = Path('.')
else:
source = None
for s in sources.keys():
try:
Path(path).relative_to(s)
except ValueError:
if glob.globmatch(path, str(s), flags=glob.GLOBSTAR):
is_wildcard = True
source = path
used_sources.add(s)
break
else:
source = s
used_sources.add(source)
break
if not source:
return
src = repo_path / path
source_name = Path(source).name
relative_path = Path(path).relative_to(source)
if src.is_dir() and is_wildcard:
sources[source] = None
used_sources.add(source)
dst = dst_root / source_name / relative_path
return (path, src, dst)
def sane_globmatch(path, matchers):
if len(matchers) == 0:
return False
return globmatch(path, matchers, flags=GLOBSTAR)
class TestGlobMatch:
"""
Tests that are performed against globmatch.
Each case entry is a list of 4 parameters.
* Pattern
* Filename
* Expected result (boolean of whether pattern matched filename)
* Flags
The default flags are NEGATE | GLOBSTAR | EXTGLOB | BRACE. Any flags passed through via entry are XORed.
So if any of the default flags are passed via an entry, they will be disabled. All other flags will
enable the feature.
"""
cases = [
['*.!(js|css)', 'bar.min.js', True, glob.N],
['!*.+(js|css)', 'bar.min.js', False, glob.N],
['*.+(js|css)', 'bar.min.js', True, glob.N],
['*.!(j)', 'a-integration-test.js', True, glob.N],
['!(*-integration-test.js)', 'a-integration-test.js', False, glob.N],
['*-!(integration-)test.js', 'a-integration-test.js', True, glob.N],
['*-!(integration)-test.js', 'a-integration-test.js', False, glob.N],
['*!(-integration)-test.js', 'a-integration-test.js', True, glob.N],
['*!(-integration-)test.js', 'a-integration-test.js', True, glob.N],
['*!(integration)-test.js', 'a-integration-test.js', True, glob.N],
['*!(integration-test).js', 'a-integration-test.js', True, glob.N],
['*-!(integration-test).js', 'a-integration-test.js', True, glob.N],
['*-!(integration-test.js)', 'a-integration-test.js', True, glob.N],
['*-!(integra)tion-test.js', 'a-integration-test.js', False, glob.N],
['*-integr!(ation)-test.js', 'a-integration-test.js', False, glob.N],
['*-integr!(ation-t)est.js', 'a-integration-test.js', False, glob.N],
['*-i!(ntegration-)test.js', 'a-integration-test.js', False, glob.N],
['*i!(ntegration-)test.js', 'a-integration-test.js', True, glob.N],
['*te!(gration-te)st.js', 'a-integration-test.js', True, glob.N],
['*-!(integration)?test.js', 'a-integration-test.js', False, glob.N],
['*?!(integration)?test.js', 'a-integration-test.js', True, glob.N],
['foo-integration-test.js', 'foo-integration-test.js', True, glob.N],
['!(*-integration-test.js)', 'foo-integration-test.js', False, glob.N],
['*.!(js).js', 'foo.jszzz.js', True, glob.N],
['*.!(js)', 'asd.jss', True, glob.N],
['*.!(js).!(xy)', 'asd.jss.xyz', True, glob.N],
['*.!(js).!(xy)', 'asd.jss.xy', False, glob.N],
['*.!(js).!(xy)', 'asd.js.xyz', False, glob.N],
['*.!(js).!(xy)', 'asd.js.xy', False, glob.N],
['*.!(js).!(xy)', 'asd.sjs.zxy', True, glob.N],
['*.!(js).!(xy)', 'asd..xyz', True, glob.N],
['*.!(js).!(xy)', 'asd..xy', False, glob.N],
['*.!(js|x).!(xy)', 'asd..xy', False, glob.N],
['*.!(js)', 'foo.js.js', True, glob.N],
['*(*.json|!(*.js))', 'testjson.json', True, glob.N],
['+(*.json|!(*.js))', 'testjson.json', True, glob.N],
['@(*.json|!(*.js))', 'testjson.json', True, glob.N],
['?(*.json|!(*.js))', 'testjson.json', True, glob.N],
['*(*.json|!(*.js))', 'foojs.js', False,
glob.N], # XXX bash 4.3 disagrees!
['+(*.json|!(*.js))', 'foojs.js', False,
glob.N], # XXX bash 4.3 disagrees!
['@(*.json|!(*.js))', 'foojs.js', False, glob.N],
['?(*.json|!(*.js))', 'foojs.js', False, glob.N],
['*(*.json|!(*.js))', 'other.bar', True, glob.N],
['+(*.json|!(*.js))', 'other.bar', True, glob.N],
['@(*.json|!(*.js))', 'other.bar', True, glob.N],
['?(*.json|!(*.js))', 'other.bar', True, glob.N]
]
@classmethod
def setup_class(cls):
"""Setup default flag options."""
# The tests we scraped were written with this assumed.
cls.flags = glob.NEGATE | glob.GLOBSTAR | glob.EXTGLOB | glob.BRACE
@classmethod
def evaluate(cls, case):
"""Evaluate case."""
pattern = case[0]
filename = case[1]
goal = case[2]
flags = cls.flags
if len(case) > 3:
flags ^= case[3]
print("PATTERN: ", pattern)
print("FILE: ", filename)
print("GOAL: ", goal)
print("FLAGS: ", bin(flags))
assert glob.globmatch(
filename, pattern,
flags=flags) == goal, "Expression did not evaluate as %s" % goal
