def icollect(file_paths,
ignored_globs=None,
match_cache={},
match_function=fnmatch):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:param match_cache: Dictionary to use for caching results
:param match_function: The function to use for glob matching
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
if ignored_globs is None:
ignored_globs = []
for index, glob in enumerate(ignored_globs):
dirname, basename = os.path.split(glob)
if not has_wildcard(dirname) and basename == '**':
logging.warning('Detected trailing globstar in ignore glob '
f"'{glob}'. "
"Please remove the unnecessary '**' from its end.")
ignored_globs[index] = glob.rstrip('*')
for file_path in file_paths:
if file_path not in match_cache:
match_cache[file_path] = list(iglob(file_path))
for match in match_cache[file_path]:
if not ignored_globs or not match_function(match, ignored_globs):
yield match, file_path
def icollect_bears(bear_dirs, bear_names, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directory name or list of such that can contain bears
:param bear_names: names of bears
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields bear classes
"""
for bear_dir in filter(os.path.isdir, icollect(bear_dirs)):
for bear_name in bear_names:
for matching_file in iglob(
os.path.join(bear_dir, bear_name + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def icollect_bears(bear_dirs, bear_globs, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directory name or list of such that can contain bears
:param bear_globs: globs of bears to collect
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]), icollect(bear_dirs)):
for bear_glob in bear_globs:
for matching_file in iglob(os.path.join(bear_dir, bear_glob + ".py")):
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING,
)
def icollect_bears(bear_dirs, bear_globs, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directory name or list of such that can contain bears
:param bear_globs: globs of bears to collect
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dirs)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
for matching_file in iglob(
os.path.join(bear_dir, bear_glob + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def icollect(file_paths, ignored_globs=None, match_cache={},
match_function=fnmatch):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:param match_cache: Dictionary to use for caching results
:param match_function: The function to use for glob matching
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
if ignored_globs is None:
ignored_globs = []
for index, glob in enumerate(ignored_globs):
dirname, basename = os.path.split(glob)
if not has_wildcard(dirname) and basename == '**':
logging.warning("Detected trailing globstar in ignore glob '{}'. "
"Please remove the unnecessary '**' from its end."
.format(glob))
ignored_globs[index] = glob.rstrip('*')
for file_path in file_paths:
if file_path not in match_cache:
match_cache[file_path] = list(iglob(file_path))
for match in match_cache[file_path]:
if not ignored_globs or not match_function(match, ignored_globs):
yield match, file_path
def icollect_bears(bear_dirs, bear_names, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directory name or list of such that can contain bears
:param bear_names: names of bears
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields bear classes
"""
for bear_dir in filter(os.path.isdir, icollect(bear_dirs)):
for bear_name in bear_names:
for matching_file in iglob(
os.path.join(bear_dir, bear_name + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def list_glob_results(values=None):
"""
Expands the globs of all given values and concatenates the results.
:param values: List of file-globs or files.
:return: List of matched files.
"""
return functools.reduce(lambda seed, value: seed + list(iglob(value)),
values if values else (), [])
def icollect(file_paths):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: list of file paths that can include globs
:return: iterator that yields paths of all matching files
"""
for file_path in file_paths:
for match in iglob(file_path):
yield match
def list_glob_results(values=None):
"""
Expands the globs of all given values and concatenates the results.
:param values: List of file-globs or files.
:return: List of matched files.
"""
return functools.reduce(
lambda seed, value: seed + list(iglob(value)),
values if values else (),
[])
def icollect(file_paths):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: file path or list of such that can include globs
:return: iterator that yields paths of all matching files
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
for match in iglob(file_path):
yield match
def icollect(file_paths):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: file path or list of such that can include globs
:return: iterator that yields paths of all matching files
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
for match in iglob(file_path):
yield match
def icollect(file_paths, ignored_globs=None):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
for match in iglob(file_path):
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def icollect(file_paths, ignored_globs=None):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: file path or list of such that can include globs
:param ignored_globs: list of globs to ignore when matching files
:return: iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
for match in iglob(file_path):
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dir_glob: Directory globs or list of such that can contain bears
:param bear_globs: Globs of bears to collect
:param kinds: List of bear kinds to be collected
:param log_printer: Log_printer to handle logging
:return: Iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dir_glob)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py'))
matching_files = sorted(matching_files)
for matching_file in matching_files:
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_exception(
(f'Unable to collect bears from {matching_file} '
'because there '
'is a conflict with the version of a dependency '
'you have installed. This may be resolved by '
'creating a separate virtual environment for coala '
f'or running `pip3 install \"{exception.req}\"`. '
'Be aware that '
'the latter solution might break other python '
'packages that depend on the currently installed '
'version.'),
exception,
log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_exception(
f'Unable to collect bears from {matching_file}. '
'Probably the '
'file is malformed or the module code raises an '
'exception.',
exception,
log_level=LOG_LEVEL.WARNING)
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer=None):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dir_glob: Directory globs or list of such that can contain bears
:param bear_globs: Globs of bears to collect
:param kinds: List of bear kinds to be collected
:param log_printer: Log_printer to handle logging
:return: Iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dir_glob)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
matching_files = iglob(os.path.join(bear_dir, bear_glob + '.py'))
matching_files = sorted(matching_files)
for matching_file in matching_files:
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_exception(
('Unable to collect bears from {file} because there '
'is a conflict with the version of a dependency '
'you have installed. This may be resolved by '
'creating a separate virtual environment for coala '
'or running `pip3 install \"{pkg}\"`. Be aware that '
'the latter solution might break other python '
'packages that depend on the currently installed '
'version.').format(file=matching_file,
pkg=exception.req),
exception, log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_exception(
'Unable to collect bears from {file}. Probably the '
'file is malformed or the module code raises an '
'exception.'.format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def icollect_bears(bear_dir_glob, bear_globs, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dir_glob: Directory globs or list of such that can contain bears
:param bear_globs: Globs of bears to collect
:param kinds: List of bear kinds to be collected
:param log_printer: Log_printer to handle logging
:return: Iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dir_glob)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
for matching_file in iglob(
os.path.join(bear_dir, bear_glob + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_printer.log_exception(
("Unable to collect bears from {file} because there "
"is a conflict with the version of a dependency "
"you have installed. This may be resolved by "
"creating a separate virtual environment for coala "
"or running `pip install {pkg}`. Be aware that the "
"latter solution might break other python packages "
"that depend on the currently installed "
"version.").format(file=matching_file,
pkg=exception.req),
exception,
log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def icollect_bears(bear_dirs, bear_globs, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directory name or list of such that can contain bears
:param bear_globs: globs of bears to collect
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields a tuple with bear class and
which bear_glob was used to find that bear class.
"""
for bear_dir, dir_glob in filter(lambda x: os.path.isdir(x[0]),
icollect(bear_dirs)):
# Since we get a real directory here and since we
# pass this later to iglob, we need to escape this.
bear_dir = glob_escape(bear_dir)
for bear_glob in bear_globs:
for matching_file in iglob(
os.path.join(bear_dir, bear_glob + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear, bear_glob
except pkg_resources.VersionConflict as exception:
log_printer.log_exception(
("Unable to collect bears from {file} because there "
"is a conflict with the version of a dependency "
"you have installed. This may be resolved by "
"creating a separate virtual environment for coala "
"or running `pip install {pkg}`. Be aware that the "
"latter solution might break other python packages "
"that depend on the currently installed "
"version.").format(file=matching_file,
pkg=exception.req),
exception, log_level=LOG_LEVEL.WARNING)
except BaseException as exception:
log_printer.log_exception(
"Unable to collect bears from {file}. Probably the "
"file is malformed or the module code raises an "
"exception.".format(file=matching_file),
exception,
log_level=LOG_LEVEL.WARNING)
def icollect(file_paths, ignored_globs=None, match_cache={}):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:param match_cache: Dictionary to use for caching results
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
if file_path not in match_cache:
match_cache[file_path] = list(iglob(file_path))
for match in match_cache[file_path]:
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def icollect(file_paths, ignored_globs=None, match_cache={}):
"""
Evaluate globs in file paths and return all matching files.
:param file_paths: File path or list of such that can include globs
:param ignored_globs: List of globs to ignore when matching files
:param match_cache: Dictionary to use for caching results
:return: Iterator that yields tuple of path of a matching
file, the glob where it was found
"""
if isinstance(file_paths, str):
file_paths = [file_paths]
for file_path in file_paths:
if file_path not in match_cache:
match_cache[file_path] = list(iglob(file_path))
for match in match_cache[file_path]:
if not ignored_globs or not fnmatch(match, ignored_globs):
yield match, file_path
def icollect_bears(bear_dirs, bear_names, kinds, log_printer):
"""
Collect all bears from bear directories that have a matching kind.
:param bear_dirs: directories that can contain bears
:param bear_names: names of bears
:param kinds: list of bear kinds to be collected
:param log_printer: log_printer to handle logging
:return: iterator that yields bear classes
"""
for bear_dir in filter(os.path.isdir, icollect(bear_dirs)):
for bear_name in bear_names:
for matching_file in iglob(
os.path.join(bear_dir, bear_name + '.py')):
try:
for bear in _import_bears(matching_file, kinds):
yield bear
except:
log_printer.warn(_("Unable to collect bears from {file}. "
"Probably the file is malformed or "
"the module code raises an exception.")
.format(file=matching_file))