def test_path_to_unicode_and_path_to_bytes_are_idempotent(self):
a = b'foo\xb1bar'
b = u'foo\udcb1bar'
assert a == path_to_bytes(path_to_unicode(a))
assert a == path_to_bytes(path_to_unicode(b))
assert b == path_to_unicode(path_to_bytes(a))
assert b == path_to_unicode(path_to_bytes(b))
def get_file_infos(location):
"""
Return a mapping of file information collected from the file or
directory at `location`.
"""
from commoncode import fileutils
from commoncode import filetype
from commoncode.hash import multi_checksums
from typecode import contenttype
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
infos = OrderedDict()
is_file = filetype.is_file(location)
is_dir = filetype.is_dir(location)
T = contenttype.get_type(location)
infos['type'] = filetype.get_type(location, short=False)
name = fileutils.file_name(location)
if is_file:
base_name, extension = fileutils.splitext(location)
else:
base_name = name
extension = ''
if on_linux:
infos['name'] = path_to_unicode(name)
infos['base_name'] = path_to_unicode(base_name)
infos['extension'] = path_to_unicode(extension)
else:
infos['name'] = name
infos['base_name'] = base_name
infos['extension'] = extension
infos['date'] = is_file and filetype.get_last_modified_date(
location) or None
infos['size'] = T.size
infos.update(multi_checksums(location, (
'sha1',
'md5',
)))
infos['files_count'] = is_dir and filetype.get_file_count(location) or None
infos['mime_type'] = is_file and T.mimetype_file or None
infos['file_type'] = is_file and T.filetype_file or None
infos['programming_language'] = is_file and T.programming_language or None
infos['is_binary'] = bool(is_file and T.is_binary)
infos['is_text'] = bool(is_file and T.is_text)
infos['is_archive'] = bool(is_file and T.is_archive)
infos['is_media'] = bool(is_file and T.is_media)
infos['is_source'] = bool(is_file and T.is_source)
infos['is_script'] = bool(is_file and T.is_script)
return infos
def get_file_infos(location):
"""
Return a mapping of file information collected from the file or
directory at `location`.
"""
from commoncode import fileutils
from commoncode import filetype
from commoncode.hash import multi_checksums
from typecode import contenttype
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
infos = OrderedDict()
is_file = filetype.is_file(location)
is_dir = filetype.is_dir(location)
T = contenttype.get_type(location)
infos['type'] = filetype.get_type(location, short=False)
name = fileutils.file_name(location)
if is_file:
base_name, extension = fileutils.splitext(location)
else:
base_name = name
extension = ''
if on_linux:
infos['name'] = path_to_unicode(name)
infos['base_name'] = path_to_unicode(base_name)
infos['extension'] = path_to_unicode(extension)
else:
infos['name'] = name
infos['base_name'] = base_name
infos['extension'] = extension
infos['date'] = is_file and filetype.get_last_modified_date(location) or None
infos['size'] = T.size
infos.update(multi_checksums(location, ('sha1', 'md5',)))
infos['files_count'] = is_dir and filetype.get_file_count(location) or None
infos['mime_type'] = is_file and T.mimetype_file or None
infos['file_type'] = is_file and T.filetype_file or None
infos['programming_language'] = is_file and T.programming_language or None
infos['is_binary'] = bool(is_file and T.is_binary)
infos['is_text'] = bool(is_file and T.is_text)
infos['is_archive'] = bool(is_file and T.is_archive)
infos['is_media'] = bool(is_file and T.is_media)
infos['is_source'] = bool(is_file and T.is_source)
infos['is_script'] = bool(is_file and T.is_script)
return infos
def build_ignorer(ignores, unignores):
"""
Return a callable suitable for path ignores with OS-specific encoding
preset.
"""
ignores = ignores or {}
unignores = unignores or {}
if on_linux:
ignores = {path_to_bytes(k): v for k, v in ignores.items()}
unignores = {path_to_bytes(k): v for k, v in unignores.items()}
else:
ignores = {path_to_unicode(k): v for k, v in ignores.items()}
unignores = {path_to_unicode(k): v for k, v in unignores.items()}
return partial(ignore.is_ignored, ignores=ignores, unignores=unignores)
def build_ignorer(ignores, unignores):
"""
Return a callable suitable for path ignores with OS-specific encoding
preset.
"""
ignores = ignores or {}
unignores = unignores or {}
if on_linux:
ignores = {path_to_bytes(k): v for k, v in ignores.items()}
unignores = {path_to_bytes(k): v for k, v in unignores.items()}
else:
ignores = {path_to_unicode(k): v for k, v in ignores.items()}
unignores = {path_to_unicode(k): v for k, v in unignores.items()}
return partial(ignore.is_ignored, ignores=ignores, unignores=unignores)
def fixed_width_file_name(path, max_length=25):
"""
Return a fixed width file name of at most `max_length` characters
extracted from the `path` string and usable for fixed width display.
If the file_name is longer than `max_length`, it is truncated in the
middle with using three dots "..." as an ellipsis and the extension
is kept.
For example:
>>> short = fixed_width_file_name('0123456789012345678901234.c')
>>> assert '0123456789...5678901234.c' == short
"""
if not path:
return ''
# get the path as unicode for display!
path = path_to_unicode(path)
filename = fileutils.file_name(path)
if len(filename) <= max_length:
return filename
base_name, extension = fileutils.splitext(filename)
number_of_dots = 3
len_extension = len(extension)
remaining_length = max_length - len_extension - number_of_dots
if remaining_length < (len_extension +
number_of_dots) or remaining_length < 5:
return ''
prefix_and_suffix_length = abs(remaining_length // 2)
prefix = base_name[:prefix_and_suffix_length]
ellipsis = number_of_dots * '.'
suffix = base_name[-prefix_and_suffix_length:]
return '{prefix}{ellipsis}{suffix}{extension}'.format(**locals())
def fixed_width_file_name(path, max_length=25):
"""
Return a fixed width file name of at most `max_length` characters
extracted from the `path` string and usable for fixed width display.
If the file_name is longer than `max_length`, it is truncated in the
middle with using three dots "..." as an ellipsis and the extension
is kept.
For example:
>>> short = fixed_width_file_name('0123456789012345678901234.c')
>>> assert '0123456789...5678901234.c' == short
"""
if not path:
return ''
# get the path as unicode for display!
path = path_to_unicode(path)
filename = fileutils.file_name(path)
if len(filename) <= max_length:
return filename
base_name, extension = fileutils.splitext(filename)
number_of_dots = 3
len_extension = len(extension)
remaining_length = max_length - len_extension - number_of_dots
if remaining_length < (len_extension + number_of_dots) or remaining_length < 5:
return ''
prefix_and_suffix_length = abs(remaining_length // 2)
prefix = base_name[:prefix_and_suffix_length]
ellipsis = number_of_dots * '.'
suffix = base_name[-prefix_and_suffix_length:]
return '{prefix}{ellipsis}{suffix}{extension}'.format(**locals())
def paths_from_keys(base_path, keys):
"""
Return a tuple of (parent dir path, filename) for a cache entry built from a cache
keys triple and a base_directory. Ensure that the parent directory exist.
"""
if on_linux:
keys = [path_to_bytes(k) for k in keys]
base_path = path_to_bytes(base_path)
else:
keys = [path_to_unicode(k) for k in keys]
base_path = path_to_unicode(base_path)
dir1, dir2, file_name = keys
parent = os.path.join(base_path, dir1, dir2)
fileutils.create_dir(parent)
return parent, file_name
def paths_from_keys(base_path, keys):
"""
Return a tuple of (parent dir path, filename) for a cache entry built from a cache
keys triple and a base_directory. Ensure that the parent directory exist.
"""
if on_linux:
keys = [path_to_bytes(k) for k in keys]
base_path = path_to_bytes(base_path)
else:
keys = [path_to_unicode(k) for k in keys]
base_path = path_to_unicode(base_path)
dir1, dir2, file_name = keys
parent = os.path.join(base_path, dir1, dir2)
fileutils.create_dir(parent)
return parent, file_name
def resource_paths(base_path, diag, scans_cache_class, pre_scan_plugins=()):
"""
Yield `Resource` objects for all the files found at base_path
(either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
if base_path:
if on_linux:
base_path = path_to_bytes(base_path)
else:
base_path = path_to_unicode(base_path)
base_path = os.path.abspath(os.path.normpath(os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = {}
if pre_scan_plugins:
for plugin in pre_scan_plugins:
ignores.update(plugin.get_ignores())
ignores.update(ignore.ignores_VCS)
ignorer = build_ignorer(ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignorer)
for abs_path in resources:
resource = Resource(scans_cache_class, abs_path, base_is_dir, len_base_path)
# always fetch infos and cache.
resource.put_info(scan_infos(abs_path, diag=diag))
yield resource
def update_path_environment(new_path, _os_module=os):
"""
Update the PATH environment variable by adding `new_path` to the front
of PATH if `new_path` is not alreday in the PATH.
"""
# note: _os_module is used to facilitate mock testing using an
# object with a sep string attribute and an environ mapping
# attribute
if not new_path:
return
new_path = new_path.strip()
if not new_path:
return
path_env = _os_module.environ.get(b'PATH')
if not path_env:
# this is quite unlikely to ever happen, but here for safety
path_env = ''
# ensure we use unicode or bytes depending on OSes
if on_linux:
new_path = path_to_bytes(new_path)
path_env = path_to_bytes(path_env)
sep = _os_module.pathsep
else:
new_path = path_to_unicode(new_path)
path_env = path_to_unicode(path_env)
sep = unicode(_os_module.pathsep)
path_segments = path_env.split(sep)
# add lib path to the front of the PATH env var
# this will use bytes on Linux and unicode elsewhere
if new_path not in path_segments:
if not path_env:
new_path_env = new_path
else:
new_path_env = sep.join([new_path, path_env])
if not on_linux:
# recode to bytes using FS encoding
new_path_env = path_to_bytes(new_path_env)
# ... and set the variable back as bytes
_os_module.environ[b'PATH'] = new_path_env
def log_file_path(cls, logfile_fd, path):
"""
Log file path in the cache logfile_fd **opened** file descriptor.
"""
# we dump one path per line written as bytes or unicode
if on_linux:
path = path_to_bytes(path) + b'\n'
else:
path = path_to_unicode(path) + '\n'
logfile_fd.write(path)
def log_file_path(cls, logfile_fd, path):
"""
Log file path in the cache logfile_fd **opened** file descriptor.
"""
# we dump one path per line written as bytes or unicode
if on_linux:
path = path_to_bytes(path) + b'\n'
else:
path = path_to_unicode(path) + '\n'
logfile_fd.write(path)
def get_relative_path(path, len_base_path, base_is_dir):
"""
Return a posix relative path from the posix 'path' relative to a
base path of `len_base_path` length where the base is a directory if
`base_is_dir` True or a file otherwise.
"""
path = path_to_unicode(path)
if base_is_dir:
rel_path = path[len_base_path:]
else:
rel_path = fileutils.file_name(path)
return rel_path.lstrip('/')
def get_relative_path(path, len_base_path, base_is_dir):
"""
Return a posix relative path from the posix 'path' relative to a
base path of `len_base_path` length where the base is a directory if
`base_is_dir` True or a file otherwise.
"""
path = path_to_unicode(path)
if base_is_dir:
rel_path = path[len_base_path:]
else:
rel_path = fileutils.file_name(path)
return rel_path.lstrip('/')
def scan_one(location, scanners, diag=False):
"""
Scan one file or directory at `location` and return a scan result
mapping, calling every scanner callable in the `scanners` mapping of
(scan name -> scan function).
The scan result mapping contain a 'scan_errors' key with a list of
error messages. If `diag` is True, 'scan_errors' error messages also
contain detailed diagnostic information such as a traceback if
available.
"""
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
scan_result = OrderedDict()
scan_errors = []
for scan_name, scanner in scanners.items():
if not scanner:
continue
try:
scan_details = scanner(location)
# consume generators
if isinstance(scan_details, GeneratorType):
scan_details = list(scan_details)
scan_result[scan_name] = scan_details
except TimeoutError:
raise
except Exception as e:
# never fail but instead add an error message and keep an empty scan:
scan_result[scan_name] = []
messages = ['ERROR: ' + scan_name + ': ' + e.message]
if diag:
messages.append('ERROR: ' + scan_name + ': ' +
traceback.format_exc())
scan_errors.extend(messages)
# put errors last, after scans proper
scan_result['scan_errors'] = scan_errors
return scan_result
def scan_one(location, scanners, diag=False):
"""
Scan one file or directory at `location` and return a scan result
mapping, calling every scanner callable in the `scanners` mapping of
(scan name -> scan function).
The scan result mapping contain a 'scan_errors' key with a list of
error messages. If `diag` is True, 'scan_errors' error messages also
contain detailed diagnostic information such as a traceback if
available.
"""
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
scan_result = OrderedDict()
scan_errors = []
for scan_name, scanner in scanners.items():
if not scanner:
continue
try:
scan_details = scanner(location)
# consume generators
if isinstance(scan_details, GeneratorType):
scan_details = list(scan_details)
scan_result[scan_name] = scan_details
except TimeoutError:
raise
except Exception as e:
# never fail but instead add an error message and keep an empty scan:
scan_result[scan_name] = []
messages = ['ERROR: ' + scan_name + ': ' + e.message]
if diag:
messages.append('ERROR: ' + scan_name + ': ' + traceback.format_exc())
scan_errors.extend(messages)
# put errors last, after scans proper
scan_result['scan_errors'] = scan_errors
return scan_result
def resource_paths(base_path, diag, scans_cache_class, pre_scan_plugins=None):
"""
Yield `Resource` objects for all the files found at base_path
(either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
if base_path:
if on_linux:
base_path = path_to_bytes(base_path)
else:
base_path = path_to_unicode(base_path)
base_path = os.path.abspath(os.path.normpath(
os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = {}
if pre_scan_plugins:
for plugin in pre_scan_plugins:
ignores.update(plugin.get_ignores())
ignores.update(ignore.ignores_VCS)
ignorer = build_ignorer(ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignorer)
for abs_path in resources:
resource = Resource(scans_cache_class, abs_path, base_is_dir,
len_base_path)
# always fetch infos and cache.
resource.put_info(scan_infos(abs_path, diag=diag))
if pre_scan_plugins:
for plugin in pre_scan_plugins:
resource = plugin.process_resource(resource)
if resource:
yield resource
def iterate(self, scan_names, root_dir=None, paths_subset=tuple()):
"""
Yield scan data for all cached scans e.g. the whole cache given
a list of scan names.
If a `paths_subset` sequence of paths is provided, then only
these paths are iterated.
The logfile MUST have been closed before calling this method.
"""
if on_linux:
paths_subset = set(path_to_bytes(p) for p in paths_subset)
else:
paths_subset = set(path_to_unicode(p) for p in paths_subset)
if on_linux:
log_opener = partial(open, self.cache_files_log, 'rb')
else:
log_opener = partial(codecs.open,
self.cache_files_log,
'rb',
encoding='utf-8')
EOL = b'\n' if on_linux else '\n'
with log_opener() as cached_files:
# iterate paths, one by line
for file_log in cached_files:
# must be unicode
path = file_log.rstrip(EOL)
if paths_subset and path not in paths_subset:
continue
file_info = self.get_info(path)
if on_linux:
unicode_path = path_to_unicode(path)
else:
unicode_path = path
if root_dir:
# must be unicode
if on_linux:
root_dir = path_to_unicode(root_dir)
rooted_path = posixpath.join(root_dir, unicode_path)
else:
rooted_path = unicode_path
rooted_path = fileutils.as_posixpath(rooted_path)
logger_debug('iterate:', 'rooted_path:', rooted_path)
# rare but possible corner case
if file_info is None:
no_info = (
'ERROR: file info unavailable in cache: '
'This is either a bug or processing was aborted with CTRL-C.'
)
scan_result = OrderedDict(path=rooted_path)
scan_result['scan_errors'] = [no_info]
if TRACE:
logger_debug('iterate:', 'scan_result:', scan_result,
'for path:', rooted_path, '\n')
yield scan_result
continue
_unicode_path_from_file_info = file_info.pop('path')
scan_result = OrderedDict(path=rooted_path)
if 'infos' in scan_names:
# info are always collected but only returned if requested
# we flatten these as direct attributes of a file object
scan_result.update(file_info.items())
if not scan_result.get('scan_errors'):
scan_result['scan_errors'] = []
# check if we have more than just infos
if ['infos'] != scan_names:
errors = scan_result['scan_errors']
scan_details = self.get_scan(path, file_info)
if scan_details is None:
no_scan_details = (
'ERROR: scan details unavailable in cache: '
'This is either a bug or processing was aborted with CTRL-C.'
)
errors.append(no_scan_details)
else:
# append errors to other top level errors if any
scan_errors = scan_details.pop('scan_errors', [])
errors.extend(scan_errors)
scan_result.update(scan_details)
if TRACE:
logger_debug('iterate:', 'scan_result:', scan_result,
'for path:', rooted_path, '\n')
yield scan_result
def iterate(self, scan_names, root_dir=None, paths_subset=tuple()):
"""
Yield scan data for all cached scans e.g. the whole cache given
a list of scan names.
If a `paths_subset` sequence of paths is provided, then only
these paths are iterated.
The logfile MUST have been closed before calling this method.
"""
if on_linux:
paths_subset = set(path_to_bytes(p) for p in paths_subset)
else:
paths_subset = set(path_to_unicode(p) for p in paths_subset)
if on_linux:
log_opener = partial(open, self.cache_files_log, 'rb')
else:
log_opener = partial(codecs.open, self.cache_files_log, 'rb', encoding='utf-8')
EOL = b'\n' if on_linux else '\n'
with log_opener() as cached_files:
# iterate paths, one by line
for file_log in cached_files:
# must be unicode
path = file_log.rstrip(EOL)
if paths_subset and path not in paths_subset:
continue
file_info = self.get_info(path)
if on_linux:
unicode_path = path_to_unicode(path)
else:
unicode_path = path
if root_dir:
# must be unicode
if on_linux:
root_dir = path_to_unicode(root_dir)
rooted_path = posixpath.join(root_dir, unicode_path)
else:
rooted_path = unicode_path
rooted_path = fileutils.as_posixpath(rooted_path)
logger_debug('iterate:', 'rooted_path:', rooted_path)
# rare but possible corner case
if file_info is None:
no_info = ('ERROR: file info unavailable in cache: '
'This is either a bug or processing was aborted with CTRL-C.')
scan_result = OrderedDict(path=rooted_path)
scan_result['scan_errors'] = [no_info]
if TRACE:
logger_debug('iterate:', 'scan_result:', scan_result, 'for path:', rooted_path, '\n')
yield scan_result
continue
_unicode_path_from_file_info = file_info.pop('path')
scan_result = OrderedDict(path=rooted_path)
if 'infos' in scan_names:
# info are always collected but only returned if requested
# we flatten these as direct attributes of a file object
scan_result.update(file_info.items())
if not scan_result.get('scan_errors'):
scan_result['scan_errors'] = []
# check if we have more than just infos
if ['infos'] != scan_names:
errors = scan_result['scan_errors']
scan_details = self.get_scan(path, file_info)
if scan_details is None:
no_scan_details = (
'ERROR: scan details unavailable in cache: '
'This is either a bug or processing was aborted with CTRL-C.')
errors.append(no_scan_details)
else:
# append errors to other top level errors if any
scan_errors = scan_details.pop('scan_errors', [])
errors.extend(scan_errors)
scan_result.update(scan_details)
if TRACE:
logger_debug('iterate:', 'scan_result:', scan_result, 'for path:', rooted_path, '\n')
yield scan_result