def test_get_relative_path(self):
# plain file without parent
assert 'file' == utils.get_relative_path(path='/file', len_base_path=5, base_is_dir=False)
# plain file in a deep path
assert 'that' == utils.get_relative_path(path='/this/file/that', len_base_path=5, base_is_dir=False)
# plain path with directories
assert 'file/that' == utils.get_relative_path(path='/this/file/that', len_base_path=5, base_is_dir=True)
assert 'that' == utils.get_relative_path(path='/this/file/that', len_base_path=10, base_is_dir=True)
assert 'this/file/that' == utils.get_relative_path(path='/foo//this/file/that', len_base_path=4, base_is_dir=True)
def extract_end():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = as_posixpath(xev.source)
source = utils.get_relative_path(original_input, abs_input, source)
for e in xev.errors:
summary.append(
style('ERROR extracting: %(source)s: %(e)r' % locals(),
fg='red',
reset=False))
for warn in xev.warnings:
summary.append(
style('WARNING extracting: %(source)s: %(warn)r' %
locals(),
fg='yellow',
reset=False))
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
summary.append(style('Extracting done.', fg=summary_color, reset=True))
return '\n'.join(summary)
def display_extract_summary():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = fileutils.as_posixpath(xev.source)
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
source = utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir)
for e in xev.errors:
echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red')
for warn in xev.warnings:
echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow')
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
echo_stderr('Extracting done.', fg=summary_color, reset=True)
def resource_paths(base_path, user_ignores):
"""
Yield tuples of (absolute path, base_path-relative path) for all the files found
at base_path (either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
base_path = os.path.abspath(os.path.normpath(
os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = dict()
ignores.update(user_ignores)
ignores.update(ignore.ignores_VCS)
ignored = partial(ignore.is_ignored, ignores=ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignored)
for abs_path in resources:
posix_path = fileutils.as_posixpath(abs_path)
# fix paths: keep the path as relative to the original base_path
rel_path = get_relative_path(posix_path, len_base_path, base_is_dir)
yield abs_path, rel_path
def extract_end():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = as_posixpath(xev.source)
source = utils.get_relative_path(original_input, abs_input, source)
for e in xev.errors:
summary.append(style('ERROR extracting: %(source)s: %(e)r' % locals(), fg='red', reset=False))
for warn in xev.warnings:
summary.append(style('WARNING extracting: %(source)s: %(warn)r' % locals(), fg='yellow', reset=False))
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
summary.append(style('Extracting done.', fg=summary_color, reset=True))
return '\n'.join(summary)
def display_extract_summary():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = fileutils.as_posixpath(xev.source)
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
source = utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir)
for e in xev.errors:
echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red')
for warn in xev.warnings:
echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow')
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
echo_stderr('Extracting done.', fg=summary_color, reset=True)
def test_get_relative_path(self):
assert '.' == utils.get_relative_path(base='.',
base_resolved='/',
path='/')
assert 'file' == utils.get_relative_path(base='file',
base_resolved='/file',
path='/file')
assert 'this/file/that' == utils.get_relative_path(
base='this/file/',
base_resolved='/this/file/',
path='/this/file/that')
assert 'this/file/that' == utils.get_relative_path(
base='this/file/',
base_resolved='/this/file/',
path='/this/file/that')
assert 'this/file/that' == utils.get_relative_path(
base='this/file/',
base_resolved='/foo/this/file/',
path='/foo//this/file/that')
assert 'this/file/that' == utils.get_relative_path(
base='this/file',
base_resolved='/foo/this/file/',
path='/foo//this/file/that')
assert 'this/file/that' == utils.get_relative_path(
base='this/file',
base_resolved='/foo/this/file/',
path='/foo/this/file/that')
def test_get_relative_path(self):
assert '.' == utils.get_relative_path(base='.', base_resolved='/', path='/')
assert 'file' == utils.get_relative_path(base='file', base_resolved='/file', path='/file')
assert 'this/file/that' == utils.get_relative_path(base='this/file/', base_resolved='/this/file/', path='/this/file/that')
assert 'this/file/that' == utils.get_relative_path(base='this/file/', base_resolved='/this/file/', path='/this/file/that')
assert 'this/file/that' == utils.get_relative_path(base='this/file/', base_resolved='/foo/this/file/', path='/foo//this/file/that')
assert 'this/file/that' == utils.get_relative_path(base='this/file', base_resolved='/foo/this/file/', path='/foo//this/file/that')
assert 'this/file/that' == utils.get_relative_path(base='this/file', base_resolved='/foo/this/file/', path='/foo/this/file/that')
def test_get_relative_path(self):
# plain file without parent
assert 'file' == utils.get_relative_path(path='/file',
len_base_path=5,
base_is_dir=False)
# plain file in a deep path
assert 'that' == utils.get_relative_path(path='/this/file/that',
len_base_path=5,
base_is_dir=False)
# plain path with directories
assert 'file/that' == utils.get_relative_path(path='/this/file/that',
len_base_path=5,
base_is_dir=True)
assert 'that' == utils.get_relative_path(path='/this/file/that',
len_base_path=10,
base_is_dir=True)
assert 'this/file/that' == utils.get_relative_path(
path='/foo//this/file/that', len_base_path=4, base_is_dir=True)
def __init__(self, scan_cache_class, abs_path, base_is_dir, len_base_path):
self.scan_cache_class = scan_cache_class()
self.is_cached = False
self.abs_path = abs_path
self.base_is_dir = base_is_dir
posix_path = as_posixpath(abs_path)
# fix paths: keep the path as relative to the original
# base_path. This is always Unicode
self.rel_path = get_relative_path(posix_path, len_base_path, base_is_dir)
self.infos = OrderedDict()
self.infos['path'] = self.rel_path
def __init__(self, scan_cache_class, abs_path, base_is_dir, len_base_path):
self.scan_cache_class = scan_cache_class()
self.is_cached = False
self.abs_path = abs_path
self.base_is_dir = base_is_dir
posix_path = as_posixpath(abs_path)
# fix paths: keep the path as relative to the original
# base_path. This is always Unicode
self.rel_path = get_relative_path(posix_path, len_base_path,
base_is_dir)
self.infos = OrderedDict()
self.infos['path'] = self.rel_path
def extract_event(item):
"""
Display an extract event.
"""
if not item:
return ''
if verbose:
if item.done:
return ''
line = utils.get_relative_path(original_input, abs_input, as_posixpath(item.source)) or ''
else:
line = fileutils.file_name(item.source) or ''
return 'Extracting: %(line)s' % locals()
def extract_event(item):
"""
Display an extract event.
"""
if not item:
return ''
if verbose:
if item.done:
return ''
line = utils.get_relative_path(original_input, abs_input,
as_posixpath(item.source)) or ''
else:
line = fileutils.file_name(item.source) or ''
return 'Extracting: %(line)s' % locals()
def extract_event(item):
"""
Display an extract event.
"""
if quiet:
return ''
if not item:
return ''
source = item.source
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
if verbose:
if item.done:
return ''
line = source and utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or ''
else:
line = source and fileutils.file_name(source) or ''
if not isinstance(line, unicode):
line = toascii(line, translit=True).decode('utf-8', 'replace')
return 'Extracting: %(line)s' % locals()
def extract_event(item):
"""
Display an extract event.
"""
if quiet:
return ''
if not item:
return ''
source = item.source
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
if verbose:
if item.done:
return ''
line = source and utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or ''
else:
line = source and fileutils.file_name(source) or ''
if not isinstance(line, unicode):
line = toascii(line, translit=True).decode('utf-8', 'replace')
return 'Extracting: %(line)s' % locals()
def scan(input_path, copyright=True, license=True, package=True, # @ReservedAssignment
email=False, url=False, info=True, verbose=False, quiet=False): # @ReservedAssignment
"""
Do the scans proper, return results.
"""
# save paths to report paths relative to the original input
original_input = fileutils.as_posixpath(input_path)
abs_input = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input_path)))
# note: "flag and function" expressions return the function if flag is True
scanners = {
'copyrights': copyright and get_copyrights,
'licenses': license and get_licenses,
'packages': package and get_package_infos,
'emails': email and get_emails,
'urls': url and get_urls,
'infos': info and get_file_infos,
}
results = []
# note: we inline progress display functions to close on some args
def scan_start():
"""Progress event displayed at start of scan"""
return style('Scanning files...', fg='green')
def scan_event(item):
"""Progress event displayed each time a file is scanned"""
if item:
line = verbose and item or fileutils.file_name(item) or ''
return 'Scanning: %(line)s' % locals()
def scan_end():
"""Progress event displayed at end of scan"""
has_warnings = False
has_errors = False
summary = []
summary_color = 'green'
summary_color = has_warnings and 'yellow' or summary_color
summary_color = has_errors and 'red' or summary_color
summary.append(style('Scanning done.', fg=summary_color, reset=True))
return '\n'.join(summary)
ignored = partial(ignore.is_ignored, ignores=ignore.ignores_VCS, unignores={})
resources = fileutils.resource_iter(abs_input, ignored=ignored)
with utils.progressmanager(resources,
item_show_func=scan_event,
start_show_func=scan_start,
finish_show_func=scan_end,
verbose=verbose,
show_pos=True,
quiet=quiet
) as progressive_resources:
for resource in progressive_resources:
res = fileutils.as_posixpath(resource)
# fix paths: keep the location as relative to the original input
relative_path = utils.get_relative_path(original_input, abs_input, res)
scan_result = OrderedDict(location=relative_path)
# Should we yield instead?
scan_result.update(scan_one(res, scanners))
results.append(scan_result)
# TODO: eventually merge scans for the same resource location...
# TODO: fix absolute paths as relative to original input argument...
return results
def scan(
input_path,
copyright=True,
license=True,
package=True, # @ReservedAssignment
info=True,
verbose=False,
quiet=False): # @ReservedAssignment
"""
Do the scans proper, return results.
"""
# save paths to report paths relative to the original input
original_input = fileutils.as_posixpath(input_path)
abs_input = fileutils.as_posixpath(
os.path.abspath(os.path.expanduser(input_path)))
# note: "flag and function" expressions return the function if flag is True
scanners = {
'copyrights': copyright and get_copyrights,
'licenses': license and get_licenses,
'packages': package and get_package_infos,
'infos': info and get_file_infos,
}
results = []
# note: we inline progress display functions to close on some args
def scan_start():
"""Progress event displayed at start of scan"""
return style('Scanning files...', fg='green')
def scan_event(item):
"""Progress event displayed each time a file is scanned"""
if item:
line = verbose and item or fileutils.file_name(item) or ''
return 'Scanning: %(line)s' % locals()
def scan_end():
"""Progress event displayed at end of scan"""
has_warnings = False
has_errors = False
summary = []
summary_color = 'green'
summary_color = has_warnings and 'yellow' or summary_color
summary_color = has_errors and 'red' or summary_color
summary.append(style('Scanning done.', fg=summary_color, reset=True))
return '\n'.join(summary)
ignored = partial(ignore.is_ignored,
ignores=ignore.ignores_VCS,
unignores={})
resources = fileutils.resource_iter(abs_input, ignored=ignored)
with utils.progressmanager(resources,
item_show_func=scan_event,
start_show_func=scan_start,
finish_show_func=scan_end,
verbose=verbose,
show_pos=True,
quiet=quiet) as progressive_resources:
for resource in progressive_resources:
res = fileutils.as_posixpath(resource)
# fix paths: keep the location as relative to the original input
relative_path = utils.get_relative_path(original_input, abs_input,
res)
scan_result = OrderedDict(location=relative_path)
# Should we yield instead?
scan_result.update(scan_one(res, scanners))
results.append(scan_result)
# TODO: eventually merge scans for the same resource location...
# TODO: fix absolute paths as relative to original input argument...
return results
