def test_is_dir(self):
test_dir = self.get_test_loc('symlink', copy=True)
temp_dir = fileutils.get_temp_dir()
test_link = join(temp_dir, 'test-dir-link')
os.symlink(test_dir, test_link)
assert filetype.is_dir(test_link, follow_symlinks=True)
assert not filetype.is_dir(test_link, follow_symlinks=False)
def walk(location, ignored=None, follow_symlinks=False):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks (unless `follow_symlinks` is True),
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
If `follow_symlinks` is True, then symlinks will not be ignored and be
collected like regular files and directories
"""
if on_linux and py2:
location = fsencode(location)
# TODO: consider using the new "scandir" module for some speed-up.
is_ignored = ignored(location) if ignored else False
if is_ignored:
if TRACE:
logger_debug('walk: ignored:', location, is_ignored)
return
if filetype.is_file(location, follow_symlinks=follow_symlinks):
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location, follow_symlinks=follow_symlinks):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or (ignored and ignored(loc)):
if (follow_symlinks and filetype.is_link(loc)
and not filetype.is_broken_link(location)):
pass
else:
if TRACE:
ign = ignored and ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc, follow_symlinks=follow_symlinks):
dirs.append(name)
elif filetype.is_file(loc, follow_symlinks=follow_symlinks):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr),
ignored,
follow_symlinks=follow_symlinks):
yield tripple
def parse(location):
"""
Return a Package built from parsing a file or directory at 'location'
"""
if filetype.is_dir(location):
package = parse_unpackaged_source(location)
if package:
parse_dependencies(location, package)
return package
else:
file_name = fileutils.file_name(location)
parsers = {
'setup.py': parse_setup_py,
'requirements.txt': parse_requirements_txt,
'requirements.in': parse_requirements_txt,
'Pipfile.lock': parse_pipfile_lock,
'metadata.json': parse_metadata,
'PKG-INFO': parse_unpackaged_source,
'.whl': parse_wheel,
'.egg': parse_egg_binary,
'.tar.gz': parse_source_distribution,
'.zip': parse_source_distribution,
}
for name, parser in parsers.items():
if file_name.endswith(name):
package = parser(location)
if package:
parent_directory = fileutils.parent_directory(location)
parse_dependencies(parent_directory, package)
return package
def __init__(self, location):
if not location or (not os.path.exists(location) and not filetype.is_broken_link(location)):
raise IOError("[Errno 2] No such file or directory: " "'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# FIXME: the way the True and False values are checked in properties is verbose and contrived at best
# and is due to use None/True/False as different values
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
def parse_with_dparse(location):
is_dir = filetype.is_dir(location)
if is_dir:
return
file_name = fileutils.file_name(location)
dependency_type = get_dependency_type(file_name)
if dependency_type not in (filetypes.requirements_txt,
filetypes.conda_yml,
filetypes.tox_ini,
filetypes.pipfile,
filetypes.pipfile_lock):
return
if py2:
mode = 'rb'
else:
mode = 'r'
with open(location, mode) as f:
content = f.read()
df = dparse.parse(content, file_type=dependency_type)
df_dependencies = df.dependencies
if not df_dependencies:
return
package_dependencies = []
for df_dependency in df_dependencies:
specs = list(df_dependency.specs._specs)
is_resolved = False
requirement = None
purl = PackageURL(
type='pypi',
name=df_dependency.name
).to_string()
if specs:
requirement = str(df_dependency.specs)
for spec in specs:
operator = spec.operator
version = spec.version
if any(operator == element for element in ('==', '===')):
is_resolved = True
purl = PackageURL(
type='pypi',
name=df_dependency.name,
version=version
).to_string()
package_dependencies.append(
models.DependentPackage(
purl=purl,
scope='dependencies',
is_runtime=True,
is_optional=False,
is_resolved=is_resolved,
requirement=requirement
)
)
return package_dependencies
def resource_paths(base_path, user_ignores):
"""
Yield tuples of (absolute path, base_path-relative path) for all the files found
at base_path (either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
base_path = os.path.abspath(os.path.normpath(
os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = dict()
ignores.update(user_ignores)
ignores.update(ignore.ignores_VCS)
ignored = partial(ignore.is_ignored, ignores=ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignored)
for abs_path in resources:
posix_path = fileutils.as_posixpath(abs_path)
# fix paths: keep the path as relative to the original base_path
rel_path = get_relative_path(posix_path, len_base_path, base_is_dir)
yield abs_path, rel_path
def parse2(location):
"""
Parse using the pkginfo library according the file types and return package.
"""
is_dir = filetype.is_dir(location)
if is_dir:
parser = parse_unpackaged_source
package = parser(location)
if package:
parse_dependencies(location, package)
return package
else:
file_name = fileutils.file_name(location)
parsers = {
'setup.py': parse_unpackaged_source,
'.whl': parse_wheel,
'.egg': parse_egg_binary,
'.tar.gz': parse_source_distribution,
'.zip': parse_source_distribution,
}
for name, parser in parsers.items():
if file_name.endswith(name):
package = parser(location)
if package:
parent_directory = fileutils.parent_directory(location)
parse_dependencies(parent_directory, package)
return package
def resource_paths(base_path, diag, scans_cache_class, pre_scan_plugins=()):
"""
Yield `Resource` objects for all the files found at base_path
(either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
if base_path:
if on_linux:
base_path = path_to_bytes(base_path)
else:
base_path = path_to_unicode(base_path)
base_path = os.path.abspath(os.path.normpath(os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = {}
if pre_scan_plugins:
for plugin in pre_scan_plugins:
ignores.update(plugin.get_ignores())
ignores.update(ignore.ignores_VCS)
ignorer = build_ignorer(ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignorer)
for abs_path in resources:
resource = Resource(scans_cache_class, abs_path, base_is_dir, len_base_path)
# always fetch infos and cache.
resource.put_info(scan_infos(abs_path, diag=diag))
yield resource
def chmod(location, flags, recurse=False):
"""
Update permissions for `location` with with `flags`. `flags` is one of R,
RW, RX or RWX with the same semantics as in the chmod command. Update is
done recursively if `recurse`.
"""
if not location or not os.path.exists(location):
return
if on_linux and py2:
location = fsencode(location)
location = os.path.abspath(location)
new_flags = flags
if filetype.is_dir(location):
# POSIX dirs need to be executable to be readable,
# and to be writable so we can change perms of files inside
new_flags = RWX
# FIXME: do we really need to change the parent directory perms?
# FIXME: may just check them instead?
parent = os.path.dirname(location)
current_stat = stat.S_IMODE(os.stat(parent).st_mode)
if not is_rwx(parent):
os.chmod(parent, current_stat | RWX)
if filetype.is_regular(location):
current_stat = stat.S_IMODE(os.stat(location).st_mode)
os.chmod(location, current_stat | new_flags)
if recurse:
chmod_tree(location, flags)
def chmod(location, flags, recurse=False):
"""
Update permissions for `location` with with `flags`. `flags` is one of R,
RW, RX or RWX with the same semantics as in the chmod command. Update is
done recursively if `recurse`.
"""
if not location or not os.path.exists(location):
return
location = os.path.abspath(location)
new_flags = flags
if filetype.is_dir(location):
# POSIX dirs need to be executable to be readable,
# and to be writable so we can change perms of files inside
new_flags = RWX
# FIXME: do we really need to change the parent directory perms?
# FIXME: may just check them instead?
parent = os.path.dirname(location)
current_stat = stat.S_IMODE(os.stat(parent).st_mode)
if not is_rwx(parent):
os.chmod(parent, current_stat | RWX)
if filetype.is_regular(location):
current_stat = stat.S_IMODE(os.stat(location).st_mode)
os.chmod(location, current_stat | new_flags)
if recurse:
chmod_tree(location, flags)
def __init__(self, location):
if (not location
or (not os.path.exists(location)
and not filetype.is_broken_link(location))):
raise IOError("[Errno 2] No such file or directory: "
"'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
def __init__(self, location):
if (not location
or (not os.path.exists(location)
and not filetype.is_broken_link(location))):
raise IOError("[Errno 2] No such file or directory: "
"'%(location)r'" % locals())
self.location = location
# flags and values
self.is_file = filetype.is_file(location)
self.is_dir = filetype.is_dir(location)
self.is_regular = filetype.is_regular(location)
self.is_special = filetype.is_special(location)
self.date = filetype.get_last_modified_date(location)
self.is_link = filetype.is_link(location)
self.is_broken_link = filetype.is_broken_link(location)
# FIXME: the way the True and False values are checked in properties is verbose and contrived at best
# and is due to use None/True/False as different values
# computed on demand
self._size = None
self._link_target = None
self._mimetype_python = None
self._filetype_file = None
self._mimetype_file = None
self._filetype_pygments = None
self._is_pdf_with_text = None
self._is_text = None
self._is_binary = None
self._contains_text = None
def parse_with_dparse(location):
is_dir = filetype.is_dir(location)
if is_dir:
return
file_name = fileutils.file_name(location)
if file_name not in (filetypes.requirements_txt, filetypes.conda_yml,
filetypes.tox_ini, filetypes.pipfile,
filetypes.pipfile_lock):
return
if py2:
mode = 'rb'
else:
mode = 'r'
with open(location, mode) as f:
content = f.read()
df = dparse.parse(content, file_type=file_name)
df_dependencies = df.dependencies
if not df_dependencies:
return
package_dependencies = []
for df_dependency in df_dependencies:
specs = df_dependency.specs
requirement = None
if specs:
requirement = str(specs)
package_dependencies.append(
models.DependentPackage(
purl=PackageURL(type='pypi',
name=df_dependency.name).to_string(),
scope='dependencies',
is_runtime=True,
is_optional=False,
requirement=requirement,
))
return package_dependencies
def walk(location, ignored=ignore_nothing):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks, .
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
"""
if on_linux:
location = path_to_bytes(location)
# TODO: consider using the new "scandir" module for some speed-up.
if TRACE:
ign = ignored(location)
logger_debug('walk: ignored:', location, ign)
if ignored(location):
return
if filetype.is_file(location):
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or ignored(loc):
if TRACE:
ign = ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc):
dirs.append(name)
elif filetype.is_file(loc):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr), ignored):
yield tripple
def walk(location, ignored=ignore_nothing):
"""
Walk location returning the same tuples as os.walk but with a different
behavior:
- always walk top-down, breadth-first.
- always ignore and never follow symlinks, .
- always ignore special files (FIFOs, etc.)
- optionally ignore files and directories by invoking the `ignored`
callable on files and directories returning True if it should be ignored.
- location is a directory or a file: for a file, the file is returned.
"""
if on_linux:
location = path_to_bytes(location)
# TODO: consider using the new "scandir" module for some speed-up.
if TRACE:
ign = ignored(location)
logger_debug('walk: ignored:', location, ign)
if ignored(location):
return
if filetype.is_file(location) :
yield parent_directory(location), [], [file_name(location)]
elif filetype.is_dir(location):
dirs = []
files = []
# TODO: consider using scandir
for name in os.listdir(location):
loc = os.path.join(location, name)
if filetype.is_special(loc) or ignored(loc):
if TRACE:
ign = ignored(loc)
logger_debug('walk: ignored:', loc, ign)
continue
# special files and symlinks are always ignored
if filetype.is_dir(loc):
dirs.append(name)
elif filetype.is_file(loc):
files.append(name)
yield location, dirs, files
for dr in dirs:
for tripple in walk(os.path.join(location, dr), ignored):
yield tripple
def get_file_infos(location):
"""
Return a mapping of file information collected from the file or
directory at `location`.
"""
from commoncode import fileutils
from commoncode import filetype
from commoncode.hash import multi_checksums
from typecode import contenttype
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
infos = OrderedDict()
is_file = filetype.is_file(location)
is_dir = filetype.is_dir(location)
T = contenttype.get_type(location)
infos['type'] = filetype.get_type(location, short=False)
name = fileutils.file_name(location)
if is_file:
base_name, extension = fileutils.splitext(location)
else:
base_name = name
extension = ''
if on_linux:
infos['name'] = path_to_unicode(name)
infos['base_name'] = path_to_unicode(base_name)
infos['extension'] = path_to_unicode(extension)
else:
infos['name'] = name
infos['base_name'] = base_name
infos['extension'] = extension
infos['date'] = is_file and filetype.get_last_modified_date(
location) or None
infos['size'] = T.size
infos.update(multi_checksums(location, (
'sha1',
'md5',
)))
infos['files_count'] = is_dir and filetype.get_file_count(location) or None
infos['mime_type'] = is_file and T.mimetype_file or None
infos['file_type'] = is_file and T.filetype_file or None
infos['programming_language'] = is_file and T.programming_language or None
infos['is_binary'] = bool(is_file and T.is_binary)
infos['is_text'] = bool(is_file and T.is_text)
infos['is_archive'] = bool(is_file and T.is_archive)
infos['is_media'] = bool(is_file and T.is_media)
infos['is_source'] = bool(is_file and T.is_source)
infos['is_script'] = bool(is_file and T.is_script)
return infos
def chmod_tree(location, flags):
"""
Update permissions recursively in a directory tree `location`.
"""
if filetype.is_dir(location):
for top, dirs, files in walk(location):
for d in dirs:
chmod(os.path.join(top, d), flags, recurse=False)
for f in files:
chmod(os.path.join(top, f), flags, recurse=False)
def chmod_tree(location, flags):
"""
Update permissions recursively in a directory tree `location`.
"""
if filetype.is_dir(location):
for top, dirs, files in walk(location):
for d in dirs:
chmod(os.path.join(top, d), flags, recurse=False)
for f in files:
chmod(os.path.join(top, f), flags, recurse=False)
def get_file_infos(location):
"""
Return a mapping of file information collected from the file or
directory at `location`.
"""
from commoncode import fileutils
from commoncode import filetype
from commoncode.hash import multi_checksums
from typecode import contenttype
if on_linux:
location = path_to_bytes(location)
else:
location = path_to_unicode(location)
infos = OrderedDict()
is_file = filetype.is_file(location)
is_dir = filetype.is_dir(location)
T = contenttype.get_type(location)
infos['type'] = filetype.get_type(location, short=False)
name = fileutils.file_name(location)
if is_file:
base_name, extension = fileutils.splitext(location)
else:
base_name = name
extension = ''
if on_linux:
infos['name'] = path_to_unicode(name)
infos['base_name'] = path_to_unicode(base_name)
infos['extension'] = path_to_unicode(extension)
else:
infos['name'] = name
infos['base_name'] = base_name
infos['extension'] = extension
infos['date'] = is_file and filetype.get_last_modified_date(location) or None
infos['size'] = T.size
infos.update(multi_checksums(location, ('sha1', 'md5',)))
infos['files_count'] = is_dir and filetype.get_file_count(location) or None
infos['mime_type'] = is_file and T.mimetype_file or None
infos['file_type'] = is_file and T.filetype_file or None
infos['programming_language'] = is_file and T.programming_language or None
infos['is_binary'] = bool(is_file and T.is_binary)
infos['is_text'] = bool(is_file and T.is_text)
infos['is_archive'] = bool(is_file and T.is_archive)
infos['is_media'] = bool(is_file and T.is_media)
infos['is_source'] = bool(is_file and T.is_source)
infos['is_script'] = bool(is_file and T.is_script)
return infos
def delete(location, _err_handler=_rm_handler):
"""
Delete a directory or file at `location` recursively. Similar to "rm -rf"
in a shell or a combo of os.remove and shutil.rmtree.
"""
if not location:
return
if os.path.exists(location) or filetype.is_broken_link(location):
chmod(os.path.dirname(location), RW, recurse=False)
if filetype.is_dir(location):
shutil.rmtree(location, False, _rm_handler)
else:
os.remove(location)
def delete(location, _err_handler=_rm_handler):
"""
Delete a directory or file at `location` recursively. Similar to "rm -rf"
in a shell or a combo of os.remove and shutil.rmtree.
"""
if not location:
return
if os.path.exists(location) or filetype.is_broken_link(location):
chmod(os.path.dirname(location), RW, recurse=False)
if filetype.is_dir(location):
shutil.rmtree(location, False, _rm_handler)
else:
os.remove(location)
def _get_root_dir(input_path, strip_root=False):
"""
Return a root dir name or None.
"""
if strip_root:
root_dir = None
else:
_scanned_path = os.path.abspath(os.path.normpath(os.path.expanduser(input_path)))
if filetype.is_dir(_scanned_path):
root_dir = _scanned_path
else:
root_dir = fileutils.parent_directory(_scanned_path)
root_dir = fileutils.file_name(root_dir)
return root_dir
def delete(location, _err_handler=_rm_handler):
"""
Delete a directory or file at `location` recursively. Similar to "rm -rf"
in a shell or a combo of os.remove and shutil.rmtree.
This function is design to never fails: instead it logs warnings if the
file or directory was not deleted correctly.
"""
if not location:
return
if os.path.exists(location) or filetype.is_broken_link(location):
if filetype.is_dir(location):
shutil.rmtree(location, False, _rm_handler)
else:
os.remove(location)
def _get_root_dir(input_path, strip_root=False, full_root=False):
"""
Return a root dir name or None.
On Windows, the path uses POSIX (forward slash) separators.
"""
if strip_root:
return
scanned_path = os.path.abspath(
os.path.normpath(os.path.expanduser(input_path)))
scanned_path = fileutils.as_posixpath(scanned_path)
if full_root:
return scanned_path
if filetype.is_dir(scanned_path):
root_dir = scanned_path
else:
root_dir = fileutils.parent_directory(scanned_path)
return fileutils.file_name(root_dir)
def get_file_infos(location, as_list=True):
"""
Return a list of dictionaries of informations collected from the file or
directory at location.
"""
from commoncode import fileutils
from commoncode import filetype
from commoncode.hash import multi_checksums
from scancode import utils
from typecode import contenttype
infos = OrderedDict()
is_file = filetype.is_file(location)
is_dir = filetype.is_dir(location)
T = contenttype.get_type(location)
infos['type'] = filetype.get_type(location, short=False)
infos['name'] = fileutils.file_name(location)
infos['extension'] = is_file and fileutils.file_extension(location) or ''
infos['date'] = is_file and filetype.get_last_modified_date(
location) or None
infos['size'] = T.size
infos.update(multi_checksums(location, (
'sha1',
'md5',
)))
infos['files_count'] = is_dir and filetype.get_file_count(location) or None
infos['mime_type'] = is_file and T.mimetype_file or None
infos['file_type'] = is_file and T.filetype_file or None
infos['programming_language'] = is_file and T.programming_language or None
infos['is_binary'] = bool(is_file and T.is_binary)
infos['is_text'] = bool(is_file and T.is_text)
infos['is_archive'] = bool(is_file and T.is_archive)
infos['is_media'] = bool(is_file and T.is_media)
infos['is_source'] = bool(is_file and T.is_source)
infos['is_script'] = bool(is_file and T.is_script)
if as_list:
return [infos]
else:
return infos
def _get_root_dir(input_path, strip_root=False, full_root=False):
"""
Return a root dir name or None.
On Windows, the path uses POSIX (forward slash) separators.
"""
if strip_root:
return
scanned_path = os.path.abspath(os.path.normpath(os.path.expanduser(input_path)))
scanned_path = fileutils.as_posixpath(scanned_path)
if filetype.is_dir(scanned_path):
root_dir = scanned_path
else:
root_dir = fileutils.parent_directory(scanned_path)
root_dir = fileutils.as_posixpath(root_dir)
if full_root:
return root_dir
else:
return fileutils.file_name(root_dir)
def resource_paths(base_path, diag, scans_cache_class, pre_scan_plugins=None):
"""
Yield `Resource` objects for all the files found at base_path
(either a directory or file) given an absolute base_path. Only yield
Files, not directories.
absolute path is a native OS path.
base_path-relative path is a POSIX path.
The relative path is guaranted to be unicode and may be URL-encoded and may not
be suitable to address an actual file.
"""
if base_path:
if on_linux:
base_path = path_to_bytes(base_path)
else:
base_path = path_to_unicode(base_path)
base_path = os.path.abspath(os.path.normpath(
os.path.expanduser(base_path)))
base_is_dir = filetype.is_dir(base_path)
len_base_path = len(base_path)
ignores = {}
if pre_scan_plugins:
for plugin in pre_scan_plugins:
ignores.update(plugin.get_ignores())
ignores.update(ignore.ignores_VCS)
ignorer = build_ignorer(ignores, unignores={})
resources = fileutils.resource_iter(base_path, ignored=ignorer)
for abs_path in resources:
resource = Resource(scans_cache_class, abs_path, base_is_dir,
len_base_path)
# always fetch infos and cache.
resource.put_info(scan_infos(abs_path, diag=diag))
if pre_scan_plugins:
for plugin in pre_scan_plugins:
resource = plugin.process_resource(resource)
if resource:
yield resource
def is_node_modules(location):
return (filetype.is_dir(location)
and fileutils.file_name(location).lower() == 'node_modules')
def rebuild_rootfs(image, target_dir, layerid_len=DEFAULT_ID_LEN):
"""
Extract and merge all layers to target_dir. Extraction is done in
sequence from bottom (root) to top (latest layer).
Return a mapping of errors and a list of whiteouts/deleted files.
The extraction process consists of these steps:
- extract the layer in a temp directory
- move layer to the target directory, overwriting existing files
- if any, remove AUFS special files/dirs in the target directory
- if any, remove whiteouts file/directory pairs in the target directory
"""
from extractcode.extract import extract_file
assert filetype.is_dir(target_dir)
assert os.path.exists(target_dir)
extract_errors = []
# log whiteouts deletions
whiteouts = []
for layer_id, layer in image.layers.items():
layer_tarball = join(image.repo_dir, layer_id[:layerid_len],
LAYER_TAR_FILE)
logger.debug('Extracting layer tarball: %(layer_tarball)r' % locals())
temp_target = fileutils.get_temp_dir('conan-docker')
xevents = list(extract_file(layer_tarball, temp_target))
for x in xevents:
if x.warnings or x.errors:
extract_errors.extend(xevents)
# FIXME: the order of ops is WRONG: we are getting whiteouts incorrectly
# it should be:
# 1. extract a layer to temp.
# 2. find whiteouts in that layer.
# 3. remove whiteouts in the previous layer stack (e.g. the WIP rootfs)
# 4. finall copy the extracted layer over the WIP rootfs
# move extracted layer to target_dir
logger.debug(
'Moving extracted layer from: %(temp_target)r to: %(target_dir)r')
fileutils.copytree(temp_target, target_dir)
fileutils.delete(temp_target)
logger.debug(
'Merging extracted layers and applying AUFS whiteouts/deletes')
for top, dirs, files in fileutils.walk(target_dir):
# delete AUFS dirs and apply whiteout deletions
for dr in dirs[:]:
whiteable_dir = join(top, dr)
if dr.startswith(WHITEOUT_PREFIX):
# delete the .wh. dir...
dirs.remove(dr)
logger.debug('Deleting whiteout dir: %(whiteable_dir)r' %
locals())
fileutils.delete(whiteable_dir)
# ... and delete the corresponding dir it does "whiteout"
base_dir = dr[len(WHITEOUT_PREFIX):]
try:
dirs.remove(base_dir)
except ValueError:
# FIXME: should we really raise an exception here?
msg = ('Inconsistent layers: '
'missing directory to whiteout: %(base_dir)r' %
locals())
raise InconsistentLayersError(msg)
wdo = join(top, base_dir)
logger.debug('Deleting real dir: %(wdo)r' % locals())
fileutils.delete(wdo)
whiteouts.append(wdo)
# delete AUFS special dirs
elif dr.startswith(WHITEOUT_SPECIAL_DIR_PREFIX):
dirs.remove(dr)
logger.debug(
'Deleting AUFS special dir: %(whiteable_dir)r' %
locals())
fileutils.delete(whiteable_dir)
# delete AUFS files and apply whiteout deletions
all_files = set(files)
for fl in all_files:
whiteable_file = join(top, fl)
if fl.startswith(WHITEOUT_PREFIX):
# delete the .wh. marker file...
logger.debug('Deleting whiteout file: %(whiteable_file)r' %
locals())
fileutils.delete(whiteable_file)
# ... and delete the corresponding file it does "whiteout"
# e.g. logically delete
base_file = fl[len(WHITEOUT_PREFIX):]
wfo = join(top, base_file)
whiteouts.append(wfo)
if base_file in all_files:
logger.debug('Deleting real file: %(wfo)r' % locals())
fileutils.delete(wfo)
# delete AUFS special files
elif fl.startswith(WHITEOUT_SPECIAL_DIR_PREFIX):
logger.debug(
'Deleting AUFS special file: %(whiteable_file)r' %
locals())
fileutils.delete(whiteable_file)
whiteouts.append(whiteable_file)
return extract_errors, whiteouts
def parse_with_dparse(location):
"""
Return a list of DependentPackage built from a dparse-supported dependency
manifest such as requirements.txt, Conda manifest or Pipfile.lock files, or
return an empty list.
"""
is_dir = filetype.is_dir(location)
if is_dir:
return
file_name = fileutils.file_name(location)
dependency_type = get_dparse_dependency_type(file_name)
if not dependency_type:
return
with open(location) as f:
content = f.read()
dep_file = dparse.parse(content, file_type=dependency_type)
if not dep_file:
return []
dependent_packages = []
for dependency in dep_file.dependencies:
requirement = dependency.name
is_resolved = False
purl = PackageURL(type='pypi', name=dependency.name)
# note: dparse.dependencies.Dependency.specs comes from
# packaging.requirements.Requirement.specifier
# which in turn is a packaging.specifiers.SpecifierSet objects
# and a SpecifierSet._specs is a set of either:
# packaging.specifiers.Specifier or packaging.specifiers.LegacySpecifier
# and each of these have a .operator and .version property
# a packaging.specifiers.SpecifierSet
specifiers_set = dependency.specs
# a list of packaging.specifiers.Specifier
specifiers = specifiers_set._specs
if specifiers:
# SpecifierSet stringifies to comma-separated sorted Specifiers
requirement = str(specifiers_set)
# are we pinned e.g. resolved?
if len(specifiers) == 1:
specifier = list(specifiers)[0]
if specifier.operator in ('==', '==='):
is_resolved = True
purl = purl._replace(version=specifier.version)
dependent_packages.append(
models.DependentPackage(
purl=purl.to_string(),
# are we always this scope? what if we have requirements-dev.txt?
scope='install',
is_runtime=True,
is_optional=False,
is_resolved=is_resolved,
requirement=requirement))
return dependent_packages
def extractcode(ctx, input, verbose, quiet, shallow, replace_originals, ignore,
*args, **kwargs): # NOQA
"""extract archives and compressed files found in the <input> file or directory tree.
Archives found inside an extracted archive are extracted recursively.
Extraction for each archive is done in-place in a new directory named
'<archive file name>-extract' created side-by-side with an archive.
"""
abs_location = fileutils.as_posixpath(path.abspath(path.expanduser(input)))
def extract_event(item):
"""
Display an extract event.
"""
if quiet:
return ''
if not item:
return ''
source = item.source
if not isinstance(source, compat.unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
if verbose:
if item.done:
return ''
line = source and get_relative_path(path=source,
len_base_path=len_base_path,
base_is_dir=base_is_dir) or ''
else:
line = source and fileutils.file_name(source) or ''
if not isinstance(line, compat.unicode):
line = toascii(line, translit=True).decode('utf-8', 'replace')
return 'Extracting: %(line)s' % locals()
def display_extract_summary():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_result_with_errors:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = fileutils.as_posixpath(xev.source)
if not isinstance(source, compat.unicode):
source = toascii(source,
translit=True).decode('utf-8', 'replace')
source = get_relative_path(path=source,
len_base_path=len_base_path,
base_is_dir=base_is_dir)
for e in xev.errors:
echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(),
fg='red')
for warn in xev.warnings:
echo_stderr('WARNING extracting: %(source)s: %(warn)s' %
locals(),
fg='yellow')
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
echo_stderr('Extracting done.', fg=summary_color, reset=True)
# use for relative paths computation
len_base_path = len(abs_location)
base_is_dir = filetype.is_dir(abs_location)
extract_result_with_errors = []
unique_extract_events_with_errors = set()
has_extract_errors = False
extractibles = extract_archives(abs_location,
recurse=not shallow,
replace_originals=replace_originals,
ignore_pattern=ignore)
if not quiet:
echo_stderr('Extracting archives...', fg='green')
with cliutils.progressmanager(extractibles,
item_show_func=extract_event,
verbose=verbose) as extraction_events:
for xev in extraction_events:
if xev.done and (xev.warnings or xev.errors):
has_extract_errors = has_extract_errors or xev.errors
if repr(xev) not in unique_extract_events_with_errors:
extract_result_with_errors.append(xev)
unique_extract_events_with_errors.add(repr(xev))
display_extract_summary()
else:
for xev in extractibles:
if xev.done and (xev.warnings or xev.errors):
has_extract_errors = has_extract_errors or xev.errors
rc = 1 if has_extract_errors else 0
ctx.exit(rc)
def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs): # @ReservedAssignment
"""extract archives and compressed files found in the <input> file or directory tree.
Use this command before scanning proper as an <input> preparation step.
Archives found inside an extracted archive are extracted recursively.
Extraction is done in-place in a directory named '-extract' side-by-side with an archive.
"""
abs_location = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input)))
def extract_event(item):
"""
Display an extract event.
"""
if quiet:
return ''
if not item:
return ''
source = item.source
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
if verbose:
if item.done:
return ''
line = source and utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or ''
else:
line = source and fileutils.file_name(source) or ''
if not isinstance(line, unicode):
line = toascii(line, translit=True).decode('utf-8', 'replace')
return 'Extracting: %(line)s' % locals()
def display_extract_summary():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = fileutils.as_posixpath(xev.source)
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
source = utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir)
for e in xev.errors:
echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red')
for warn in xev.warnings:
echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow')
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
echo_stderr('Extracting done.', fg=summary_color, reset=True)
# use for relative paths computation
len_base_path = len(abs_location)
base_is_dir = filetype.is_dir(abs_location)
extract_results = []
has_extract_errors = False
if not quiet:
echo_stderr('Extracting archives...', fg='green')
with utils.progressmanager(extract_archives(abs_location, recurse=not shallow), item_show_func=extract_event,
verbose=verbose, quiet=quiet) as extraction_events:
for xev in extraction_events:
if xev.done and (xev.warnings or xev.errors):
has_extract_errors = has_extract_errors or xev.errors
extract_results.append(xev)
if not quiet:
display_extract_summary()
rc = 1 if has_extract_errors else 0
ctx.exit(rc)
def extractcode(ctx, input, verbose, quiet, shallow, *args, **kwargs): # @ReservedAssignment
"""extract archives and compressed files found in the <input> file or directory tree.
Use this command before scanning proper as an <input> preparation step.
Archives found inside an extracted archive are extracted recursively.
Extraction is done in-place in a directory named '-extract' side-by-side with an archive.
"""
abs_location = fileutils.as_posixpath(os.path.abspath(os.path.expanduser(input)))
def extract_event(item):
"""
Display an extract event.
"""
if quiet:
return ''
if not item:
return ''
source = item.source
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
if verbose:
if item.done:
return ''
line = source and utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir) or ''
else:
line = source and fileutils.file_name(source) or ''
if not isinstance(line, unicode):
line = toascii(line, translit=True).decode('utf-8', 'replace')
return 'Extracting: %(line)s' % locals()
def display_extract_summary():
"""
Display a summary of warnings and errors if any.
"""
has_warnings = False
has_errors = False
summary = []
for xev in extract_results:
has_errors = has_errors or bool(xev.errors)
has_warnings = has_warnings or bool(xev.warnings)
source = fileutils.as_posixpath(xev.source)
if not isinstance(source, unicode):
source = toascii(source, translit=True).decode('utf-8', 'replace')
source = utils.get_relative_path(path=source, len_base_path=len_base_path, base_is_dir=base_is_dir)
for e in xev.errors:
echo_stderr('ERROR extracting: %(source)s: %(e)s' % locals(), fg='red')
for warn in xev.warnings:
echo_stderr('WARNING extracting: %(source)s: %(warn)s' % locals(), fg='yellow')
summary_color = 'green'
if has_warnings:
summary_color = 'yellow'
if has_errors:
summary_color = 'red'
echo_stderr('Extracting done.', fg=summary_color, reset=True)
# use for relative paths computation
len_base_path = len(abs_location)
base_is_dir = filetype.is_dir(abs_location)
extract_results = []
has_extract_errors = False
if not quiet:
echo_stderr('Extracting archives...', fg='green')
with utils.progressmanager(extract_archives(abs_location, recurse=not shallow), item_show_func=extract_event,
verbose=verbose, quiet=quiet) as extraction_events:
for xev in extraction_events:
if xev.done and (xev.warnings or xev.errors):
has_extract_errors = has_extract_errors or xev.errors
extract_results.append(xev)
if not quiet:
display_extract_summary()
rc = 1 if has_extract_errors else 0
ctx.exit(rc)
def is_node_modules(location):
return (filetype.is_dir(location)
and fileutils.file_name(location).lower() == 'node_modules')
