def save(self, desc=None):
"""
save statistics for future reference
Will save any statistics which should be persisted for future
considerations. This is to help render a "complete" report of statistics
when re-running releng-tool with packages which may already been
completed.
Args:
desc (optional): description of this save event (for logging)
"""
if not ensure_dir_exists(self.out_dir):
verbose('unable to generate output directory for statistics')
return None
if desc:
desc = ' ({})'.format(desc)
else:
desc = ''
try:
with open(self.dat_file, 'wb') as f:
pickle.dump(self.data, f, protocol=2) # 2 for py2/py3 support
debug('saved statistics' + desc)
except IOError:
verbose('failed to save statistics' + desc)
def process(self, pkg):
"""
process a provided package
This request will process a package through the various stages (if these
stages are applicable to the current run state and have yet been
executed from a previous run). A package-specific script environment
will be prepared and a package will go through the process of:
- Extraction
- Patching
- License management (if needed)
- Bootstrapping
- Configuration
- Building
- Installing
- Post-processing
Args:
pkg: the package to process
Returns:
returns whether or not the pipeline should continue processing
"""
gaction = self.opts.gbl_action
paction = self.opts.pkg_action
target = self.opts.target_action
# skip if generating license information and no license
# files exist for this package
if gaction == GlobalAction.LICENSES and not pkg.license_files:
return True
# extracting
flag = pkg._ff_extract
if check_file_flag(flag) == FileFlag.NO_EXIST:
self.engine.stats.track_duration_start(pkg.name, 'extract')
# none/local-vcs-type packages do not need to fetch
if pkg.vcs_type in (VcsType.LOCAL, VcsType.NONE):
pass
elif not extract_stage(self.engine, pkg):
raise RelengToolExtractionStageFailure
# now that the extraction stage has (most likely)
# created a build directory, ensure the output directory
# exists as well (for file flags and other content)
if not ensure_dir_exists(pkg.build_output_dir):
raise RelengToolExtractionStageFailure
self.engine.stats.track_duration_end(pkg.name, 'extract')
if process_file_flag(True, flag) != FileFlag.CONFIGURED:
return False
if gaction == GlobalAction.EXTRACT:
return True
if paction == PkgAction.EXTRACT and pkg.name == target:
return False
# process the package data with a package-specific environment
with self._stage_env(pkg) as pkg_env:
return self._process_data(pkg, pkg_env)
def _dummy_pkg(self, container, pkg=None):
uid = uuid.uuid4().hex
if pkg:
container = os.path.join(container, pkg)
assert ensure_dir_exists(container)
metadata = os.path.join(container, 'metadata')
with open(metadata, 'w') as f:
f.write(uid)
return uid
def _fetch_submodule(opts, name, cache_dir, revision, site):
"""
fetch a submodule into a provided cache/bar repository
Fetches an individual submodule into the provided cache directory. The
origin of the submodule is provided via the ``site`` argument. A revision,
if provided, can be used to help verify the target revision desired for a
submodule; however, it is not required (e.g. when a repository does not set
an explicit submodule revision).
Args:
opts: fetch options
name: the name of the submodule (for state messages)
cache_dir: the cache/bare repository to fetch into
revision: the revision (branch, tag, hash) to fetch
site: the site to fetch the submodule from
Returns:
``True`` if the submodule has been fetched; ``False`` otherwise
"""
git_dir = '--git-dir=' + cache_dir
# check if we have the target revision cached; if so, submodule is ready
if os.path.isdir(cache_dir) and not opts.ignore_cache:
if not revision:
return _sync_git_origin(cache_dir, site)
if revision_exists(git_dir, revision) in REVISION_EXISTS:
return _sync_git_origin(cache_dir, site)
log('processing submodule (package: {}) {}...', opts.name, name)
sys.stdout.flush()
# validate any cache directory (if one exists)
has_cache, bad_validation = _validate_cache(cache_dir)
if bad_validation:
return None
# if we have no cache for this repository, build one
if not has_cache:
if not ensure_dir_exists(cache_dir):
return False
if not _create_bare_git_repo(cache_dir):
return False
# ensure configuration is properly synchronized
if not _sync_git_origin(cache_dir, site):
return False
# fetch sources for this submodule
desc = 'submodule ({}): {}'.format(opts.name, name)
return _fetch_srcs(opts, cache_dir, revision, desc=desc)
def generate(self):
"""
generate a final report of statistics
To be invoked at the end of a releng-tool process, this call will
generate reports/etc. for any tracked statistics information based on
the current and previous invoked executions (if any).
"""
if not ensure_dir_exists(self.out_dir):
verbose('unable to generate output directory for statistics')
return None
self._generate_duration()
def fetch(opts):
"""
support fetching from svn sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
revision = opts.revision
site = opts.site
work_dir = opts.work_dir
if not SVN.exists():
err('unable to fetch package; svn is not installed')
return None
note('fetching {}...'.format(name))
sys.stdout.flush()
log('checking out sources')
if not SVN.execute(['checkout', '-r', revision, site, work_dir],
cwd=work_dir):
err('unable to checkout module')
return None
log('caching sources')
def svn_exclude(file):
if file.endswith('.svn'):
return True
return False
# ensure cache file's directory exists
cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir))
if not ensure_dir_exists(cache_dir):
return None
with tarfile.open(cache_file, 'w:gz') as tar:
tar.add(work_dir, arcname=name, exclude=svn_exclude)
return cache_file
def test_utilio_ensuredirexists(self):
with prepare_workdir() as work_dir:
result = ensure_dir_exists(work_dir)
self.assertTrue(result)
new_dir = os.path.join(work_dir, 'test1')
self.assertFalse(os.path.exists(new_dir))
result = ensure_dir_exists(new_dir)
self.assertTrue(result)
self.assertTrue(os.path.exists(new_dir))
new_file = os.path.join(work_dir, 'test2')
with open(new_file, 'ab'):
pass
self.assertTrue(os.path.isfile(new_file))
result = ensure_dir_exists(new_file)
self.assertFalse(result)
self.assertTrue(os.path.isfile(new_file))
with self.assertRaises(SystemExit):
ensure_dir_exists(new_file, critical=True)
self.assertTrue(os.path.isfile(new_file))
def _save_dvcs_cache(self):
"""
save dvcs cache information
Will save any DVCS cache information which future runs of releng-tool
can be used to hint where package cache data is stored.
"""
if not self._dvcs_cache_enabled:
return
if not ensure_dir_exists(self.opts.cache_dir):
verbose('unable to generate output directory for dvcs cache')
return
try:
with open(self._dvcs_cache_fname, 'wb') as f:
pickle.dump(self._dvcs_cache, f,
protocol=2) # 2 for py2/py3 support
debug('saved dvcs cache')
except IOError:
verbose('failed to save dvcs cache')
def _copy_tree(src_folder, dst_folder, quiet=False, critical=True):
if not ensure_dir_exists(dst_folder, quiet=quiet, critical=critical):
return False
for entry in os.listdir(src_folder):
src = os.path.join(src_folder, entry)
dst = os.path.join(dst_folder, entry)
if os.path.islink(src):
target = os.readlink(src)
if os.path.islink(dst) or os.path.isfile(dst):
path_remove(dst)
os.symlink(target, dst)
_copystat(src, dst)
elif os.path.isdir(src):
_copy_tree(src, dst, quiet=quiet, critical=critical)
else:
_copyfile(src, dst)
_copystat(src, dst)
_copystat(src_folder, dst_folder)
return True
def _stage_license(self, pkg):
"""
process license files for a specific package processing
If a package contains one or more files containing licenses information,
this information will be populated in the package's license folder.
Args:
pkg: the package being processed
Returns:
``True`` if the license information was copied; ``False`` if these
license information could not be copied
"""
# skip if package has no license files
if not pkg.license_files:
if pkg.license and not pkg.is_internal and not pkg.no_extraction:
warn('package defines no license files: ' + pkg.name)
return True
# ensure package-specific license directory exists
pkg_license_dir = os.path.join(self.opts.license_dir, pkg.nv)
if not ensure_dir_exists(pkg_license_dir):
return False
# copy over each license files
for file in pkg.license_files:
src = os.path.join(pkg.build_dir, file)
dst = os.path.join(pkg_license_dir, file)
if not path_copy(src, dst, critical=False):
err('unable to copy license information: ' + pkg.name)
return False
return True
def initialize_sample(opts):
"""
initialize a sample project
Generates a sample provided in the root directory to help new users or new
project get started.
Args:
opts: options for this run
Returns:
``True`` if the sample project could be initialized; ``False`` if an
issue has occurred generating the sample project
"""
root_dir = opts.root_dir
if not ensure_dir_exists(root_dir):
return False
if os.listdir(root_dir):
err('unable to initialize sample project is non-empty directory')
return False
sample_dir = os.path.join(root_dir, 'package', 'sample')
success = True
if ensure_dir_exists(sample_dir):
# sample project
sample_defs = os.path.join(root_dir, 'package', 'sample', 'sample')
try:
with open(sample_defs, 'w') as f:
f.write('''\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
SAMPLE_DEPENDENCIES = []
SAMPLE_LICENSE = ['<license name>']
SAMPLE_LICENSE_FILES = ['<license file>']
SAMPLE_SITE = '<location for sources>'
SAMPLE_TYPE = '<package-type>'
SAMPLE_VERSION = '<package-version>'
''')
verbose('written sample file')
except IOError as e:
err('unable to generate a sample file')
verbose(str(e))
success = False
else:
success = False
# .gitignore
try:
project_gitignore = os.path.join(root_dir,
'.gitignore') # (assumption)
with open(project_gitignore, 'w') as f:
f.write('''\
# releng-tool
/cache/
/dl/
/output/
.releng-flag-*
''')
verbose('written .gitignore file')
except IOError as e:
err('unable to generate a .gitignore file')
verbose(str(e))
success = False
# releng project
try:
project_defs = os.path.join(root_dir, 'releng')
with open(project_defs, 'w') as f:
f.write('''\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
packages = [
'sample',
]
''')
verbose('written releng file')
except IOError as e:
err('unable to generate a releng file')
verbose(str(e))
success = False
if success:
log('initialized empty releng-tool project')
else:
warn('partially initialized a releng-tool project')
return success
def path_copy(src, dst, quiet=False, critical=True, dst_dir=None):
"""
copy a file or directory into a target file or directory
This call will attempt to copy a provided file or directory, defined by
``src`` into a destination file or directory defined by ``dst``. If ``src``
is a file, then ``dst`` is considered to be a file or directory; if ``src``
is a directory, ``dst`` is considered a target directory. If a target
directory or target file's directory does not exist, it will be
automatically created. In the event that a file or directory could not be
copied, an error message will be output to standard error (unless ``quiet``
is set to ``True``). If ``critical`` is set to ``True`` and the specified
file/directory could not be copied for any reason, this call will issue a
system exit (``SystemExit``).
An example when using in the context of script helpers is as follows:
.. code-block:: python
# (stage)
# my-file
releng_copy('my-file', 'my-file2')
# (stage)
# my-file
# my-file2
releng_copy('my-file', 'my-directory/')
# (stage)
# my-directory/my-file
# my-file
# my-file2
releng_copy('my-directory/', 'my-directory2/')
# (stage)
# my-directory/my-file
# my-directory2/my-file
# my-file
# my-file2
Args:
src: the source directory or file
dst: the destination directory or file\\* (\\*if ``src`` is a file)
quiet (optional): whether or not to suppress output
critical (optional): whether or not to stop execution on failure
dst_dir (optional): force hint that the destination is a directory
Returns:
``True`` if the copy has completed with no error; ``False`` if the copy
has failed
Raises:
SystemExit: if the copy operation fails with ``critical=True``
"""
success = False
errmsg = None
try:
if os.path.isfile(src):
attempt_copy = True
if dst_dir:
base_dir = dst
else:
base_dir = os.path.dirname(dst)
if not os.path.isdir(base_dir):
attempt_copy = ensure_dir_exists(base_dir, quiet=quiet)
if attempt_copy:
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
if os.path.islink(src):
target = os.readlink(src)
if os.path.islink(dst) or os.path.isfile(dst):
path_remove(dst)
os.symlink(target, dst)
else:
_copyfile(src, dst)
_copystat(src, dst)
success = True
elif os.path.exists(src):
if src == dst:
errmsg = "'{!s}' and '{!s}' " \
"are the same folder".format(src, dst)
elif _copy_tree(src, dst, quiet=quiet, critical=critical):
success = True
else:
errmsg = 'source does not exist: {}'.format(src)
except (IOError, ShutilError) as e:
errmsg = str(e)
if not quiet and errmsg:
err('unable to copy source contents to target location\n'
' {}', errmsg)
if not success and critical:
sys.exit(-1)
return success
def fetch(opts):
"""
support fetching from mercurial sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_dir = opts.cache_dir
name = opts.name
revision = opts.revision
site = opts.site
if not HG.exists():
err('unable to fetch package; hg (mercurial) is not installed')
return None
hg_dir = ['--repository', cache_dir]
# check if we have the target revision; if so, full stop
if os.path.isdir(cache_dir) and not opts.ignore_cache:
if HG.execute(hg_dir + ['--quiet', 'log', '--rev', revision],
cwd=cache_dir,
quiet=True):
return cache_dir
note('fetching {}...', name)
sys.stdout.flush()
# if we have no cache for this repository, build one
if not os.path.isdir(cache_dir):
if not ensure_dir_exists(cache_dir):
return None
if not HG.execute([
'--noninteractive', '--verbose', 'clone', '--noupdate', site,
cache_dir
],
cwd=cache_dir):
err('unable to clone mercurial repository')
return None
log('fetching most recent sources')
if not HG.execute(hg_dir + ['--noninteractive', '--verbose', 'pull'],
cwd=cache_dir):
err('unable to fetch from remote repository')
return None
log('verifying target revision exists')
if not HG.execute(hg_dir + ['--quiet', 'log', '--rev', revision],
cwd=cache_dir,
quiet=True):
err(
'unable to find matching revision in repository: {}\n'
' (revision: {})', name, revision)
return None
return cache_dir
def fetch(opts):
"""
support fetching from git sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_dir = opts.cache_dir
name = opts.name
revision = opts.revision
if not GIT.exists():
err('unable to fetch package; git is not installed')
return None
git_dir = '--git-dir=' + cache_dir
# check if we have the target revision cached; if so, package is ready
if os.path.isdir(cache_dir) and not opts.ignore_cache:
erv = revision_exists(git_dir, revision)
if erv in REVISION_EXISTS:
# ensure configuration is properly synchronized
if not _sync_git_configuration(opts):
return None
# if no explicit ignore-cache request and if the revision is a
# branch, force ignore-cache on and allow fetching to proceed
if opts.ignore_cache is None and erv == GitExistsType.EXISTS_BRANCH:
opts.ignore_cache = True
# return cache dir if not verifying or verification succeeds
elif not opts._git_verify_revision or _verify_revision(
git_dir, revision, quiet=True):
return cache_dir
note('fetching {}...', name)
sys.stdout.flush()
# validate any cache directory (if one exists)
has_cache, bad_validation = _validate_cache(cache_dir)
if bad_validation:
return None
# if we have no cache for this repository, build one
if not has_cache:
if not ensure_dir_exists(cache_dir):
return None
if not _create_bare_git_repo(cache_dir):
return None
# ensure configuration is properly synchronized
if not _sync_git_configuration(opts):
return None
# fetch sources for this repository
if not _fetch_srcs(opts, cache_dir, revision, refspecs=opts._git_refspecs):
return None
# verify revision (if configured to check it)
if opts._git_verify_revision:
if not _verify_revision(git_dir, revision):
err(
'''\
failed to validate git revision
Package has been configured to require the verification of the GPG signature
for the target revision. The verification has failed. Ensure that the revision
is signed and that the package's public key has been registered in the system.
Package: {}
Revision: {}''', name, revision)
return None
# fetch submodules (if configured to do so)
if opts._git_submodules:
if not _fetch_submodules(opts, cache_dir, revision):
return None
return cache_dir
def fetch(opts):
"""
support fetching from bzr sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
revision = opts.revision
site = opts.site
if not BZR.exists():
err('unable to fetch package; bzr is not installed')
return None
note('fetching {}...', name)
sys.stdout.flush()
cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir))
if not ensure_dir_exists(cache_dir):
return None
export_opts = [
'export',
cache_file,
site,
'--format=tgz',
'--root=' + name,
'--revision=' + revision,
]
# some environments may have issue export bzr sources due to certificate
# issues; this quirk allows injecting certifi-provided certificates for
# all bzr exports
if 'releng.bzr.certifi' in opts._quirks:
global CERTIFI_MISSING_WARNED
if certifi:
verbose('performing bzr fetch with certifi certificates')
pkg_site = certifi.where()
export_opts.append('-Ossl.ca_certs=' + pkg_site)
elif not CERTIFI_MISSING_WARNED:
CERTIFI_MISSING_WARNED = True
warn('''\
unable to perform bzr fetch with certifi certificates
A quirk has been enabled to export bzr images using certifi
certificates; however, certifi is not installed on this system.
''')
log('exporting sources')
if not BZR.execute(export_opts, poll=True):
err('unable to export module')
return None
return cache_file
def fetch(opts):
"""
support fetching from cvs sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
revision = opts.revision
site = opts.site
work_dir = opts.work_dir
cache_basename = os.path.basename(cache_file)
cache_stem, __ = interpret_stem_extension(cache_basename)
if not CVS.exists():
err('unable to fetch package; cvs is not installed')
return None
note('fetching {}...', name)
sys.stdout.flush()
try:
cvsroot, module = site.rsplit(' ', 1)
except ValueError:
err('''\
improper cvs site defined
The provided CVS site does not define both the CVSROOT as well as the target
module to checkout. For example:
:pserver:[email protected]:/var/lib/cvsroot mymodule
Site: {}''', site)
return None
log('checking out sources')
if not CVS.execute(['-d', cvsroot, 'checkout', '-d', cache_stem,
'-r', revision, module], cwd=work_dir):
err('unable to checkout module')
return None
cvs_module_dir = os.path.join(work_dir, cache_stem)
if not os.path.exists(cvs_module_dir):
err('no sources available for the provided revision')
return None
log('caching sources')
def cvs_filter(info):
if info.name.endswith('CVS'):
return None
return info
cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir))
if not ensure_dir_exists(cache_dir):
return None
with tarfile.open(cache_file, 'w:gz') as tar:
tar.add(cvs_module_dir, arcname=cache_stem, filter=cvs_filter)
return cache_file
def extract(opts):
"""
support extraction of an archive into a build directory
With provided extraction options (``RelengExtractOptions``), the extraction
stage will be processed. The archive's extension will be used in attempt to
finding a matching tool/implementation which can be used to extract the
contents of the file. In the event that the method of extraction cannot be
determined, it will be assumed that the file is in fact not extractable.
Files which are not extracted are just copied into the build directly (e.g.
single resource files).
Args:
opts: the extraction options
Returns:
``True`` if the extraction stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
strip_count = opts.strip_count
work_dir = opts.work_dir
cache_basename = os.path.basename(cache_file)
__, cache_ext = interpret_stem_extension(cache_basename)
is_extractable = False
if cache_ext:
cache_ext = cache_ext.lower()
# if the user defines a tool override for this extension type, use
# whatever the user wants to use (passing the file and directory to
# extract to)
extract_override = getattr(opts, '_extract_override', None)
if extract_override and cache_ext in extract_override:
is_extractable = True
tool_cmd = extract_override[cache_ext].format(file=cache_file,
dir=work_dir)
if not execute(tool_cmd.split(), cwd=work_dir, critical=False):
err('unable to extract with tool override\n'
' (command: {})', tool_cmd)
return None
# attempt to extract the (compressed) tar archive with the host's
# tar tool; if it does not exist, we'll fallback to using python's
# internal implementation (tarfile)
elif cache_ext.startswith(TAR_SUPPORTED):
is_extractable = True
# before attempting to use an external tar command, only allow
# using it if the `force-local` option is available whenever a
# colon character is provided, to prevent tar from assuming the
# path is a remote target
needs_force_local = False
if ':' in cache_file:
needs_force_local = True
has_extracted = False
if TAR.exists() and (TAR.force_local or not needs_force_local):
tar_args = [
'--extract',
'--file=' + cache_file,
'--strip-components={}'.format(strip_count),
'--verbose',
]
if needs_force_local:
tar_args.append('--force-local')
if TAR.execute(tar_args, cwd=work_dir):
has_extracted = True
else:
warn('unable to extract archive with host tar; '
'will use fallback')
if not has_extracted:
try:
def tar_extract(members, strip_count):
for member in members:
# strip members from package defined count
if strip_count > 0:
np = os.path.normpath(member.name)
parts = np.split(os.path.sep, strip_count)
if len(parts) <= strip_count:
continue
member.name = parts[-1]
# notify the user of the target member to extract
print(member.name)
yield member
with tarfile.open(cache_file, 'r') as tar:
tar.extractall(path=work_dir,
members=tar_extract(tar, strip_count))
except Exception as e:
err(
'unable to extract tar file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
# extract a zip-extension cache file using python's internal
# implementation (zipfile)
elif cache_ext == 'zip':
is_extractable = True
try:
with ZipFile(cache_file, 'r') as zip_:
for member in zip_.namelist():
# strip members from package defined count
member_s = member
if strip_count > 0:
np = os.path.normpath(member_s)
parts = np.split(os.path.sep, strip_count)
if len(parts) <= strip_count:
continue
member_s = parts[-1]
dest = os.path.join(work_dir, member_s)
# notify the user of the target member to extract
print(member)
# if this is a directory entry, ensure the directory
# exists for the destination
if not os.path.basename(member):
ensure_dir_exists(dest)
else:
# always ensure the container directory for a file
# exists before attempting to extract a member into
# it, as not all processed zip files may process
# a directory entry (to be created) ahead of time
ensure_dir_exists(os.path.dirname(dest))
with zip_.open(member) as s, open(dest, 'wb') as f:
shutil.copyfileobj(s, f)
except Exception as e:
err(
'unable to extract zip file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
if not is_extractable:
debug('file not considered extractable: ' + cache_file)
try:
shutil.copy2(cache_file, work_dir)
except IOError as e:
err(
'unable to copy over cache file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
return True
def stage(engine, pkg, ignore_cache, extra_opts):
"""
handles the fetching stage for a package
With a provided engine and package instance, the fetching stage will be
processed.
Args:
engine: the engine
pkg: the package being fetched
ignore_cache: always attempt to ignore the cache
extra_opts: extra options for the fetch operation (if applicable)
Returns:
``True`` if the fetching stage is completed; ``False`` otherwise
"""
assert pkg.vcs_type
name = pkg.name
debug('process fetch stage: ' + name)
# packages flagged for local sources requires to be already checked out
if pkg.local_srcs:
if os.path.isdir(pkg.build_dir):
return True
err(
'''\
missing local sources for internal package: {0}
The active configuration is flagged for 'local sources' mode; however, an
internal package cannot be found in the local system. Before continuing, ensure
you have checked out all internal packages on your local system (or, disable the
local sources option to use the default process).
Package: {0}
Expected Path: {1}''', name, pkg.build_dir)
return False
# if the vcs-type is archive-based, flag that hash checks are needed
perform_file_asc_check = False
perform_file_hash_check = False
if pkg.vcs_type == VcsType.URL:
perform_file_asc_check = os.path.exists(pkg.asc_file)
perform_file_hash_check = True
fetch_opts = RelengFetchOptions()
replicate_package_attribs(fetch_opts, pkg)
fetch_opts.cache_dir = pkg.cache_dir
fetch_opts.ext = pkg.ext_modifiers
fetch_opts.extra_opts = extra_opts
fetch_opts.ignore_cache = ignore_cache
fetch_opts.name = name
fetch_opts.revision = pkg.revision
fetch_opts.site = pkg.site
fetch_opts.version = pkg.version
fetch_opts._mirror = False
fetch_opts._quirks = engine.opts.quirks
fetch_opts._urlopen_context = engine.opts.urlopen_context
cache_filename = os.path.basename(pkg.cache_file)
out_dir = engine.opts.out_dir
with generate_temp_dir(out_dir) as work_dir, \
generate_temp_dir(out_dir) as interim_cache_dir:
with interim_working_dir(work_dir):
interim_cache_file = os.path.join(interim_cache_dir,
cache_filename)
fetch_opts.cache_file = interim_cache_file
fetch_opts.work_dir = work_dir
# check if file caching should be ignored
#
# In special cases, a developer may configure a project to have a
# fetched source not to cache. For example, pulling from a branch of
# a VCS source will make a cache file from the branch and will
# remain until manually removed from a cache file. A user may wish
# to re-build the local cache file after cleaning their project.
# While the releng-tool framework separates fetching/extraction into
# two parts, ignoring cached assets can be partially achieved by
# just removing any detected cache file if a project is configured
# to ignore a cache file.
if engine.opts.devmode and pkg.devmode_ignore_cache is not None:
fetch_opts.ignore_cache = pkg.devmode_ignore_cache
if pkg.devmode_ignore_cache and os.path.exists(pkg.cache_file):
verbose('removing cache file (per configuration): ' + name)
if not path_remove(pkg.cache_file):
return False
# remove cache file if there is a force request to ignore the cache
elif engine.opts.force and ignore_cache:
if os.path.exists(pkg.cache_file):
verbose('removing cache file (forced): ' + name)
if not path_remove(pkg.cache_file):
return False
# force explicit ignore cache (to off) when not in development mode
elif not engine.opts.devmode and ignore_cache is None:
fetch_opts.ignore_cache = False
if os.path.exists(pkg.cache_file):
rv = None
if perform_file_hash_check:
hr = verify_hashes(pkg.hash_file,
pkg.cache_file,
relaxed=True)
if hr == HashResult.VERIFIED:
rv = True
elif hr == HashResult.BAD_PATH:
if not perform_file_asc_check and not pkg.is_internal:
warn('missing hash file for package: ' + name)
rv = True # no hash file to compare with; assuming ok
elif hr == HashResult.EMPTY:
if not pkg.is_internal:
warn('hash file for package is empty: ' + name)
rv = True # empty hash file; assuming ok
elif hr == HashResult.MISMATCH:
if not path_remove(pkg.cache_file):
rv = False
elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED):
rv = False
elif hr == HashResult.MISSING_ARCHIVE:
if not perform_file_asc_check:
err(
'''\
missing archive hash for verification
The hash file for this package does not have an entry for the cache file to be
verified. Ensure the hash file defines an entry for the expected cache file:
Hash File: {}
File: {}''', pkg.hash_file, cache_filename)
rv = False
else:
err(
'invalid fetch operation (internal error; '
'hash-check failure: {})', hr)
rv = False
else:
rv = True
if rv is not False and perform_file_asc_check and \
os.path.exists(pkg.cache_file):
if GPG.validate(pkg.asc_file, pkg.cache_file):
rv = True
else:
if not path_remove(pkg.cache_file):
err(
'''\
failed to validate against ascii-armor
Validation of a package resource failed to verify against a provided ASCII-armor
file. Ensure that the package's public key has been registered into gpg.
ASC File: {}
File: {}''', pkg.asc_file, cache_filename)
rv = False
else:
rv = None
if rv is not None:
if ignore_cache:
verbose('ignoring cache not supported for package: {}',
name)
return rv
# find fetching method for the target vcs-type
fetcher = None
if pkg.vcs_type in engine.registry.fetch_types:
def _(opts):
return engine.registry.fetch_types[pkg.vcs_type].fetch(
pkg.vcs_type, opts)
fetcher = _
elif pkg.vcs_type == VcsType.BZR:
fetcher = fetch_bzr
elif pkg.vcs_type == VcsType.CVS:
fetcher = fetch_cvs
elif pkg.vcs_type == VcsType.GIT:
fetcher = fetch_git
elif pkg.vcs_type == VcsType.HG:
fetcher = fetch_mercurial
elif pkg.vcs_type == VcsType.RSYNC:
fetcher = fetch_rsync
elif pkg.vcs_type == VcsType.SCP:
fetcher = fetch_scp
elif pkg.vcs_type == VcsType.SVN:
fetcher = fetch_svn
elif pkg.vcs_type == VcsType.URL:
fetcher = fetch_url
if not fetcher:
err('fetch type is not implemented: {}', pkg.vcs_type)
return False
# if this is url-type location, attempt to search on the mirror
# first (if configured)
fetched = None
if engine.opts.url_mirror and pkg.vcs_type == VcsType.URL:
original_site = fetch_opts.site
new_site = engine.opts.url_mirror + cache_filename
if original_site != new_site:
fetch_opts._mirror = True
fetch_opts.site = new_site
fetched = fetcher(fetch_opts)
fetch_opts.site = original_site
fetch_opts._mirror = False
# perform the fetch request (if not already fetched)
if not fetched:
fetched = fetcher(fetch_opts)
if not fetched:
return False
# if the fetch type has populated the package's cache directory
# directly, we are done
if fetched == pkg.cache_dir:
pass
# if the fetch type has returned a file, the file needs to be hash
# checked and then be moved into the download cache
elif fetched == interim_cache_file:
if perform_file_hash_check:
hr = verify_hashes(pkg.hash_file, fetched)
if hr == HashResult.VERIFIED:
pass
elif hr == HashResult.BAD_PATH:
if not perform_file_asc_check and not pkg.is_internal:
warn('missing hash file for package: ' + name)
elif hr == HashResult.EMPTY:
if not pkg.is_internal:
warn('hash file for package is empty: ' + name)
elif hr == HashResult.MISMATCH:
return False
elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED):
return False
elif hr == HashResult.MISSING_ARCHIVE:
if not perform_file_asc_check:
err(
'''\
missing archive hash for verification
The hash file for this package does not have an entry for the cache file to be
verified. Ensure the hash file defines an entry for the expected cache file:
Hash File: {}
File: {}''', pkg.hash_file, cache_filename)
return False
else:
err(
'invalid fetch operation (internal error; '
'hash-check failure: {})', hr)
return False
if perform_file_asc_check:
if not GPG.validate(pkg.asc_file, interim_cache_file):
err(
'''\
failed to validate against ascii-armor
Validation of a package resource failed to verify against a provided ASCII-armor
file. Ensure that the package's public key has been registered into gpg.
ASC File: {}
File: {}''', pkg.asc_file, cache_filename)
return False
debug('fetch successful; moving cache file')
# ensure the cache container/directory exists
cache_dir = os.path.dirname(pkg.cache_file)
if not ensure_dir_exists(cache_dir):
return False
try:
shutil.move(interim_cache_file, pkg.cache_file)
except shutil.Error:
err(
'invalid fetch operation (internal error; fetch mode '
'"{}" has provided a missing cache file)',
pkg.vcs_type)
return False
else:
err(
'invalid fetch operation (internal error; fetch mode "{}" '
'has returned an unsupported value)', pkg.vcs_type)
return False
return True