def run(self, result=None):
"""
run the test
Run the test, collecting the result into the TestResult object passed as
result. See `unittest.TestCase.run()` for more details.
Args:
result (optional): the test result to populate
"""
with generate_temp_dir() as repo_dir, interim_working_dir(repo_dir):
self.repo_dir = repo_dir
self.prepare_repo_dir(repo_dir)
with prepare_testenv(template=self.tool_template()) as engine:
opts = engine.opts
opts.gbl_action = self.prepare_global_action()
self.cache_dir = os.path.join(opts.cache_dir, PKG_NAME)
self.defconfig = os.path.join(opts.root_dir, PKG_DEFINITION)
self.engine = engine
self.defconfig_add('SITE', self.repo_dir)
self.prepare_defconfig(self.defconfig)
super(TestSiteToolBase, self).run(result)
def _git(self, workdir, *args):
with interim_working_dir(workdir):
out = []
if not execute(['git'] + list(args), capture=out, critical=False):
print(['git'] + list(args))
print('\n'.join(out))
assert False, 'failed to issue git command'
return '\n'.join(out)
def prepare_testenv(config=None, template=None, args=None):
"""
prepare an engine-ready environment for a test
This utility method is used to provide an `RelengEngine` instance ready for
execution on an interim working directory.
Args:
config (optional): dictionary of options to mock for arguments
template (optional): the folder holding a template project to copy into
the prepared working directory
args (optional): additional arguments to add to the "forwarded options"
Yields:
the engine
"""
class MockArgs(object):
def __getattr__(self, name):
return self.name if name in self.__dict__ else None
if config is None:
config = {}
with generate_temp_dir() as work_dir, interim_working_dir(work_dir):
# force root directory to temporary directory; or configure all working
# content based off the generated temporary directory
if 'root_dir' not in config:
config['root_dir'] = work_dir
else:
if 'cache_dir' not in config:
config['cache_dir'] = os.path.join(work_dir, 'cache')
if 'dl_dir' not in config:
config['dl_dir'] = os.path.join(work_dir, 'dl')
if 'out_dir' not in config:
config['out_dir'] = os.path.join(work_dir, 'out')
if template:
copy_template(template, work_dir)
# build arguments instance
test_args = MockArgs()
for k, v in config.items():
setattr(test_args, k, v)
# prepare engine options and build an engine instance
opts = RelengEngineOptions(args=test_args, forward_args=args)
engine = RelengEngine(opts)
yield engine
def stage(engine, pkg, script_env):
"""
handles the post-processing stage for a package
With a provided engine and package instance, the post-processing stage will
be processed. This stage is typically not advertised and is for advanced
cases where a developer wishes to manipulate their build environment after
package has completed each of its phases.
Args:
engine: the engine
pkg: the package being built
script_env: script environment information
Returns:
``True`` if the post-processing stage is completed; ``False`` otherwise
"""
verbose('post-processing {}...', pkg.name)
sys.stdout.flush()
post_script_filename = '{}-{}'.format(pkg.name, POST_SCRIPT)
post_script = os.path.join(pkg.def_dir, post_script_filename)
post_script, post_script_exists = opt_file(post_script)
if not post_script_exists:
return True
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
with interim_working_dir(build_dir):
if not run_script(post_script, script_env, subject='post-processing'):
return False
verbose('post-processing script executed: ' + post_script)
return True
def stage(engine, pkg, script_env):
"""
handles the configuration stage for a package
With a provided engine and package instance, the configuration stage will be
processed.
Args:
engine: the engine
pkg: the package being configured
script_env: script environment information
Returns:
``True`` if the configuration stage is completed; ``False`` otherwise
"""
note('configuring {}...', pkg.name)
sys.stdout.flush()
# ignore configuration step for types which do not have one
if pkg.type == PackageType.PYTHON:
return True
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
pkg_install_type = NC(pkg.install_type, PackageInstallType.TARGET)
configure_opts = RelengConfigureOptions()
replicate_package_attribs(configure_opts, pkg)
configure_opts.build_dir = build_dir
configure_opts.build_output_dir = pkg.build_output_dir
configure_opts.conf_defs = pkg.conf_defs
configure_opts.conf_env = pkg.conf_env
configure_opts.conf_opts = pkg.conf_opts
configure_opts.def_dir = pkg.def_dir
configure_opts.env = script_env
configure_opts.ext = pkg.ext_modifiers
configure_opts.host_dir = engine.opts.host_dir
configure_opts.install_type = pkg_install_type
configure_opts.name = pkg.name
configure_opts.prefix = NC(pkg.prefix, engine.opts.sysroot_prefix)
configure_opts.staging_dir = engine.opts.staging_dir
configure_opts.symbols_dir = engine.opts.symbols_dir
configure_opts.target_dir = engine.opts.target_dir
configure_opts.version = pkg.version
configure_opts._quirks = engine.opts.quirks
# if package has a job-override value, use it over any global option
if pkg.fixed_jobs:
configure_opts.jobs = pkg.fixed_jobs
configure_opts.jobsconf = pkg.fixed_jobs
else:
configure_opts.jobs = engine.opts.jobs
configure_opts.jobsconf = engine.opts.jobsconf
configurer = None
if pkg.type in engine.registry.package_types:
def _(opts):
return engine.registry.package_types[pkg.type].configure(
pkg.type, opts)
configurer = _
elif pkg.type == PackageType.AUTOTOOLS:
configurer = conf_autotools
elif pkg.type == PackageType.CMAKE:
configurer = conf_cmake
elif pkg.type == PackageType.SCRIPT:
configurer = conf_script
if not configurer:
err('configurer type is not implemented: {}', pkg.type)
return False
with interim_working_dir(build_dir):
configured = configurer(configure_opts)
if not configured:
return False
return True
def stage(engine, pkg, script_env):
"""
handles the installation stage for a package
With a provided engine and package instance, the installation stage will be
processed.
Args:
engine: the engine
pkg: the package being built
script_env: script environment information
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
note('installing {}...', pkg.name)
sys.stdout.flush()
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
pkg_install_type = NC(pkg.install_type, PackageInstallType.TARGET)
if pkg_install_type == PackageInstallType.HOST:
dest_dirs = [engine.opts.host_dir]
elif pkg_install_type == PackageInstallType.IMAGES:
dest_dirs = [engine.opts.images_dir]
elif pkg_install_type == PackageInstallType.STAGING:
dest_dirs = [engine.opts.staging_dir]
elif pkg_install_type == PackageInstallType.STAGING_AND_TARGET:
dest_dirs = [engine.opts.staging_dir, engine.opts.target_dir]
else:
# default to target directory
dest_dirs = [engine.opts.target_dir]
install_opts = RelengInstallOptions()
replicate_package_attribs(install_opts, pkg)
install_opts.build_dir = build_dir
install_opts.build_output_dir = pkg.build_output_dir
install_opts.cache_file = pkg.cache_file
install_opts.def_dir = pkg.def_dir
install_opts.dest_dirs = dest_dirs
install_opts.env = script_env
install_opts.ext = pkg.ext_modifiers
install_opts.host_dir = engine.opts.host_dir
install_opts.images_dir = engine.opts.images_dir
install_opts.install_defs = pkg.install_defs
install_opts.install_env = pkg.install_env
install_opts.install_opts = pkg.install_opts
install_opts.install_type = pkg_install_type
install_opts.name = pkg.name
install_opts.prefix = NC(pkg.prefix, engine.opts.sysroot_prefix)
install_opts.staging_dir = engine.opts.staging_dir
install_opts.symbols_dir = engine.opts.symbols_dir
install_opts.target_dir = engine.opts.target_dir
install_opts.version = pkg.version
install_opts._quirks = engine.opts.quirks
installer = None
if pkg.type in engine.registry.package_types:
def _(opts):
return engine.registry.package_types[pkg.type].install(
pkg.type, opts)
installer = _
elif pkg.type == PackageType.AUTOTOOLS:
installer = install_autotools
elif pkg.type == PackageType.CMAKE:
installer = install_cmake
elif pkg.type == PackageType.PYTHON:
installer = install_python
elif pkg.type == PackageType.SCRIPT:
installer = install_script
if not installer:
err('installer type is not implemented: {}', pkg.type)
return False
with interim_working_dir(build_dir):
installed = installer(install_opts)
if not installed:
return False
return True
def stage(engine, pkg, ignore_cache, extra_opts):
"""
handles the fetching stage for a package
With a provided engine and package instance, the fetching stage will be
processed.
Args:
engine: the engine
pkg: the package being fetched
ignore_cache: always attempt to ignore the cache
extra_opts: extra options for the fetch operation (if applicable)
Returns:
``True`` if the fetching stage is completed; ``False`` otherwise
"""
assert pkg.vcs_type
name = pkg.name
debug('process fetch stage: ' + name)
# packages flagged for local sources requires to be already checked out
if pkg.local_srcs:
if os.path.isdir(pkg.build_dir):
return True
err(
'''\
missing local sources for internal package: {0}
The active configuration is flagged for 'local sources' mode; however, an
internal package cannot be found in the local system. Before continuing, ensure
you have checked out all internal packages on your local system (or, disable the
local sources option to use the default process).
Package: {0}
Expected Path: {1}''', name, pkg.build_dir)
return False
# if the vcs-type is archive-based, flag that hash checks are needed
perform_file_asc_check = False
perform_file_hash_check = False
if pkg.vcs_type == VcsType.URL:
perform_file_asc_check = os.path.exists(pkg.asc_file)
perform_file_hash_check = True
fetch_opts = RelengFetchOptions()
replicate_package_attribs(fetch_opts, pkg)
fetch_opts.cache_dir = pkg.cache_dir
fetch_opts.ext = pkg.ext_modifiers
fetch_opts.extra_opts = extra_opts
fetch_opts.ignore_cache = ignore_cache
fetch_opts.name = name
fetch_opts.revision = pkg.revision
fetch_opts.site = pkg.site
fetch_opts.version = pkg.version
fetch_opts._mirror = False
fetch_opts._quirks = engine.opts.quirks
fetch_opts._urlopen_context = engine.opts.urlopen_context
cache_filename = os.path.basename(pkg.cache_file)
out_dir = engine.opts.out_dir
with generate_temp_dir(out_dir) as work_dir, \
generate_temp_dir(out_dir) as interim_cache_dir:
with interim_working_dir(work_dir):
interim_cache_file = os.path.join(interim_cache_dir,
cache_filename)
fetch_opts.cache_file = interim_cache_file
fetch_opts.work_dir = work_dir
# check if file caching should be ignored
#
# In special cases, a developer may configure a project to have a
# fetched source not to cache. For example, pulling from a branch of
# a VCS source will make a cache file from the branch and will
# remain until manually removed from a cache file. A user may wish
# to re-build the local cache file after cleaning their project.
# While the releng-tool framework separates fetching/extraction into
# two parts, ignoring cached assets can be partially achieved by
# just removing any detected cache file if a project is configured
# to ignore a cache file.
if engine.opts.devmode and pkg.devmode_ignore_cache is not None:
fetch_opts.ignore_cache = pkg.devmode_ignore_cache
if pkg.devmode_ignore_cache and os.path.exists(pkg.cache_file):
verbose('removing cache file (per configuration): ' + name)
if not path_remove(pkg.cache_file):
return False
# remove cache file if there is a force request to ignore the cache
elif engine.opts.force and ignore_cache:
if os.path.exists(pkg.cache_file):
verbose('removing cache file (forced): ' + name)
if not path_remove(pkg.cache_file):
return False
# force explicit ignore cache (to off) when not in development mode
elif not engine.opts.devmode and ignore_cache is None:
fetch_opts.ignore_cache = False
if os.path.exists(pkg.cache_file):
rv = None
if perform_file_hash_check:
hr = verify_hashes(pkg.hash_file,
pkg.cache_file,
relaxed=True)
if hr == HashResult.VERIFIED:
rv = True
elif hr == HashResult.BAD_PATH:
if not perform_file_asc_check and not pkg.is_internal:
warn('missing hash file for package: ' + name)
rv = True # no hash file to compare with; assuming ok
elif hr == HashResult.EMPTY:
if not pkg.is_internal:
warn('hash file for package is empty: ' + name)
rv = True # empty hash file; assuming ok
elif hr == HashResult.MISMATCH:
if not path_remove(pkg.cache_file):
rv = False
elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED):
rv = False
elif hr == HashResult.MISSING_ARCHIVE:
if not perform_file_asc_check:
err(
'''\
missing archive hash for verification
The hash file for this package does not have an entry for the cache file to be
verified. Ensure the hash file defines an entry for the expected cache file:
Hash File: {}
File: {}''', pkg.hash_file, cache_filename)
rv = False
else:
err(
'invalid fetch operation (internal error; '
'hash-check failure: {})', hr)
rv = False
else:
rv = True
if rv is not False and perform_file_asc_check and \
os.path.exists(pkg.cache_file):
if GPG.validate(pkg.asc_file, pkg.cache_file):
rv = True
else:
if not path_remove(pkg.cache_file):
err(
'''\
failed to validate against ascii-armor
Validation of a package resource failed to verify against a provided ASCII-armor
file. Ensure that the package's public key has been registered into gpg.
ASC File: {}
File: {}''', pkg.asc_file, cache_filename)
rv = False
else:
rv = None
if rv is not None:
if ignore_cache:
verbose('ignoring cache not supported for package: {}',
name)
return rv
# find fetching method for the target vcs-type
fetcher = None
if pkg.vcs_type in engine.registry.fetch_types:
def _(opts):
return engine.registry.fetch_types[pkg.vcs_type].fetch(
pkg.vcs_type, opts)
fetcher = _
elif pkg.vcs_type == VcsType.BZR:
fetcher = fetch_bzr
elif pkg.vcs_type == VcsType.CVS:
fetcher = fetch_cvs
elif pkg.vcs_type == VcsType.GIT:
fetcher = fetch_git
elif pkg.vcs_type == VcsType.HG:
fetcher = fetch_mercurial
elif pkg.vcs_type == VcsType.RSYNC:
fetcher = fetch_rsync
elif pkg.vcs_type == VcsType.SCP:
fetcher = fetch_scp
elif pkg.vcs_type == VcsType.SVN:
fetcher = fetch_svn
elif pkg.vcs_type == VcsType.URL:
fetcher = fetch_url
if not fetcher:
err('fetch type is not implemented: {}', pkg.vcs_type)
return False
# if this is url-type location, attempt to search on the mirror
# first (if configured)
fetched = None
if engine.opts.url_mirror and pkg.vcs_type == VcsType.URL:
original_site = fetch_opts.site
new_site = engine.opts.url_mirror + cache_filename
if original_site != new_site:
fetch_opts._mirror = True
fetch_opts.site = new_site
fetched = fetcher(fetch_opts)
fetch_opts.site = original_site
fetch_opts._mirror = False
# perform the fetch request (if not already fetched)
if not fetched:
fetched = fetcher(fetch_opts)
if not fetched:
return False
# if the fetch type has populated the package's cache directory
# directly, we are done
if fetched == pkg.cache_dir:
pass
# if the fetch type has returned a file, the file needs to be hash
# checked and then be moved into the download cache
elif fetched == interim_cache_file:
if perform_file_hash_check:
hr = verify_hashes(pkg.hash_file, fetched)
if hr == HashResult.VERIFIED:
pass
elif hr == HashResult.BAD_PATH:
if not perform_file_asc_check and not pkg.is_internal:
warn('missing hash file for package: ' + name)
elif hr == HashResult.EMPTY:
if not pkg.is_internal:
warn('hash file for package is empty: ' + name)
elif hr == HashResult.MISMATCH:
return False
elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED):
return False
elif hr == HashResult.MISSING_ARCHIVE:
if not perform_file_asc_check:
err(
'''\
missing archive hash for verification
The hash file for this package does not have an entry for the cache file to be
verified. Ensure the hash file defines an entry for the expected cache file:
Hash File: {}
File: {}''', pkg.hash_file, cache_filename)
return False
else:
err(
'invalid fetch operation (internal error; '
'hash-check failure: {})', hr)
return False
if perform_file_asc_check:
if not GPG.validate(pkg.asc_file, interim_cache_file):
err(
'''\
failed to validate against ascii-armor
Validation of a package resource failed to verify against a provided ASCII-armor
file. Ensure that the package's public key has been registered into gpg.
ASC File: {}
File: {}''', pkg.asc_file, cache_filename)
return False
debug('fetch successful; moving cache file')
# ensure the cache container/directory exists
cache_dir = os.path.dirname(pkg.cache_file)
if not ensure_dir_exists(cache_dir):
return False
try:
shutil.move(interim_cache_file, pkg.cache_file)
except shutil.Error:
err(
'invalid fetch operation (internal error; fetch mode '
'"{}" has provided a missing cache file)',
pkg.vcs_type)
return False
else:
err(
'invalid fetch operation (internal error; fetch mode "{}" '
'has returned an unsupported value)', pkg.vcs_type)
return False
return True
def stage(engine, pkg):
"""
handles the extraction stage for a package
With a provided engine and package instance, the extraction stage will be
processed.
Args:
engine: the engine
pkg: the package being extracted
Returns:
``True`` if the extraction stage is completed; ``False`` otherwise
"""
# packages flagged for local sources do not have an extraction stage
if pkg.local_srcs:
return True
# skip packages flagged not to extract
if pkg.no_extraction:
return True
note('extracting {}...', pkg.name)
sys.stdout.flush()
extract_opts = RelengExtractOptions()
replicate_package_attribs(extract_opts, pkg)
extract_opts.cache_dir = pkg.cache_dir
extract_opts.cache_file = pkg.cache_file
extract_opts.ext = pkg.ext_modifiers
extract_opts.name = pkg.name
extract_opts.revision = pkg.revision
extract_opts.strip_count = pkg.strip_count
extract_opts.version = pkg.version
extract_opts._extract_override = engine.opts.extract_override
extract_opts._quirks = engine.opts.quirks
if os.path.exists(pkg.build_dir):
warn('build directory exists before extraction; removing')
if not path_remove(pkg.build_dir):
err('unable to cleanup build directory: ' + pkg.build_dir)
return False
# prepare and step into the a newly created working directory
#
# An extractor will take the contents of an archive, cache directory or
# other fetched content and populate the "work" directory. On successful
# extraction (or moving resources), the work directory will be moved to the
# package's respective build directory.
out_dir = engine.opts.out_dir
with generate_temp_dir(out_dir) as work_dir:
with interim_working_dir(work_dir):
extract_opts.work_dir = work_dir
extracter = None
hash_exclude = []
extract_types = engine.registry.extract_types
if pkg.extract_type and pkg.extract_type in extract_types:
def _(opts):
return engine.registry.extract_types[pkg.vcs_type].extract(
pkg.vcs_type, opts)
extracter = _
elif pkg.vcs_type in extract_types:
extracter = extract_types[pkg.vcs_type].extract
elif pkg.vcs_type == VcsType.GIT:
extracter = extract_git
elif pkg.vcs_type == VcsType.HG:
extracter = extract_mercurial
elif os.path.isfile(pkg.cache_file):
cache_basename = os.path.basename(pkg.cache_file)
hash_exclude.append(cache_basename)
extracter = extract_archive
if not extracter:
err('extract type is not implemented: {}', pkg.vcs_type)
return False
# perform the extract request
extracted = extracter(extract_opts)
if not extracted:
return False
result = verify_hashes(pkg.hash_file, work_dir, hash_exclude)
if result == HashResult.VERIFIED:
pass
elif result == HashResult.BAD_PATH:
if not pkg.is_internal:
warn('missing hash file for package: ' + pkg.name)
elif result == HashResult.EMPTY:
if not pkg.is_internal:
verbose('hash file for package is empty: ' + pkg.name)
elif result in (HashResult.BAD_FORMAT, HashResult.MISMATCH,
HashResult.MISSING_LISTED, HashResult.UNSUPPORTED):
return False
else:
err('invalid extract operation (internal error; '
'hash-check failure: {})', result)
return False
debug('extraction successful; moving sources into package output '
'directory: ' + pkg.build_dir)
shutil.move(work_dir, pkg.build_dir)
return True
def stage(engine, pkg, script_env):
"""
handles the build stage for a package
With a provided engine and package instance, the build stage will be
processed.
Args:
engine: the engine
pkg: the package being built
script_env: script environment information
Returns:
``True`` if the build stage is completed; ``False`` otherwise
"""
note('building {}...', pkg.name)
sys.stdout.flush()
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
build_opts = RelengBuildOptions()
replicate_package_attribs(build_opts, pkg)
build_opts.build_defs = pkg.build_defs
build_opts.build_dir = build_dir
build_opts.build_env = pkg.build_env
build_opts.build_opts = pkg.build_opts
build_opts.build_output_dir = pkg.build_output_dir
build_opts.def_dir = pkg.def_dir
build_opts.env = script_env
build_opts.ext = pkg.ext_modifiers
build_opts.host_dir = engine.opts.host_dir
build_opts.name = pkg.name
build_opts.prefix = NC(pkg.prefix, engine.opts.sysroot_prefix)
build_opts.staging_dir = engine.opts.staging_dir
build_opts.symbols_dir = engine.opts.symbols_dir
build_opts.target_dir = engine.opts.target_dir
build_opts.version = pkg.version
build_opts._quirks = engine.opts.quirks
# if package has a job-override value, use it over any global option
if pkg.fixed_jobs:
build_opts.jobs = pkg.fixed_jobs
build_opts.jobsconf = pkg.fixed_jobs
else:
build_opts.jobs = engine.opts.jobs
build_opts.jobsconf = engine.opts.jobsconf
builder = None
if pkg.type in engine.registry.package_types:
def _(opts):
return engine.registry.package_types[pkg.type].build(pkg.type, opts)
builder = _
elif pkg.type == PackageType.AUTOTOOLS:
builder = build_autotools
elif pkg.type == PackageType.CMAKE:
builder = build_cmake
elif pkg.type == PackageType.PYTHON:
builder = build_python
elif pkg.type == PackageType.SCRIPT:
builder = build_script
if not builder:
err('build type is not implemented: {}', pkg.type)
return False
with interim_working_dir(build_dir):
built = builder(build_opts)
if not built:
return False
return True