def test_utilio_runscript(self):
with prepare_workdir() as work_dir:
valid_script = os.path.join(work_dir, 'valid')
with open(valid_script, 'a') as f:
f.write('test=1\n')
result = run_script(valid_script, {})
self.assertEqual(result['test'], 1)
invalid_script = os.path.join(work_dir, 'invalid')
with open(invalid_script, 'a') as f:
f.write('bad-line\n')
result = run_script(invalid_script, {})
self.assertIsNone(result)
with self.assertRaises(NameError):
run_script(invalid_script, {}, catch=False)
def _validate_test_script(self, name):
gbls = None
for path in sys.path:
script = os.path.join(path, name)
if os.path.exists(script):
gbls = run_script(script, gbls, catch=False)
break
self.assertIsNotNone(gbls)
self.assertEqual(gbls['var'], name)
def finalize_package(self, pkg, script):
"""
finalize configuration for a package
Attempts to finalize any configuration entries of an already populated
package instance with options provided at a later stage in the
releng-tool process. This is to support projects where select
configuration options are defined in the package's source content,
instead of the main releng-tool project.
This call will accept as package instance to update and the script file
which may include a series of configuration options to apply to a
package. Note that any configuration option already set on the package
will be used over any new detected package option.
Args:
pkg: the package
script: the package script to load
Raises:
RelengToolInvalidPackageConfiguration: when an error has been
detected loading any of the
package's extended options
"""
verbose('finalize package configuration: {}', pkg.name)
debug('script {}', script)
if not os.path.isfile(script):
raise RelengToolMissingPackageScript({
'pkg_name': pkg.name,
'script': script,
})
try:
env = run_script(script, self.script_env, catch=False)
except Exception as e:
raise RelengToolInvalidPackageScript({
'description':
str(e),
'script':
script,
'traceback':
traceback.format_exc(),
})
# apply any options to unset configuration entries
self._active_package = pkg.name
self._active_env = env
self._apply_postinit_options(pkg)
# extend the active script environment if the post-init call succeeds
extend_script_env(self.script_env, env)
def install(opts):
"""
support installation project-defined scripts
With provided installation options (``RelengInstallOptions``), the
installation stage will be processed.
Args:
opts: installation options
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
assert opts
build_dir = opts.build_dir
def_dir = opts.def_dir
env = opts.env
install_script_filename = '{}-{}'.format(opts.name, INSTALL_SCRIPT)
install_script = os.path.join(def_dir, install_script_filename)
install_script, install_script_exists = opt_file(install_script)
if not install_script_exists:
if (opts._skip_remote_scripts or
'releng.disable_remote_scripts' in opts._quirks):
return True
install_script_filename = '{}-{}'.format('releng', INSTALL_SCRIPT)
install_script = os.path.join(build_dir, install_script_filename)
install_script, install_script_exists = opt_file(install_script)
if not install_script_exists:
return True
if not run_script(install_script, env, subject='install'):
return False
verbose('install script executed: ' + install_script)
return True
def configure(opts):
"""
support configuration project-defined scripts
With provided configuration options (``RelengConfigureOptions``), the
configuration stage will be processed.
Args:
opts: configuration options
Returns:
``True`` if the configuration stage is completed; ``False`` otherwise
"""
assert opts
build_dir = opts.build_dir
def_dir = opts.def_dir
env = opts.env
configure_script_filename = '{}-{}'.format(opts.name, CONFIGURE_SCRIPT)
configure_script = os.path.join(def_dir, configure_script_filename)
configure_script, configure_script_exists = opt_file(configure_script)
if not configure_script_exists:
if (opts._skip_remote_scripts
or 'releng.disable_remote_scripts' in opts._quirks):
return True
configure_script_filename = '{}-{}'.format('releng', CONFIGURE_SCRIPT)
configure_script = os.path.join(build_dir, configure_script_filename)
configure_script, configure_script_exists = opt_file(configure_script)
if not configure_script_exists:
return True
if not run_script(configure_script, env, subject='configure'):
return False
verbose('install script executed: ' + configure_script)
return True
def stage(engine, pkg, script_env):
"""
handles the post-processing stage for a package
With a provided engine and package instance, the post-processing stage will
be processed. This stage is typically not advertised and is for advanced
cases where a developer wishes to manipulate their build environment after
package has completed each of its phases.
Args:
engine: the engine
pkg: the package being built
script_env: script environment information
Returns:
``True`` if the post-processing stage is completed; ``False`` otherwise
"""
verbose('post-processing {}...', pkg.name)
sys.stdout.flush()
post_script_filename = '{}-{}'.format(pkg.name, POST_SCRIPT)
post_script = os.path.join(pkg.def_dir, post_script_filename)
post_script, post_script_exists = opt_file(post_script)
if not post_script_exists:
return True
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
with interim_working_dir(build_dir):
if not run_script(post_script, script_env, subject='post-processing'):
return False
verbose('post-processing script executed: ' + post_script)
return True
def build(opts):
"""
support building project-defined scripts
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
assert opts
build_dir = opts.build_dir
def_dir = opts.def_dir
env = opts.env
build_script_filename = '{}-{}'.format(opts.name, BUILD_SCRIPT)
build_script = os.path.join(def_dir, build_script_filename)
build_script, build_script_exists = opt_file(build_script)
if not build_script_exists:
if (opts._skip_remote_scripts or
'releng.disable_remote_scripts' in opts._quirks):
return True
build_script_filename = '{}-{}'.format('releng', BUILD_SCRIPT)
build_script = os.path.join(build_dir, build_script_filename)
build_script, build_script_exists = opt_file(build_script)
if not build_script_exists:
return True
if not run_script(build_script, env, subject='build'):
return False
verbose('install script executed: ' + build_script)
return True
def load_package(self, name, script):
"""
load a package definition
Attempts to load a package definition of a given ``name`` from the
provided ``script`` location. The script will be examine for required
and optional configuration keys. On a successful execution/parsing, a
package object will be returned along with other meta information. On
error, ``None`` types are returned.
Args:
name: the package name
script: the package script to load
Returns:
returns a tuple of three (3) containing the package instance, the
extracted environment/globals from the package script and a list of
known package dependencies
Raises:
RelengToolInvalidPackageConfiguration: when an error has been
detected loading the package
"""
verbose('loading package: {}', name)
debug('script {}', script)
opts = self.opts
if not os.path.isfile(script):
raise RelengToolMissingPackageScript({
'pkg_name': name,
'script': script,
})
pkg_def_dir = os.path.abspath(os.path.join(script, os.pardir))
self.script_env['PKG_DEFDIR'] = pkg_def_dir
try:
env = run_script(script, self.script_env, catch=False)
except Exception as e:
raise RelengToolInvalidPackageScript({
'description':
str(e),
'script':
script,
'traceback':
traceback.format_exc(),
})
self._active_package = name
self._active_env = env
# prepare helper expand values
expand_extra = {}
# version/revision extraction first
#
# Attempt to check the version first since it will be the most commonly
# used package field -- rather initially fail on a simple field first
# (for new packages and/or developers) than breaking on a possibly more
# complex field below. Note that the version field is optional, in cases
# where a package type does not need a version entry (e.g. sites which
# do not require a version value for fetching or there is not revision
# value to use instead).
#
# Note that when in development mode, the development-mode revision
# (if any is set) needs to be checked as well. This value may override
# the package's version value.
# version
pkg_version = self._fetch(Rpk.VERSION)
if not pkg_version:
pkg_version = ''
pkg_version_key = pkg_key(name, Rpk.VERSION)
expand_extra[pkg_version_key] = pkg_version
# development mode revision
pkg_has_devmode_option = False
pkg_devmode_revision = self._fetch(Rpk.DEVMODE_REVISION,
allow_expand=True,
expand_extra=expand_extra)
if pkg_devmode_revision:
pkg_has_devmode_option = True
if opts.revision_override and name in opts.revision_override:
pkg_devmode_revision = opts.revision_override[name]
if opts.devmode:
pkg_version = pkg_devmode_revision
expand_extra[pkg_version_key] = pkg_version
# revision
if opts.revision_override and name in opts.revision_override:
pkg_revision = opts.revision_override[name]
else:
pkg_revision = self._fetch(Rpk.REVISION,
allow_expand=True,
expand_extra=expand_extra)
if opts.devmode and pkg_devmode_revision:
pkg_revision = pkg_devmode_revision
elif not pkg_revision:
pkg_revision = pkg_version
# site / vcs-site detection
#
# After extracted required version information, the site / VCS type
# needs to be checked next. This will allow the manage to early detect
# if a version/revision field is required, and fail early if we have
# not detected one from above.
# site
if opts.sites_override and name in opts.sites_override:
# Site overriding is permitted to help in scenarios where a builder
# is unable to acquire a package's source from the defined site.
# This includes firewall settings or a desire to use a mirrored
# source when experiencing network connectivity issues.
pkg_site = opts.sites_override[name]
else:
pkg_site = self._fetch(Rpk.SITE,
allow_expand=True,
expand_extra=expand_extra)
# On Windows, if a file site is provided, ensure the path value is
# converted to a posix-styled path, to prevent issues with `urlopen`
# being provided an unescaped path string
if sys.platform == 'win32' and \
pkg_site and pkg_site.startswith('file://'):
pkg_site = pkg_site[len('file://'):]
abs_site = os.path.isabs(pkg_site)
pkg_site = pkg_site.replace(os.sep, posixpath.sep)
if abs_site:
pkg_site = '/' + pkg_site
pkg_site = 'file://' + pkg_site
# vcs-type
pkg_vcs_type = None
pkg_vcs_type_raw = self._fetch(Rpk.VCS_TYPE)
if pkg_vcs_type_raw:
pkg_vcs_type_raw = pkg_vcs_type_raw.lower()
if pkg_vcs_type_raw in VcsType:
pkg_vcs_type = pkg_vcs_type_raw
elif pkg_vcs_type_raw in self.registry.fetch_types:
pkg_vcs_type = pkg_vcs_type_raw
else:
raise RelengToolUnknownVcsType({
'pkg_name':
name,
'pkg_key':
pkg_key(name, Rpk.VCS_TYPE),
})
if not pkg_vcs_type:
if pkg_site:
site_lc = pkg_site.lower()
if site_lc.startswith('bzr+'):
pkg_site = pkg_site[4:]
pkg_vcs_type = VcsType.BZR
elif site_lc.startswith('cvs+'):
pkg_site = pkg_site[4:]
pkg_vcs_type = VcsType.CVS
elif site_lc.startswith((
':ext:',
':extssh:',
':gserver:',
':kserver:',
':pserver:',
)):
pkg_vcs_type = VcsType.CVS
elif site_lc.startswith('git+'):
pkg_site = pkg_site[4:]
pkg_vcs_type = VcsType.GIT
elif site_lc.endswith('.git'):
pkg_vcs_type = VcsType.GIT
elif site_lc.startswith('hg+'):
pkg_site = pkg_site[3:]
pkg_vcs_type = VcsType.HG
elif site_lc.startswith('rsync+'):
pkg_site = pkg_site[6:]
pkg_vcs_type = VcsType.RSYNC
elif site_lc.startswith('scp+'):
pkg_site = pkg_site[4:]
pkg_vcs_type = VcsType.SCP
elif site_lc.startswith('svn+'):
pkg_site = pkg_site[4:]
pkg_vcs_type = VcsType.SVN
elif site_lc == 'local':
pkg_vcs_type = VcsType.LOCAL
else:
pkg_vcs_type = VcsType.URL
else:
pkg_vcs_type = VcsType.NONE
if pkg_vcs_type == VcsType.LOCAL:
warn('package using local content: {}', name)
# check if the detected vcs type needs a revision, and fail if we do
# not have one
if not pkg_revision and pkg_vcs_type in (
VcsType.BZR,
VcsType.CVS,
VcsType.GIT,
VcsType.HG,
VcsType.SVN,
):
raise RelengToolMissingPackageRevision({
'pkg_name':
name,
'pkg_key1':
pkg_key(name, Rpk.VERSION),
'pkg_key2':
pkg_key(name, Rpk.REVISION),
'vcs_type':
pkg_vcs_type,
})
# archive extraction strip count
pkg_strip_count = self._fetch(Rpk.STRIP_COUNT,
default=DEFAULT_STRIP_COUNT)
# build subdirectory
pkg_build_subdir = self._fetch(Rpk.BUILD_SUBDIR)
# dependencies
deps = self._fetch(Rpk.DEPS, default=[])
# ignore cache
pkg_devmode_ignore_cache = self._fetch(Rpk.DEVMODE_IGNORE_CACHE)
# extension (override)
pkg_filename_ext = self._fetch(Rpk.EXTENSION)
# extract type
pkg_extract_type = self._fetch(Rpk.EXTRACT_TYPE)
if pkg_extract_type:
pkg_extract_type = pkg_extract_type.lower()
if pkg_extract_type not in self.registry.extract_types:
raise RelengToolUnknownExtractType({
'pkg_name':
name,
'pkg_key':
pkg_key(name, Rpk.EXTRACT_TYPE),
})
# is-external
pkg_is_external = self._fetch(Rpk.EXTERNAL)
# is-internal
pkg_is_internal = self._fetch(Rpk.INTERNAL)
# no extraction
pkg_no_extraction = self._fetch(Rpk.NO_EXTRACTION)
# skip any remote configuration
pkg_skip_remote_config = self._fetch(Rpk.SKIP_REMOTE_CONFIG)
# skip any remote scripts
pkg_skip_remote_scripts = self._fetch(Rpk.SKIP_REMOTE_SCRIPTS)
# type
pkg_type = None
pkg_type_raw = self._fetch(Rpk.TYPE)
if pkg_type_raw:
pkg_type_raw = pkg_type_raw.lower()
if pkg_type_raw in PackageType:
pkg_type = pkg_type_raw
elif pkg_type_raw in self.registry.package_types:
pkg_type = pkg_type_raw
else:
raise RelengToolUnknownPackageType({
'pkg_name':
name,
'pkg_key':
pkg_key(name, Rpk.TYPE),
})
if not pkg_type:
pkg_type = PackageType.SCRIPT
# ######################################################################
# git configuration options for a repository
pkg_git_config = self._fetch(Rpk.GIT_CONFIG)
# git-depth
pkg_git_depth = self._fetch(Rpk.GIT_DEPTH)
# git-refspecs
pkg_git_refspecs = self._fetch(Rpk.GIT_REFSPECS)
# git-submodules
pkg_git_submodules = self._fetch(Rpk.GIT_SUBMODULES)
# git-verify
pkg_git_verify_revision = self._fetch(Rpk.GIT_VERIFY_REVISION)
# ######################################################################
# checks
if pkg_is_external is not None and pkg_is_internal is not None:
if pkg_is_external == pkg_is_internal:
raise RelengToolConflictingConfiguration({
'pkg_name':
name,
'pkg_key1':
pkg_key(name, Rpk.EXTERNAL),
'pkg_key2':
pkg_key(name, Rpk.INTERNAL),
'desc':
'package flagged as external and internal',
})
elif pkg_is_external is not None:
pkg_is_internal = not pkg_is_external
elif pkg_is_internal is not None:
pass
elif opts.default_internal_pkgs:
pkg_is_internal = True
else:
pkg_is_internal = False
# check a site is defined for vcs types which require it
if not pkg_site and pkg_vcs_type in (
VcsType.BZR,
VcsType.CVS,
VcsType.GIT,
VcsType.HG,
VcsType.RSYNC,
VcsType.SCP,
VcsType.SVN,
VcsType.URL,
):
raise RelengToolMissingPackageSite({
'pkg_name':
name,
'pkg_key':
pkg_key(name, Rpk.SITE),
'vcs_type':
pkg_vcs_type,
})
# list of support dvcs types
SUPPORTED_DVCS = [
VcsType.GIT,
VcsType.HG,
]
is_pkg_dvcs = (pkg_vcs_type in SUPPORTED_DVCS)
# find possible extension for a cache file
#
# non-dvcs's will be always gzip-tar'ed.
if pkg_vcs_type in (
VcsType.BZR,
VcsType.CVS,
VcsType.RSYNC,
VcsType.SVN,
):
cache_ext = 'tgz'
# dvcs's will not have an extension type
elif is_pkg_dvcs:
cache_ext = None
# non-vcs type does not have an extension
elif pkg_vcs_type in (VcsType.LOCAL, VcsType.NONE):
cache_ext = None
else:
cache_ext = None
url_parts = urlparse(pkg_site)
if opts.cache_ext_transform:
# Allow a configuration to override the target cache file's
# extension based on the package's site path (for unique path
# scenarios).
cache_ext = opts.cache_ext_transform(url_parts.path)
if not cache_ext:
if pkg_filename_ext:
cache_ext = pkg_filename_ext
else:
basename = os.path.basename(url_parts.path)
__, cache_ext = interpret_stem_extension(basename)
# prepare package container and directory locations
#
# The container folder for a package will typically be a combination of
# a package's name plus version. If no version is set, the container
# will be only use the package's name. We try to use the version entry
# when possible to help manage multiple versions of output (e.g. to
# avoid conflicts when bumping versions).
#
# When the version value is used, we will attempt to cleanup/minimize
# the version to help provide the container a more "sane" path. For
# instance, if a version references a path-styled branch names (e.g.
# `bugfix/my-bug`, we want to avoid promoting a container name which
# can result in a sub-directory being made (e.g. `pkg-bugfix/my-bug/`).
if pkg_version:
pkg_nv = '{}-{}'.format(
name, ''.join(x if (x.isalnum() or x in '-._') else '_'
for x in pkg_version))
else:
pkg_nv = name
pkg_build_output_dir = os.path.join(opts.build_dir, pkg_nv)
if pkg_vcs_type == VcsType.LOCAL:
pkg_build_dir = pkg_def_dir
else:
pkg_build_dir = pkg_build_output_dir
# check if an internal package is configured to point to a local
# directory for sources
pkg_local_srcs = False
if pkg_is_internal and opts.local_srcs:
# specific package name reference in the local sources; either is
# set to the path to use, or is set to `None` to indicate at this
# package should not be retrieved locally
if name in opts.local_srcs:
if opts.local_srcs[name]:
pkg_build_dir = opts.local_srcs[name]
pkg_local_srcs = True
# check if the "global" local sources path exists; either set to
# a specific path, or set to `None` to indicate that it will use
# the parent path based off the root directory
elif GBL_LSRCS in opts.local_srcs:
if opts.local_srcs[GBL_LSRCS]:
container_dir = opts.local_srcs[GBL_LSRCS]
else:
container_dir = os.path.dirname(opts.root_dir)
pkg_build_dir = os.path.join(container_dir, name)
pkg_local_srcs = True
if pkg_build_dir == opts.root_dir:
raise RelengToolConflictingLocalSrcsPath({
'pkg_name': name,
'root': opts.root_dir,
'path': pkg_build_dir,
})
if pkg_build_subdir:
pkg_build_subdir = os.path.join(pkg_build_dir, pkg_build_subdir)
cache_dir = os.path.join(opts.dl_dir, name)
if cache_ext:
pkg_cache_file = os.path.join(cache_dir, pkg_nv + '.' + cache_ext)
else:
pkg_cache_file = os.path.join(cache_dir, pkg_nv)
# Select sources (like CMake-based projects) may wish to be using
# out-of-source tree builds. For supported project types, adjust the
# build output directory to a sub-folder of the originally assumed
# output folder.
if pkg_type == PackageType.CMAKE:
pkg_build_output_dir = os.path.join(pkg_build_output_dir,
'releng-output')
# determine the build tree for a package
#
# A build tree (introduced for the libfoo-exec action), tracks the
# directory where build commands would typically be executed for a
# package on a host system. In most cases, this will be set to the
# same path as `pkg_build_dir` (or the sub-directory, if provided);
# however, some package types may have a better working directory
# for build commands. For example, CMake projects will generate a
# build package in an out-of-source directory (e.g.
# `pkg_build_output_dir`), which is a better make to issue commands
# such as "cmake --build .".
if pkg_type == PackageType.CMAKE:
pkg_build_tree = pkg_build_output_dir
elif pkg_build_subdir:
pkg_build_tree = pkg_build_subdir
else:
pkg_build_tree = pkg_build_dir
# determine the package directory for this package
#
# Typically, a package's "cache directory" will be stored in the output
# folder's "cache/<pkg-name>" path. However, having package-name driven
# cache folder targets does not provide an easy way to manage sharing
# caches between projects if they share the same content (either the
# same site or sharing submodules). Cache targets for packages will be
# stored in a database and can be used here to decide if a package's
# cache will actually be stored in a different container.
pkg_cache_dir = os.path.join(opts.cache_dir, name)
if is_pkg_dvcs:
ckey = pkg_cache_key(pkg_site)
pkg_cache_dirname = name
# if the default cache directory exists, always prioritize it (and
# force update the cache location)
if os.path.exists(pkg_cache_dir):
self._dvcs_cache[name] = name
# if the cache content is stored in another container, use it
elif ckey in self._dvcs_cache:
pkg_cache_dirname = self._dvcs_cache[ckey]
verbose('alternative cache path for package: {} -> {}', name,
pkg_cache_dirname)
# track ckey entry to point to our cache container
#
# This package's "ckey" will be used to cache the target folder
# being used for this package, so other packages with matching site
# values could use it. In the rare case that the "ckey" entry
# already exists but is pointing to another folder that our target
# one, leave it as is (assume ownership of key is managed by another
# package).
if ckey not in self._dvcs_cache:
self._dvcs_cache[ckey] = pkg_cache_dirname
# adjust the cache directory and save any new cache changes
pkg_cache_dir = os.path.join(opts.cache_dir, pkg_cache_dirname)
self._save_dvcs_cache()
# (commons)
pkg = RelengPackage(name, pkg_version)
pkg.asc_file = os.path.join(pkg_def_dir, name + '.asc')
pkg.build_dir = pkg_build_dir
pkg.build_output_dir = pkg_build_output_dir
pkg.build_subdir = pkg_build_subdir
pkg.build_tree = pkg_build_tree
pkg.cache_dir = pkg_cache_dir
pkg.cache_file = pkg_cache_file
pkg.def_dir = pkg_def_dir
pkg.devmode_ignore_cache = pkg_devmode_ignore_cache
pkg.extract_type = pkg_extract_type
pkg.git_config = pkg_git_config
pkg.git_depth = pkg_git_depth
pkg.git_refspecs = pkg_git_refspecs
pkg.git_submodules = pkg_git_submodules
pkg.git_verify_revision = pkg_git_verify_revision
pkg.has_devmode_option = pkg_has_devmode_option
pkg.hash_file = os.path.join(pkg_def_dir, name + '.hash')
pkg.is_internal = pkg_is_internal
pkg.local_srcs = pkg_local_srcs
pkg.no_extraction = pkg_no_extraction
pkg.revision = pkg_revision
pkg.site = pkg_site
pkg.skip_remote_config = pkg_skip_remote_config
pkg.skip_remote_scripts = pkg_skip_remote_scripts
pkg.strip_count = pkg_strip_count
pkg.type = pkg_type
pkg.vcs_type = pkg_vcs_type
self._apply_postinit_options(pkg)
# (additional environment helpers)
for env in (os.environ, env):
env[pkg_key(name, 'BUILD_DIR')] = pkg_build_dir
env[pkg_key(name, 'BUILD_OUTPUT_DIR')] = pkg_build_output_dir
env[pkg_key(name, 'DEFDIR')] = pkg_def_dir
env[pkg_key(name, 'NAME')] = name
env[pkg_key(name, 'REVISION')] = pkg_revision
os.environ[pkg_key(name, Rpk.VERSION)] = pkg_version
# (internals)
prefix = '.releng_tool-stage-'
outdir = pkg.build_output_dir
pkg._ff_bootstrap = os.path.join(outdir, prefix + 'bootstrap')
pkg._ff_build = os.path.join(outdir, prefix + 'build')
pkg._ff_configure = os.path.join(outdir, prefix + 'configure')
pkg._ff_extract = os.path.join(outdir, prefix + 'extract')
pkg._ff_install = os.path.join(outdir, prefix + 'install')
pkg._ff_license = os.path.join(outdir, prefix + 'license')
pkg._ff_patch = os.path.join(outdir, prefix + 'patch')
pkg._ff_post = os.path.join(outdir, prefix + 'post')
# dump package attributes if running in a debug mode
if opts.debug:
info = {}
for key, value in pkg.__dict__.items():
if not key.startswith('_'):
info[key] = value
debug(
'''package-data: {}
==============================
{}
==============================''', name, pprint.pformat(info))
return pkg, env, deps