def install(opts):
"""
support installation cmake projects
With provided installation options (``RelengInstallOptions``), the
installation stage will be processed.
Args:
opts: installation options
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
if not CMAKE.exists():
err('unable to install package; cmake is not installed')
return False
# check if the no-install flag is set
if opts._cmake_noinstall:
verbose('configured to skip install stage for cmake')
return True
# default definitions
cmake_defs = {}
if opts.install_defs:
cmake_defs.update(expand(opts.install_defs))
# default options
cmake_opts = {
# build RelWithDebInfo (when using multi-configuration projects)
'--config': 'RelWithDebInfo',
# default install using the install target
'--target': 'install',
}
if opts.install_opts:
cmake_opts.update(expand(opts.install_opts))
# argument building
cmake_args = [
'--build',
opts.build_output_dir,
]
cmake_args.extend(prepare_definitions(cmake_defs, '-D'))
cmake_args.extend(prepare_arguments(cmake_opts))
# prepare environment for installation request; an environment dictionary is
# always needed to apply a custom DESTDIR during each install request
env = expand(opts.install_env)
if not env:
env = {}
# install to each destination
for dest_dir in opts.dest_dirs:
env['DESTDIR'] = dest_dir
if not CMAKE.execute(cmake_args, env=env):
err('failed to install cmake project: {}', opts.name)
return False
return True
def build(opts):
"""
support building python projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if opts._python_interpreter:
python_tool = PythonTool(opts._python_interpreter,
env_include=PYTHON_EXTEND_ENV)
else:
python_tool = PYTHON
if not python_tool.exists():
err('unable to build package; python is not installed')
return False
# definitions
python_defs = {}
if opts.build_defs:
python_defs.update(expand(opts.build_defs))
# default options
python_opts = {}
if opts.build_opts:
python_opts.update(expand(opts.build_opts))
# default environment
path1 = python_tool.path(sysroot=opts.staging_dir, prefix=opts.prefix)
path2 = python_tool.path(sysroot=opts.target_dir, prefix=opts.prefix)
env = {'PYTHONPATH': path1 + os.pathsep + path2}
# apply package-specific environment options
if opts.build_env:
env.update(expand(opts.build_env))
# argument building
python_args = [
'setup.py',
# ignore user's pydistutils.cfg
'--no-user-cfg',
# invoke the build operation
'build',
]
python_args.extend(prepare_definitions(python_defs))
python_args.extend(prepare_arguments(python_opts))
if not python_tool.execute(python_args, env=env):
err('failed to build python project: {}', opts.name)
return False
return True
def build(opts):
"""
support building cmake projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if not CMAKE.exists():
err('unable to build package; cmake is not installed')
return False
# definitions
cmake_defs = {}
if opts.build_defs:
cmake_defs.update(expand(opts.build_defs))
# options
cmake_opts = {
# build RelWithDebInfo (when using multi-configuration projects)
'--config': 'RelWithDebInfo',
}
if opts.build_opts:
cmake_opts.update(expand(opts.build_opts))
# argument building
cmake_args = [
# tell cmake to invoke build process in the output directory
'--build',
opts.build_output_dir,
]
cmake_args.extend(prepare_definitions(cmake_defs, '-D'))
cmake_args.extend(prepare_arguments(cmake_opts))
# enable specific number of parallel jobs is set
#
# https://cmake.org/cmake/help/v3.12/manual/cmake.1.html#build-tool-mode
if 'releng.cmake.disable_parallel_option' not in opts._quirks:
if opts.jobsconf != 1 and opts.jobs > 1:
cmake_args.append('--parallel')
cmake_args.append(str(opts.jobs))
else:
verbose('cmake parallel jobs disabled by quirk')
if not CMAKE.execute(cmake_args, env=expand(opts.build_env)):
err('failed to build cmake project: {}', opts.name)
return False
return True
def configure(opts):
"""
support configuration for autotools projects
With provided configuration options (``RelengConfigureOptions``), the
configuration stage will be processed.
Args:
opts: configuration options
Returns:
``True`` if the configuration stage is completed; ``False`` otherwise
"""
# check if autoreconf
if opts._autotools_autoreconf:
verbose('configured to run autoreconf')
if not AUTORECONF.exists():
err('unable to configure package; autoreconf is not installed')
return False
if not AUTORECONF.execute(['--verbose']):
err('failed to prepare autotools project (autoreconf): {}',
opts.name)
return False
# definitions
autotools_defs = {
'--prefix': opts.prefix,
'--exec-prefix': opts.prefix,
}
if opts.conf_defs:
autotools_defs.update(expand(opts.conf_defs))
# default options
autotools_opts = {}
if opts.conf_opts:
autotools_opts.update(expand(opts.conf_opts))
# argument building
autotools_args = []
autotools_args.extend(prepare_definitions(autotools_defs))
autotools_args.extend(prepare_arguments(autotools_opts))
if not execute(['./configure'] + autotools_args,
env_update=expand(opts.conf_env),
critical=False):
err('failed to prepare autotools project (configure): {}', opts.name)
return False
return True
def build(opts):
"""
support building autotools projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if not MAKE.exists():
err('unable to build package; make is not installed')
return False
# definitions
autotools_defs = {}
if opts.build_defs:
autotools_defs.update(expand(opts.build_defs))
# default options
autotools_opts = {}
if opts.build_opts:
autotools_opts.update(expand(opts.build_opts))
# argument building
autotools_args = []
autotools_args.extend(prepare_definitions(autotools_defs))
autotools_args.extend(prepare_arguments(autotools_opts))
if opts.jobs > 1:
autotools_args.append('--jobs')
autotools_args.append(str(opts.jobs))
if not MAKE.execute(autotools_args, env=expand(opts.build_env)):
err('failed to build autotools project: {}', opts.name)
return False
return True
def test_utilstr_expand(self):
def assertExpand(self, obj, result, kv=None):
self.assertEqual(expand(obj, kv), result)
val = expand(None)
self.assertIsNone(val)
assertExpand(self, '', '')
assertExpand(self, 'this is a simple message',
'this is a simple message')
os.environ['__RELENGTEST'] = 'test'
assertExpand(self, '$__RELENGTEST', 'test')
assertExpand(self, 'A $__RELENGTEST Z', 'A test Z')
assertExpand(self, 'A $__RELENGTEST$__RELENGTEST Z', 'A testtest Z')
assertExpand(self, 'longer $__RELENGTEST string', 'longer test string')
assertExpand(self, 'a/$__RELENGTEST/b', 'a/test/b')
assertExpand(self, 'a-$__RELENGTEST-b', 'a-test-b')
assertExpand(self, ' $__RELENGTEST ', ' test ')
assertExpand(self, '${__RELENGTEST}', 'test')
assertExpand(self, 'A${__RELENGTEST}Z', 'AtestZ')
assertExpand(self, 'A${__RELENGTEST}${__RELENGTEST}Z', 'AtesttestZ')
assertExpand(self, 'longer ${__RELENGTEST} string',
'longer test string')
assertExpand(self, ' ${__RELENGTEST} ', ' test ')
assertExpand(self, '${invalid', '${invalid')
assertExpand(self, '${}ignored', 'ignored')
assertExpand(self, '$$escaped', '$escaped')
assertExpand(self,
'${__RELENGTEST}',
'override',
kv={'__RELENGTEST': 'override'})
assertExpand(self, ['a', 'b', 'c'], ['a', 'b', 'c'])
assertExpand(self, ['${__RELENGTEST}', 'b', '${__RELENGTEST}'],
['test', 'b', 'test'])
assertExpand(self, {'a', 'b', 'c'}, {'a', 'b', 'c'})
assertExpand(self, {'a', '${__RELENGTEST}', 'c'}, {'a', 'test', 'c'})
assertExpand(self, {'key': 'value'}, {'key': 'value'})
assertExpand(self, {'${__RELENGTEST}': '${__RELENGTEST}'},
{'test': 'test'})
os.environ.pop('__RELENGTEST', None)
assertExpand(self, '$__RELENGTEST', '')
def fetch(opts):
"""
support fetching from rsync sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
site = opts.site
work_dir = opts.work_dir
cache_basename = os.path.basename(cache_file)
cache_stem, __ = interpret_stem_extension(cache_basename)
if not RSYNC.exists():
err('unable to fetch package; rsync is not installed')
return None
note('fetching {}...', name)
sys.stdout.flush()
# options
fetch_opts = {
'--recursive': '', # default recursive call
}
if opts.extra_opts:
fetch_opts.update(expand(opts.extra_opts))
# argument building
fetch_args = []
fetch_args.extend(prepare_arguments(fetch_opts))
# sanity check provided arguments
for fetch_arg in fetch_args:
if '--remove-source-files' in fetch_arg:
err('option `--remove-source-files` not permitted')
return None
elif not fetch_arg.startswith('-'):
err('invalid fetch option provided:', fetch_arg)
return None
fetch_args.append(site) # source directory
fetch_args.append(work_dir) # destination directory
if not RSYNC.execute(fetch_args, cwd=work_dir):
err('unable to rsync from source')
return None
log('successfully invoked rsync for source')
with tarfile.open(cache_file, 'w:gz') as tar:
tar.add(work_dir, arcname=cache_stem)
return cache_file
def _fetch(self, key, default=None, allow_expand=False, expand_extra=None):
"""
fetch a configuration value from a provided key
Package definitions will define one or more key-value configurations for
the release engineering process to use. For specific keys, there will be
expected value types where a key is provided. The fetch operation can be
provided a key and will return the desired value; however, if the value
is of a value that is not supported, an exception
``RelengToolInvalidPackageKeyValue`` is raised.
Args:
key: the key
default (optional): default value to use if the key does not exist
allow_expand (optional): whether or not to expand the value
expand_extra (optional): extra expand defines to use
Returns:
the value
Raises:
RelengToolInvalidPackageKeyValue: value type is invalid for the key
"""
self._active_key = key
type_ = self._key_types[key]
value = default
pkg_name = self._active_package
pkg_key_ = pkg_key(pkg_name, key)
def raise_kv_exception(type_):
raise RelengToolInvalidPackageKeyValue({
'pkg_name': pkg_name,
'pkg_key': pkg_key_,
'expected_type': type_,
})
# check if this package key has been explicitly overridden; if so,
# use its contents for the raw value to process
raw_value = self.opts.injected_kv.get(pkg_key_, None)
# if no raw value was injected, pull the key's value (if any) from the
# active environment
if raw_value is None:
raw_value = self._active_env.get(pkg_key_, None)
if raw_value is not None:
if type_ == PkgKeyType.BOOL:
value = raw_value
if not isinstance(value, bool):
raise_kv_exception('bool')
elif type_ == PkgKeyType.DICT:
value = raw_value
if allow_expand:
value = expand(value, expand_extra)
if not isinstance(value, dict):
raise_kv_exception('dictionary')
elif type_ == PkgKeyType.DICT_STR_STR:
value = interpret_dictionary_strings(raw_value)
if allow_expand:
value = expand(value, expand_extra)
if value is None:
raise_kv_exception('dict(str,str)')
elif type_ == PkgKeyType.DICT_STR_STR_OR_STRS:
value = interpret_zero_to_one_strings(raw_value)
if allow_expand:
value = expand(value, expand_extra)
if value is None:
raise_kv_exception('dict(str,str) or string(s)')
elif type_ == PkgKeyType.STR:
value = interpret_string(raw_value)
if allow_expand:
value = expand(value, expand_extra)
if value is None:
raise_kv_exception('string')
elif type_ == PkgKeyType.STRS:
value = interpret_strings(raw_value)
if allow_expand:
value = expand(value, expand_extra)
if value is None:
raise_kv_exception('string(s)')
elif type_ == PkgKeyType.INT_NONNEGATIVE:
value = raw_value
if not isinstance(value, int) or value < 0:
raise_kv_exception('non-negative int')
elif type_ == PkgKeyType.INT_POSITIVE:
value = raw_value
if not isinstance(value, int) or value <= 0:
raise_kv_exception('positive int')
else:
raise_kv_exception('<unsupported key-value>')
return value
def assertExpand(self, obj, result, kv=None):
self.assertEqual(expand(obj, kv), result)
def install(opts):
"""
support installation python projects
With provided installation options (``RelengInstallOptions``), the
installation stage will be processed.
Args:
opts: installation options
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
if opts._python_interpreter:
python_tool = PythonTool(opts._python_interpreter,
env_include=PYTHON_EXTEND_ENV)
else:
python_tool = PYTHON
if not python_tool.exists():
err('unable to install package; python is not installed')
return False
# definitions
python_defs = {
'--prefix': opts.prefix,
}
if opts.install_defs:
python_defs.update(expand(opts.install_defs))
# always remove the prefix value if:
# - *nix: setup.py may ignore provided `--root` value with an "/" prefix
# - win32: does not use the prefix value
if python_defs['--prefix'] == '/' or sys.platform == 'win32':
del python_defs['--prefix']
# default options
python_opts = {}
if opts.install_opts:
python_opts.update(expand(opts.install_opts))
# argument building
python_args = [
'setup.py',
# ignore user's pydistutils.cfg
'--no-user-cfg',
# invoke the install operation
'install',
# avoid building pyc files
'--no-compile',
]
python_args.extend(prepare_definitions(python_defs))
python_args.extend(prepare_arguments(python_opts))
# install to target destination(s)
#
# If the package already defines a root path, use it over any other
# configured destination directories.
env = expand(opts.install_env)
if '--root' in python_opts:
if not python_tool.execute(python_args, env=env):
err('failed to install python project: {}', opts.name)
return False
else:
# install to each destination
for dest_dir in opts.dest_dirs:
if not python_tool.execute(python_args + ['--root', dest_dir],
env=env):
err('failed to install python project: {}', opts.name)
return False
return True
