def install(opts): """ support installation cmake projects With provided installation options (``RelengInstallOptions``), the installation stage will be processed. Args: opts: installation options Returns: ``True`` if the installation stage is completed; ``False`` otherwise """ if not CMAKE.exists(): err('unable to install package; cmake is not installed') return False # check if the no-install flag is set if opts._cmake_noinstall: verbose('configured to skip install stage for cmake') return True # default definitions cmake_defs = {} if opts.install_defs: cmake_defs.update(expand(opts.install_defs)) # default options cmake_opts = { # build RelWithDebInfo (when using multi-configuration projects) '--config': 'RelWithDebInfo', # default install using the install target '--target': 'install', } if opts.install_opts: cmake_opts.update(expand(opts.install_opts)) # argument building cmake_args = [ '--build', opts.build_output_dir, ] cmake_args.extend(prepare_definitions(cmake_defs, '-D')) cmake_args.extend(prepare_arguments(cmake_opts)) # prepare environment for installation request; an environment dictionary is # always needed to apply a custom DESTDIR during each install request env = expand(opts.install_env) if not env: env = {} # install to each destination for dest_dir in opts.dest_dirs: env['DESTDIR'] = dest_dir if not CMAKE.execute(cmake_args, env=env): err('failed to install cmake project: {}', opts.name) return False return True
def build(opts): """ support building python projects With provided build options (``RelengBuildOptions``), the build stage will be processed. Args: opts: build options Returns: ``True`` if the building stage is completed; ``False`` otherwise """ if opts._python_interpreter: python_tool = PythonTool(opts._python_interpreter, env_include=PYTHON_EXTEND_ENV) else: python_tool = PYTHON if not python_tool.exists(): err('unable to build package; python is not installed') return False # definitions python_defs = {} if opts.build_defs: python_defs.update(expand(opts.build_defs)) # default options python_opts = {} if opts.build_opts: python_opts.update(expand(opts.build_opts)) # default environment path1 = python_tool.path(sysroot=opts.staging_dir, prefix=opts.prefix) path2 = python_tool.path(sysroot=opts.target_dir, prefix=opts.prefix) env = {'PYTHONPATH': path1 + os.pathsep + path2} # apply package-specific environment options if opts.build_env: env.update(expand(opts.build_env)) # argument building python_args = [ 'setup.py', # ignore user's pydistutils.cfg '--no-user-cfg', # invoke the build operation 'build', ] python_args.extend(prepare_definitions(python_defs)) python_args.extend(prepare_arguments(python_opts)) if not python_tool.execute(python_args, env=env): err('failed to build python project: {}', opts.name) return False return True
def build(opts): """ support building cmake projects With provided build options (``RelengBuildOptions``), the build stage will be processed. Args: opts: build options Returns: ``True`` if the building stage is completed; ``False`` otherwise """ if not CMAKE.exists(): err('unable to build package; cmake is not installed') return False # definitions cmake_defs = {} if opts.build_defs: cmake_defs.update(expand(opts.build_defs)) # options cmake_opts = { # build RelWithDebInfo (when using multi-configuration projects) '--config': 'RelWithDebInfo', } if opts.build_opts: cmake_opts.update(expand(opts.build_opts)) # argument building cmake_args = [ # tell cmake to invoke build process in the output directory '--build', opts.build_output_dir, ] cmake_args.extend(prepare_definitions(cmake_defs, '-D')) cmake_args.extend(prepare_arguments(cmake_opts)) # enable specific number of parallel jobs is set # # https://cmake.org/cmake/help/v3.12/manual/cmake.1.html#build-tool-mode if 'releng.cmake.disable_parallel_option' not in opts._quirks: if opts.jobsconf != 1 and opts.jobs > 1: cmake_args.append('--parallel') cmake_args.append(str(opts.jobs)) else: verbose('cmake parallel jobs disabled by quirk') if not CMAKE.execute(cmake_args, env=expand(opts.build_env)): err('failed to build cmake project: {}', opts.name) return False return True
def configure(opts): """ support configuration for autotools projects With provided configuration options (``RelengConfigureOptions``), the configuration stage will be processed. Args: opts: configuration options Returns: ``True`` if the configuration stage is completed; ``False`` otherwise """ # check if autoreconf if opts._autotools_autoreconf: verbose('configured to run autoreconf') if not AUTORECONF.exists(): err('unable to configure package; autoreconf is not installed') return False if not AUTORECONF.execute(['--verbose']): err('failed to prepare autotools project (autoreconf): {}', opts.name) return False # definitions autotools_defs = { '--prefix': opts.prefix, '--exec-prefix': opts.prefix, } if opts.conf_defs: autotools_defs.update(expand(opts.conf_defs)) # default options autotools_opts = {} if opts.conf_opts: autotools_opts.update(expand(opts.conf_opts)) # argument building autotools_args = [] autotools_args.extend(prepare_definitions(autotools_defs)) autotools_args.extend(prepare_arguments(autotools_opts)) if not execute(['./configure'] + autotools_args, env_update=expand(opts.conf_env), critical=False): err('failed to prepare autotools project (configure): {}', opts.name) return False return True
def build(opts): """ support building autotools projects With provided build options (``RelengBuildOptions``), the build stage will be processed. Args: opts: build options Returns: ``True`` if the building stage is completed; ``False`` otherwise """ if not MAKE.exists(): err('unable to build package; make is not installed') return False # definitions autotools_defs = {} if opts.build_defs: autotools_defs.update(expand(opts.build_defs)) # default options autotools_opts = {} if opts.build_opts: autotools_opts.update(expand(opts.build_opts)) # argument building autotools_args = [] autotools_args.extend(prepare_definitions(autotools_defs)) autotools_args.extend(prepare_arguments(autotools_opts)) if opts.jobs > 1: autotools_args.append('--jobs') autotools_args.append(str(opts.jobs)) if not MAKE.execute(autotools_args, env=expand(opts.build_env)): err('failed to build autotools project: {}', opts.name) return False return True
def test_utilstr_expand(self): def assertExpand(self, obj, result, kv=None): self.assertEqual(expand(obj, kv), result) val = expand(None) self.assertIsNone(val) assertExpand(self, '', '') assertExpand(self, 'this is a simple message', 'this is a simple message') os.environ['__RELENGTEST'] = 'test' assertExpand(self, '$__RELENGTEST', 'test') assertExpand(self, 'A $__RELENGTEST Z', 'A test Z') assertExpand(self, 'A $__RELENGTEST$__RELENGTEST Z', 'A testtest Z') assertExpand(self, 'longer $__RELENGTEST string', 'longer test string') assertExpand(self, 'a/$__RELENGTEST/b', 'a/test/b') assertExpand(self, 'a-$__RELENGTEST-b', 'a-test-b') assertExpand(self, ' $__RELENGTEST ', ' test ') assertExpand(self, '${__RELENGTEST}', 'test') assertExpand(self, 'A${__RELENGTEST}Z', 'AtestZ') assertExpand(self, 'A${__RELENGTEST}${__RELENGTEST}Z', 'AtesttestZ') assertExpand(self, 'longer ${__RELENGTEST} string', 'longer test string') assertExpand(self, ' ${__RELENGTEST} ', ' test ') assertExpand(self, '${invalid', '${invalid') assertExpand(self, '${}ignored', 'ignored') assertExpand(self, '$$escaped', '$escaped') assertExpand(self, '${__RELENGTEST}', 'override', kv={'__RELENGTEST': 'override'}) assertExpand(self, ['a', 'b', 'c'], ['a', 'b', 'c']) assertExpand(self, ['${__RELENGTEST}', 'b', '${__RELENGTEST}'], ['test', 'b', 'test']) assertExpand(self, {'a', 'b', 'c'}, {'a', 'b', 'c'}) assertExpand(self, {'a', '${__RELENGTEST}', 'c'}, {'a', 'test', 'c'}) assertExpand(self, {'key': 'value'}, {'key': 'value'}) assertExpand(self, {'${__RELENGTEST}': '${__RELENGTEST}'}, {'test': 'test'}) os.environ.pop('__RELENGTEST', None) assertExpand(self, '$__RELENGTEST', '')
def fetch(opts): """ support fetching from rsync sources With provided fetch options (``RelengFetchOptions``), the fetch stage will be processed. Args: opts: fetch options Returns: ``True`` if the fetch stage is completed; ``False`` otherwise """ assert opts cache_file = opts.cache_file name = opts.name site = opts.site work_dir = opts.work_dir cache_basename = os.path.basename(cache_file) cache_stem, __ = interpret_stem_extension(cache_basename) if not RSYNC.exists(): err('unable to fetch package; rsync is not installed') return None note('fetching {}...', name) sys.stdout.flush() # options fetch_opts = { '--recursive': '', # default recursive call } if opts.extra_opts: fetch_opts.update(expand(opts.extra_opts)) # argument building fetch_args = [] fetch_args.extend(prepare_arguments(fetch_opts)) # sanity check provided arguments for fetch_arg in fetch_args: if '--remove-source-files' in fetch_arg: err('option `--remove-source-files` not permitted') return None elif not fetch_arg.startswith('-'): err('invalid fetch option provided:', fetch_arg) return None fetch_args.append(site) # source directory fetch_args.append(work_dir) # destination directory if not RSYNC.execute(fetch_args, cwd=work_dir): err('unable to rsync from source') return None log('successfully invoked rsync for source') with tarfile.open(cache_file, 'w:gz') as tar: tar.add(work_dir, arcname=cache_stem) return cache_file
def _fetch(self, key, default=None, allow_expand=False, expand_extra=None): """ fetch a configuration value from a provided key Package definitions will define one or more key-value configurations for the release engineering process to use. For specific keys, there will be expected value types where a key is provided. The fetch operation can be provided a key and will return the desired value; however, if the value is of a value that is not supported, an exception ``RelengToolInvalidPackageKeyValue`` is raised. Args: key: the key default (optional): default value to use if the key does not exist allow_expand (optional): whether or not to expand the value expand_extra (optional): extra expand defines to use Returns: the value Raises: RelengToolInvalidPackageKeyValue: value type is invalid for the key """ self._active_key = key type_ = self._key_types[key] value = default pkg_name = self._active_package pkg_key_ = pkg_key(pkg_name, key) def raise_kv_exception(type_): raise RelengToolInvalidPackageKeyValue({ 'pkg_name': pkg_name, 'pkg_key': pkg_key_, 'expected_type': type_, }) # check if this package key has been explicitly overridden; if so, # use its contents for the raw value to process raw_value = self.opts.injected_kv.get(pkg_key_, None) # if no raw value was injected, pull the key's value (if any) from the # active environment if raw_value is None: raw_value = self._active_env.get(pkg_key_, None) if raw_value is not None: if type_ == PkgKeyType.BOOL: value = raw_value if not isinstance(value, bool): raise_kv_exception('bool') elif type_ == PkgKeyType.DICT: value = raw_value if allow_expand: value = expand(value, expand_extra) if not isinstance(value, dict): raise_kv_exception('dictionary') elif type_ == PkgKeyType.DICT_STR_STR: value = interpret_dictionary_strings(raw_value) if allow_expand: value = expand(value, expand_extra) if value is None: raise_kv_exception('dict(str,str)') elif type_ == PkgKeyType.DICT_STR_STR_OR_STRS: value = interpret_zero_to_one_strings(raw_value) if allow_expand: value = expand(value, expand_extra) if value is None: raise_kv_exception('dict(str,str) or string(s)') elif type_ == PkgKeyType.STR: value = interpret_string(raw_value) if allow_expand: value = expand(value, expand_extra) if value is None: raise_kv_exception('string') elif type_ == PkgKeyType.STRS: value = interpret_strings(raw_value) if allow_expand: value = expand(value, expand_extra) if value is None: raise_kv_exception('string(s)') elif type_ == PkgKeyType.INT_NONNEGATIVE: value = raw_value if not isinstance(value, int) or value < 0: raise_kv_exception('non-negative int') elif type_ == PkgKeyType.INT_POSITIVE: value = raw_value if not isinstance(value, int) or value <= 0: raise_kv_exception('positive int') else: raise_kv_exception('<unsupported key-value>') return value
def assertExpand(self, obj, result, kv=None): self.assertEqual(expand(obj, kv), result)
def install(opts): """ support installation python projects With provided installation options (``RelengInstallOptions``), the installation stage will be processed. Args: opts: installation options Returns: ``True`` if the installation stage is completed; ``False`` otherwise """ if opts._python_interpreter: python_tool = PythonTool(opts._python_interpreter, env_include=PYTHON_EXTEND_ENV) else: python_tool = PYTHON if not python_tool.exists(): err('unable to install package; python is not installed') return False # definitions python_defs = { '--prefix': opts.prefix, } if opts.install_defs: python_defs.update(expand(opts.install_defs)) # always remove the prefix value if: # - *nix: setup.py may ignore provided `--root` value with an "/" prefix # - win32: does not use the prefix value if python_defs['--prefix'] == '/' or sys.platform == 'win32': del python_defs['--prefix'] # default options python_opts = {} if opts.install_opts: python_opts.update(expand(opts.install_opts)) # argument building python_args = [ 'setup.py', # ignore user's pydistutils.cfg '--no-user-cfg', # invoke the install operation 'install', # avoid building pyc files '--no-compile', ] python_args.extend(prepare_definitions(python_defs)) python_args.extend(prepare_arguments(python_opts)) # install to target destination(s) # # If the package already defines a root path, use it over any other # configured destination directories. env = expand(opts.install_env) if '--root' in python_opts: if not python_tool.execute(python_args, env=env): err('failed to install python project: {}', opts.name) return False else: # install to each destination for dest_dir in opts.dest_dirs: if not python_tool.execute(python_args + ['--root', dest_dir], env=env): err('failed to install python project: {}', opts.name) return False return True