def save(self, desc=None): """ save statistics for future reference Will save any statistics which should be persisted for future considerations. This is to help render a "complete" report of statistics when re-running releng-tool with packages which may already been completed. Args: desc (optional): description of this save event (for logging) """ if not ensure_dir_exists(self.out_dir): verbose('unable to generate output directory for statistics') return None if desc: desc = ' ({})'.format(desc) else: desc = '' try: with open(self.dat_file, 'wb') as f: pickle.dump(self.data, f, protocol=2) # 2 for py2/py3 support debug('saved statistics' + desc) except IOError: verbose('failed to save statistics' + desc)
def install(opts): """ support installation cmake projects With provided installation options (``RelengInstallOptions``), the installation stage will be processed. Args: opts: installation options Returns: ``True`` if the installation stage is completed; ``False`` otherwise """ if not CMAKE.exists(): err('unable to install package; cmake is not installed') return False # check if the no-install flag is set if opts._cmake_noinstall: verbose('configured to skip install stage for cmake') return True # default definitions cmake_defs = {} if opts.install_defs: cmake_defs.update(expand(opts.install_defs)) # default options cmake_opts = { # build RelWithDebInfo (when using multi-configuration projects) '--config': 'RelWithDebInfo', # default install using the install target '--target': 'install', } if opts.install_opts: cmake_opts.update(expand(opts.install_opts)) # argument building cmake_args = [ '--build', opts.build_output_dir, ] cmake_args.extend(prepare_definitions(cmake_defs, '-D')) cmake_args.extend(prepare_arguments(cmake_opts)) # prepare environment for installation request; an environment dictionary is # always needed to apply a custom DESTDIR during each install request env = expand(opts.install_env) if not env: env = {} # install to each destination for dest_dir in opts.dest_dirs: env['DESTDIR'] = dest_dir if not CMAKE.execute(cmake_args, env=env): err('failed to install cmake project: {}', opts.name) return False return True
def build(opts): """ support building cmake projects With provided build options (``RelengBuildOptions``), the build stage will be processed. Args: opts: build options Returns: ``True`` if the building stage is completed; ``False`` otherwise """ if not CMAKE.exists(): err('unable to build package; cmake is not installed') return False # definitions cmake_defs = {} if opts.build_defs: cmake_defs.update(expand(opts.build_defs)) # options cmake_opts = { # build RelWithDebInfo (when using multi-configuration projects) '--config': 'RelWithDebInfo', } if opts.build_opts: cmake_opts.update(expand(opts.build_opts)) # argument building cmake_args = [ # tell cmake to invoke build process in the output directory '--build', opts.build_output_dir, ] cmake_args.extend(prepare_definitions(cmake_defs, '-D')) cmake_args.extend(prepare_arguments(cmake_opts)) # enable specific number of parallel jobs is set # # https://cmake.org/cmake/help/v3.12/manual/cmake.1.html#build-tool-mode if 'releng.cmake.disable_parallel_option' not in opts._quirks: if opts.jobsconf != 1 and opts.jobs > 1: cmake_args.append('--parallel') cmake_args.append(str(opts.jobs)) else: verbose('cmake parallel jobs disabled by quirk') if not CMAKE.execute(cmake_args, env=expand(opts.build_env)): err('failed to build cmake project: {}', opts.name) return False return True
def _verbose_exists(self, tool): """ verbose log that a provided tool exists Will generate a verbose log which will indicate to a user that a provided tool has been detected on the host system. Args: tool: the tool """ verbose('prerequisite exists: ' + tool.tool)
def finalize_package(self, pkg, script): """ finalize configuration for a package Attempts to finalize any configuration entries of an already populated package instance with options provided at a later stage in the releng-tool process. This is to support projects where select configuration options are defined in the package's source content, instead of the main releng-tool project. This call will accept as package instance to update and the script file which may include a series of configuration options to apply to a package. Note that any configuration option already set on the package will be used over any new detected package option. Args: pkg: the package script: the package script to load Raises: RelengToolInvalidPackageConfiguration: when an error has been detected loading any of the package's extended options """ verbose('finalize package configuration: {}', pkg.name) debug('script {}', script) if not os.path.isfile(script): raise RelengToolMissingPackageScript({ 'pkg_name': pkg.name, 'script': script, }) try: env = run_script(script, self.script_env, catch=False) except Exception as e: raise RelengToolInvalidPackageScript({ 'description': str(e), 'script': script, 'traceback': traceback.format_exc(), }) # apply any options to unset configuration entries self._active_package = pkg.name self._active_env = env self._apply_postinit_options(pkg) # extend the active script environment if the post-init call succeeds extend_script_env(self.script_env, env)
def generate(self): """ generate a final report of statistics To be invoked at the end of a releng-tool process, this call will generate reports/etc. for any tracked statistics information based on the current and previous invoked executions (if any). """ if not ensure_dir_exists(self.out_dir): verbose('unable to generate output directory for statistics') return None self._generate_duration()
def configure(opts): """ support configuration for autotools projects With provided configuration options (``RelengConfigureOptions``), the configuration stage will be processed. Args: opts: configuration options Returns: ``True`` if the configuration stage is completed; ``False`` otherwise """ # check if autoreconf if opts._autotools_autoreconf: verbose('configured to run autoreconf') if not AUTORECONF.exists(): err('unable to configure package; autoreconf is not installed') return False if not AUTORECONF.execute(['--verbose']): err('failed to prepare autotools project (autoreconf): {}', opts.name) return False # definitions autotools_defs = { '--prefix': opts.prefix, '--exec-prefix': opts.prefix, } if opts.conf_defs: autotools_defs.update(expand(opts.conf_defs)) # default options autotools_opts = {} if opts.conf_opts: autotools_opts.update(expand(opts.conf_opts)) # argument building autotools_args = [] autotools_args.extend(prepare_definitions(autotools_defs)) autotools_args.extend(prepare_arguments(autotools_opts)) if not execute(['./configure'] + autotools_args, env_update=expand(opts.conf_env), critical=False): err('failed to prepare autotools project (configure): {}', opts.name) return False return True
def load(self): """ load any persisted statistics Will load any statistics which may have been persisted from a previous run. This is to help render a "complete" report of statistics when re-running releng-tool with packages which may already been completed. """ if not os.path.exists(self.dat_file): return try: with open(self.dat_file, 'rb') as f: self.data = pickle.load(f) debug('loaded statistics') except IOError: verbose('failed to load original statistics (io error)') except ValueError: verbose('failed to load original statistics (pickle error)')
def _load_dvcs_cache(self): """ load any persisted dvcs cache information DVCS can be cached and shared over multiple projects. The following loads any cached DVCS database stored in the project's output folder where may hint the folder name for a project's cache. """ if not self._dvcs_cache_enabled: return if os.path.exists(self._dvcs_cache_fname): try: with open(self._dvcs_cache_fname, 'rb') as f: self._dvcs_cache = pickle.load(f) debug('loaded dvcs cache database') except IOError: verbose('failed to load dvcs cache database (io error)') except ValueError: verbose('failed to load dvcs cache database (pickle error)')
def validate(self, asc, target): """ validate ascii-armored file against a target Accepting an ASCII-armored file, use gpg to validate the public key against the provided target. Args: asc: the asc file target: the target file Returns: ``True`` if the target has been validated; ``False`` otherwise """ rv, out = self.execute_rv('--verify', asc, target) if rv == 0: verbose('validated: {}', asc) elif out: log(out) return rv == 0
def _save_dvcs_cache(self): """ save dvcs cache information Will save any DVCS cache information which future runs of releng-tool can be used to hint where package cache data is stored. """ if not self._dvcs_cache_enabled: return if not ensure_dir_exists(self.opts.cache_dir): verbose('unable to generate output directory for dvcs cache') return try: with open(self._dvcs_cache_fname, 'wb') as f: pickle.dump(self._dvcs_cache, f, protocol=2) # 2 for py2/py3 support debug('saved dvcs cache') except IOError: verbose('failed to save dvcs cache')
def configure(opts): """ support configuration project-defined scripts With provided configuration options (``RelengConfigureOptions``), the configuration stage will be processed. Args: opts: configuration options Returns: ``True`` if the configuration stage is completed; ``False`` otherwise """ assert opts build_dir = opts.build_dir def_dir = opts.def_dir env = opts.env configure_script_filename = '{}-{}'.format(opts.name, CONFIGURE_SCRIPT) configure_script = os.path.join(def_dir, configure_script_filename) configure_script, configure_script_exists = opt_file(configure_script) if not configure_script_exists: if (opts._skip_remote_scripts or 'releng.disable_remote_scripts' in opts._quirks): return True configure_script_filename = '{}-{}'.format('releng', CONFIGURE_SCRIPT) configure_script = os.path.join(build_dir, configure_script_filename) configure_script, configure_script_exists = opt_file(configure_script) if not configure_script_exists: return True if not run_script(configure_script, env, subject='configure'): return False verbose('install script executed: ' + configure_script) return True
def stage(engine, pkg, script_env): """ handles the post-processing stage for a package With a provided engine and package instance, the post-processing stage will be processed. This stage is typically not advertised and is for advanced cases where a developer wishes to manipulate their build environment after package has completed each of its phases. Args: engine: the engine pkg: the package being built script_env: script environment information Returns: ``True`` if the post-processing stage is completed; ``False`` otherwise """ verbose('post-processing {}...', pkg.name) sys.stdout.flush() post_script_filename = '{}-{}'.format(pkg.name, POST_SCRIPT) post_script = os.path.join(pkg.def_dir, post_script_filename) post_script, post_script_exists = opt_file(post_script) if not post_script_exists: return True if pkg.build_subdir: build_dir = pkg.build_subdir else: build_dir = pkg.build_dir with interim_working_dir(build_dir): if not run_script(post_script, script_env, subject='post-processing'): return False verbose('post-processing script executed: ' + post_script) return True
def build(opts): """ support building project-defined scripts With provided build options (``RelengBuildOptions``), the build stage will be processed. Args: opts: build options Returns: ``True`` if the building stage is completed; ``False`` otherwise """ assert opts build_dir = opts.build_dir def_dir = opts.def_dir env = opts.env build_script_filename = '{}-{}'.format(opts.name, BUILD_SCRIPT) build_script = os.path.join(def_dir, build_script_filename) build_script, build_script_exists = opt_file(build_script) if not build_script_exists: if (opts._skip_remote_scripts or 'releng.disable_remote_scripts' in opts._quirks): return True build_script_filename = '{}-{}'.format('releng', BUILD_SCRIPT) build_script = os.path.join(build_dir, build_script_filename) build_script, build_script_exists = opt_file(build_script) if not build_script_exists: return True if not run_script(build_script, env, subject='build'): return False verbose('install script executed: ' + build_script) return True
def install(opts): """ support installation project-defined scripts With provided installation options (``RelengInstallOptions``), the installation stage will be processed. Args: opts: installation options Returns: ``True`` if the installation stage is completed; ``False`` otherwise """ assert opts build_dir = opts.build_dir def_dir = opts.def_dir env = opts.env install_script_filename = '{}-{}'.format(opts.name, INSTALL_SCRIPT) install_script = os.path.join(def_dir, install_script_filename) install_script, install_script_exists = opt_file(install_script) if not install_script_exists: if (opts._skip_remote_scripts or 'releng.disable_remote_scripts' in opts._quirks): return True install_script_filename = '{}-{}'.format('releng', INSTALL_SCRIPT) install_script = os.path.join(build_dir, install_script_filename) install_script, install_script_exists = opt_file(install_script) if not install_script_exists: return True if not run_script(install_script, env, subject='install'): return False verbose('install script executed: ' + install_script) return True
def _verify_revision(git_dir, revision, quiet=False): """ verify the gpg signature for a target revision The GPG signature for a provided revision (tag or commit) will be checked to validate the revision. Args: git_dir: the Git directory revision: the revision to verify quiet (optional): whether or not the log if verification is happening Returns: ``True`` if the revision is signed; ``False`` otherwise """ if not quiet: log('verifying the gpg signature on the target revision') else: verbose('verifying the gpg signature on the target revision') if GIT.execute( [git_dir, 'rev-parse', '--quiet', '--verify', revision + '^{tag}'], quiet=True): verified_cmd = 'verify-tag' else: verified_cmd = 'verify-commit' # acquire the commit if (if not already set), to ensure we can verify # against commits or branches rv, revision = GIT.execute_rv(git_dir, 'rev-parse', revision) if rv != 0: verbose('failed to determine the commit id for a revision') return False return GIT.execute([git_dir, verified_cmd, revision], quiet=quiet)
def initialize_sample(opts): """ initialize a sample project Generates a sample provided in the root directory to help new users or new project get started. Args: opts: options for this run Returns: ``True`` if the sample project could be initialized; ``False`` if an issue has occurred generating the sample project """ root_dir = opts.root_dir if not ensure_dir_exists(root_dir): return False if os.listdir(root_dir): err('unable to initialize sample project is non-empty directory') return False sample_dir = os.path.join(root_dir, 'package', 'sample') success = True if ensure_dir_exists(sample_dir): # sample project sample_defs = os.path.join(root_dir, 'package', 'sample', 'sample') try: with open(sample_defs, 'w') as f: f.write('''\ #!/usr/bin/env python # -*- coding: utf-8 -*- SAMPLE_DEPENDENCIES = [] SAMPLE_LICENSE = ['<license name>'] SAMPLE_LICENSE_FILES = ['<license file>'] SAMPLE_SITE = '<location for sources>' SAMPLE_TYPE = '<package-type>' SAMPLE_VERSION = '<package-version>' ''') verbose('written sample file') except IOError as e: err('unable to generate a sample file') verbose(str(e)) success = False else: success = False # .gitignore try: project_gitignore = os.path.join(root_dir, '.gitignore') # (assumption) with open(project_gitignore, 'w') as f: f.write('''\ # releng-tool /cache/ /dl/ /output/ .releng-flag-* ''') verbose('written .gitignore file') except IOError as e: err('unable to generate a .gitignore file') verbose(str(e)) success = False # releng project try: project_defs = os.path.join(root_dir, 'releng') with open(project_defs, 'w') as f: f.write('''\ #!/usr/bin/env python # -*- coding: utf-8 -*- packages = [ 'sample', ] ''') verbose('written releng file') except IOError as e: err('unable to generate a releng file') verbose(str(e)) success = False if success: log('initialized empty releng-tool project') else: warn('partially initialized a releng-tool project') return success
def _generate_duration(self): """ generate duration-related statistics When generating a statistics report, this call creating/adds information about durations which may have been captured. """ if 'duration' not in self.data: return durations = self.data['duration'] pkgs = list(durations.keys()) pkgs = sorted(pkgs) categories = set() for pkg_data in durations.values(): categories.update(pkg_data.keys()) categories = sorted(categories) ordered_categories = [ 'boot', 'fetch', 'extract', 'patch', 'configure', 'build', 'install', 'post', ] for ordered_category in ordered_categories: if ordered_category not in categories: ordered_categories.remove(ordered_category) for category in categories: if category not in ordered_categories: ordered_categories.append(category) categories = ordered_categories # duration statistics to csv verbose('generating duration statistics (csv)...') dur_csv = os.path.join(self.out_dir, 'durations.csv') try: with open(dur_csv, 'w') as f: # header f.write('# pkg') for category in categories: f.write(',' + category) f.write('\n') # data for pkg in pkgs: f.write(pkg) for category in categories: if category in durations[pkg]: value = durations[pkg][category] else: value = 0 f.write(',' + str(int(value))) f.write('\n') except IOError as e: verbose('failed to write duration statistics: {}', e) # duration statistics to plot (if available) if has_matplotlib: verbose('generating duration statistics (pdf)...') BAR_HEIGHT = 0.4 EXTRA_HEIGHT = 1 FIG_WIDTH = 10 fig_height_pkgs = (BAR_HEIGHT + EXTRA_HEIGHT) * len(pkgs) fig_height_total = (BAR_HEIGHT + EXTRA_HEIGHT) * (len(pkgs) + 1) figsize_pkgs = (FIG_WIDTH, fig_height_pkgs) figsize_total = (FIG_WIDTH, fig_height_total) fig_pkgs, ax_pkgs = plt.subplots(figsize=figsize_pkgs) fig_total, ax_total = plt.subplots(figsize=figsize_total) axs = [ax_pkgs, ax_total] figs = [fig_pkgs, fig_total] pkgs.reverse() pkgs_total = list(pkgs) pkgs_total.insert(0, 'total') offset = [0] * len(pkgs) offset_total = [0] * len(pkgs_total) for category in categories: width = [] width_total = [] total = 0 for pkg in pkgs: if category in durations[pkg]: duration = durations[pkg][category] width.append(duration) width_total.append(duration) total += duration else: width.append(0) width_total.append(0) width_total.insert(0, total) ax_pkgs.barh(pkgs, width, height=BAR_HEIGHT, left=offset, label=category) ax_total.barh(pkgs_total, width_total, height=BAR_HEIGHT, left=offset_total, label=category) offset = numpy.add(offset, width) offset_total = numpy.add(offset_total, width_total) # provide some spacing near the right xlim = int(math.ceil(max(offset) / 10.)) * 10 if xlim - max(offset) < 10: xlim += 10 ax_pkgs.set_xlim([0, xlim]) xlim_total = int(math.ceil(max(offset_total) / 10.)) * 10 if xlim_total - max(offset_total) < 10: xlim_total += 10 ax_total.set_xlim([0, xlim_total]) # labels for ax in axs: ax.set_title('Package Stage Durations') ax.set_xlabel('Duration (seconds)') ax.legend() ax.grid(axis='x', linestyle=':', linewidth=0.4) # ensure rotated labels state in render area for fig in figs: fig.tight_layout() # generate figures dur_pdf = os.path.join(self.out_dir, 'durations.pdf') fig_pkgs.savefig(dur_pdf) dur_pdf_total = os.path.join(self.out_dir, 'durations-total.pdf') fig_total.savefig(dur_pdf_total) # close/cleanup figures plt.close() else: debug('duration statistics plot not supported')
def _execute(args, cwd=None, env=None, env_update=None, quiet=None, critical=True, poll=False, capture=None): """ execute the provided command/arguments Runs the command described by ``args`` until completion. A caller can adjust the working directory of the executed command by explicitly setting the directory in ``cwd``. The execution request will return the command's return code as well as any captured output. The environment variables used on execution can be manipulated in two ways. First, the environment can be explicitly controlled by applying a new environment content using the ``env`` dictionary. Key of the dictionary will be used as environment variable names, whereas the respective values will be the respective environment variable's value. If ``env`` is not provided, the existing environment of the executing context will be used. Second, a caller can instead update the existing environment by using the ``env_update`` option. Like ``env``, the key-value pairs match to respective environment key-value pairs. The difference with this option is that the call will use the original environment values and update select values which match in the updated environment request. When ``env`` and ``env_update`` are both provided, ``env_update`` will be updated the options based off of ``env`` instead of the original environment of the caller. If ``critical`` is set to ``True`` and the execution fails for any reason, this call will issue a system exit (``SystemExit``). By default, the critical flag is enabled (i.e. ``critical=True``). In special cases, an executing process may not provide carriage returns/new lines to simple output processing. This can lead the output of a process to be undesirably buffered. To workaround this issue, the execution call can instead poll for output results by using the ``poll`` option with a value of ``True``. By default, polling is disabled with a value of ``False``. A caller may wish to capture the provided output from a process for examination. If a list is provided in the call argument ``capture``, the list will be populated with the output provided from an invoked process. Args: args: the list of arguments to execute cwd (optional): working directory to use env (optional): environment variables to use for the process env_update (optional): environment variables to append for the process quiet (optional): whether or not to suppress output (defaults to ``False``) critical (optional): whether or not to stop execution on failure (defaults to ``True``) poll (optional): force polling stdin/stdout for output data (defaults to ``False``) capture (optional): list to capture output into Returns: the return code of the execution request Raises: SystemExit: if the execution operation fails with ``critical=True`` """ # append provided environment updates (if any) to the provided or existing # environment dictionary final_env = None if env: final_env = dict(env) if env_update: if not final_env: final_env = os.environ.copy() final_env.update(env_update) # if quiet is undefined, default its state based on whether or not the # caller wishes to capture output to a list if quiet is None: quiet = capture is not None cmd_str = None rv = 1 if args: # force any `None` arguments to empty strings, as a subprocess request # will not accept it; ideally, a call should not be passing a `None` # entry, but providing flexibility when it has been done args = [arg if arg is not None else '' for arg in args] # attempt to always invoke using a script's interpreter (if any) to # help deal with long-path calls if sys.platform != 'win32': args = prepend_shebang_interpreter(args) # python 2.7 can have trouble with unicode environment variables; # forcing all values to an ascii type if final_env and sys.version_info[0] < 3: debug('detected python 2.7; sanity checking environment variables') for k, v in final_env.items(): if isinstance(v, unicode): # pylint: disable=E0602 # noqa: F821 final_env[k] = v.encode('ascii', 'replace') if is_verbose(): debug('(wd) {}', cwd if cwd else os.getcwd()) cmd_str = _cmd_args_to_str(args) verbose('invoking: ' + cmd_str) sys.stdout.flush() try: # check if this execution should poll (for carriage returns and new # lines); note if quiet mode is enabled, do not attempt to poll # since none of the output will be printed anyways. if poll and not quiet: debug('will poll process for output') bufsize = 0 universal_newlines = False else: bufsize = 1 universal_newlines = True proc = subprocess.Popen( args, bufsize=bufsize, cwd=cwd, env=final_env, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=universal_newlines, ) if bufsize == 0: line = bytearray() while True: c = proc.stdout.read(1) if not c and proc.poll() is not None: break line += c if c == b'\r' or c == b'\n': decoded_line = line.decode('utf_8') if c == b'\n' and capture is not None: capture.append(decoded_line) if not quiet: sys.stdout.write(decoded_line) sys.stdout.flush() del line[:] else: for line in iter(proc.stdout.readline, ''): if capture is not None or not quiet: line = line.rstrip() if capture is not None: capture.append(line) if not quiet: print(line) sys.stdout.flush() proc.communicate() rv = proc.returncode except OSError as e: if not quiet: if not cmd_str: cmd_str = _cmd_args_to_str(args) err('unable to execute command: {}\n' ' {}', cmd_str, e) if rv != 0: if critical: if args: cmd_str = _cmd_args_to_str(args) else: cmd_str = '<empty>' err('failed to issue command: ' + cmd_str) # trigger a hard stop sys.exit(-1) elif args: debug('failed to issue last command') else: debug('failed to issue an empty command') return rv
def stage(engine, pkg, ignore_cache, extra_opts): """ handles the fetching stage for a package With a provided engine and package instance, the fetching stage will be processed. Args: engine: the engine pkg: the package being fetched ignore_cache: always attempt to ignore the cache extra_opts: extra options for the fetch operation (if applicable) Returns: ``True`` if the fetching stage is completed; ``False`` otherwise """ assert pkg.vcs_type name = pkg.name debug('process fetch stage: ' + name) # packages flagged for local sources requires to be already checked out if pkg.local_srcs: if os.path.isdir(pkg.build_dir): return True err( '''\ missing local sources for internal package: {0} The active configuration is flagged for 'local sources' mode; however, an internal package cannot be found in the local system. Before continuing, ensure you have checked out all internal packages on your local system (or, disable the local sources option to use the default process). Package: {0} Expected Path: {1}''', name, pkg.build_dir) return False # if the vcs-type is archive-based, flag that hash checks are needed perform_file_asc_check = False perform_file_hash_check = False if pkg.vcs_type == VcsType.URL: perform_file_asc_check = os.path.exists(pkg.asc_file) perform_file_hash_check = True fetch_opts = RelengFetchOptions() replicate_package_attribs(fetch_opts, pkg) fetch_opts.cache_dir = pkg.cache_dir fetch_opts.ext = pkg.ext_modifiers fetch_opts.extra_opts = extra_opts fetch_opts.ignore_cache = ignore_cache fetch_opts.name = name fetch_opts.revision = pkg.revision fetch_opts.site = pkg.site fetch_opts.version = pkg.version fetch_opts._mirror = False fetch_opts._quirks = engine.opts.quirks fetch_opts._urlopen_context = engine.opts.urlopen_context cache_filename = os.path.basename(pkg.cache_file) out_dir = engine.opts.out_dir with generate_temp_dir(out_dir) as work_dir, \ generate_temp_dir(out_dir) as interim_cache_dir: with interim_working_dir(work_dir): interim_cache_file = os.path.join(interim_cache_dir, cache_filename) fetch_opts.cache_file = interim_cache_file fetch_opts.work_dir = work_dir # check if file caching should be ignored # # In special cases, a developer may configure a project to have a # fetched source not to cache. For example, pulling from a branch of # a VCS source will make a cache file from the branch and will # remain until manually removed from a cache file. A user may wish # to re-build the local cache file after cleaning their project. # While the releng-tool framework separates fetching/extraction into # two parts, ignoring cached assets can be partially achieved by # just removing any detected cache file if a project is configured # to ignore a cache file. if engine.opts.devmode and pkg.devmode_ignore_cache is not None: fetch_opts.ignore_cache = pkg.devmode_ignore_cache if pkg.devmode_ignore_cache and os.path.exists(pkg.cache_file): verbose('removing cache file (per configuration): ' + name) if not path_remove(pkg.cache_file): return False # remove cache file if there is a force request to ignore the cache elif engine.opts.force and ignore_cache: if os.path.exists(pkg.cache_file): verbose('removing cache file (forced): ' + name) if not path_remove(pkg.cache_file): return False # force explicit ignore cache (to off) when not in development mode elif not engine.opts.devmode and ignore_cache is None: fetch_opts.ignore_cache = False if os.path.exists(pkg.cache_file): rv = None if perform_file_hash_check: hr = verify_hashes(pkg.hash_file, pkg.cache_file, relaxed=True) if hr == HashResult.VERIFIED: rv = True elif hr == HashResult.BAD_PATH: if not perform_file_asc_check and not pkg.is_internal: warn('missing hash file for package: ' + name) rv = True # no hash file to compare with; assuming ok elif hr == HashResult.EMPTY: if not pkg.is_internal: warn('hash file for package is empty: ' + name) rv = True # empty hash file; assuming ok elif hr == HashResult.MISMATCH: if not path_remove(pkg.cache_file): rv = False elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED): rv = False elif hr == HashResult.MISSING_ARCHIVE: if not perform_file_asc_check: err( '''\ missing archive hash for verification The hash file for this package does not have an entry for the cache file to be verified. Ensure the hash file defines an entry for the expected cache file: Hash File: {} File: {}''', pkg.hash_file, cache_filename) rv = False else: err( 'invalid fetch operation (internal error; ' 'hash-check failure: {})', hr) rv = False else: rv = True if rv is not False and perform_file_asc_check and \ os.path.exists(pkg.cache_file): if GPG.validate(pkg.asc_file, pkg.cache_file): rv = True else: if not path_remove(pkg.cache_file): err( '''\ failed to validate against ascii-armor Validation of a package resource failed to verify against a provided ASCII-armor file. Ensure that the package's public key has been registered into gpg. ASC File: {} File: {}''', pkg.asc_file, cache_filename) rv = False else: rv = None if rv is not None: if ignore_cache: verbose('ignoring cache not supported for package: {}', name) return rv # find fetching method for the target vcs-type fetcher = None if pkg.vcs_type in engine.registry.fetch_types: def _(opts): return engine.registry.fetch_types[pkg.vcs_type].fetch( pkg.vcs_type, opts) fetcher = _ elif pkg.vcs_type == VcsType.BZR: fetcher = fetch_bzr elif pkg.vcs_type == VcsType.CVS: fetcher = fetch_cvs elif pkg.vcs_type == VcsType.GIT: fetcher = fetch_git elif pkg.vcs_type == VcsType.HG: fetcher = fetch_mercurial elif pkg.vcs_type == VcsType.RSYNC: fetcher = fetch_rsync elif pkg.vcs_type == VcsType.SCP: fetcher = fetch_scp elif pkg.vcs_type == VcsType.SVN: fetcher = fetch_svn elif pkg.vcs_type == VcsType.URL: fetcher = fetch_url if not fetcher: err('fetch type is not implemented: {}', pkg.vcs_type) return False # if this is url-type location, attempt to search on the mirror # first (if configured) fetched = None if engine.opts.url_mirror and pkg.vcs_type == VcsType.URL: original_site = fetch_opts.site new_site = engine.opts.url_mirror + cache_filename if original_site != new_site: fetch_opts._mirror = True fetch_opts.site = new_site fetched = fetcher(fetch_opts) fetch_opts.site = original_site fetch_opts._mirror = False # perform the fetch request (if not already fetched) if not fetched: fetched = fetcher(fetch_opts) if not fetched: return False # if the fetch type has populated the package's cache directory # directly, we are done if fetched == pkg.cache_dir: pass # if the fetch type has returned a file, the file needs to be hash # checked and then be moved into the download cache elif fetched == interim_cache_file: if perform_file_hash_check: hr = verify_hashes(pkg.hash_file, fetched) if hr == HashResult.VERIFIED: pass elif hr == HashResult.BAD_PATH: if not perform_file_asc_check and not pkg.is_internal: warn('missing hash file for package: ' + name) elif hr == HashResult.EMPTY: if not pkg.is_internal: warn('hash file for package is empty: ' + name) elif hr == HashResult.MISMATCH: return False elif hr in (HashResult.BAD_FORMAT, HashResult.UNSUPPORTED): return False elif hr == HashResult.MISSING_ARCHIVE: if not perform_file_asc_check: err( '''\ missing archive hash for verification The hash file for this package does not have an entry for the cache file to be verified. Ensure the hash file defines an entry for the expected cache file: Hash File: {} File: {}''', pkg.hash_file, cache_filename) return False else: err( 'invalid fetch operation (internal error; ' 'hash-check failure: {})', hr) return False if perform_file_asc_check: if not GPG.validate(pkg.asc_file, interim_cache_file): err( '''\ failed to validate against ascii-armor Validation of a package resource failed to verify against a provided ASCII-armor file. Ensure that the package's public key has been registered into gpg. ASC File: {} File: {}''', pkg.asc_file, cache_filename) return False debug('fetch successful; moving cache file') # ensure the cache container/directory exists cache_dir = os.path.dirname(pkg.cache_file) if not ensure_dir_exists(cache_dir): return False try: shutil.move(interim_cache_file, pkg.cache_file) except shutil.Error: err( 'invalid fetch operation (internal error; fetch mode ' '"{}" has provided a missing cache file)', pkg.vcs_type) return False else: err( 'invalid fetch operation (internal error; fetch mode "{}" ' 'has returned an unsupported value)', pkg.vcs_type) return False return True
def _fetch_submodules(opts, cache_dir, revision): """ fetch the submodules on a provided cache/bar repository Using a provided bare repository, submodules configured at the provided revision will be fetched into the bare repository's modules directory. If it has been detected that a submodule contains additional submodules, they will also be fetched into a cache directory. Args: opts: fetch options cache_dir: the cache/bare repository revision: the revision (branch, tag, hash) to fetch Returns: ``True`` if submodules have been processed; ``False`` otherwise """ assert revision git_dir = '--git-dir=' + cache_dir # find a .gitmodules configuration on the target revision submodule_ref = '{}:.gitmodules'.format(revision) rv, raw_submodules = GIT.execute_rv(git_dir, 'show', submodule_ref) if rv != 0: submodule_ref = 'origin/' + submodule_ref rv, raw_submodules = GIT.execute_rv(git_dir, 'show', submodule_ref) if rv != 0: verbose('no git submodules file detected for this revision') return True debug('parsing git submodules file...') cfg = GIT.parse_cfg_str(raw_submodules) if not cfg: verbose('no git submodules file detected for this revision') return False for sec_name in cfg.sections(): if not sec_name.startswith('submodule'): continue if not cfg.has_option(sec_name, 'path') or \ not cfg.has_option(sec_name, 'url'): debug('submodule section missing path/url') continue submodule_path = cfg.get(sec_name, 'path') submodule_revision = None if cfg.has_option(sec_name, 'branch'): submodule_revision = cfg.get(sec_name, 'branch') submodule_url = cfg.get(sec_name, 'url') verbose('detected submodule: {}', submodule_path) debug('submodule revision: {}', submodule_revision if submodule_revision else '(none)') debug('submodule url: {}', submodule_url) ckey = pkg_cache_key(submodule_url) root_cache_dir = os.path.abspath( os.path.join(opts.cache_dir, os.pardir)) submodule_cache_dir = os.path.join(root_cache_dir, ckey) verbose('submodule_cache_dir: {}', submodule_cache_dir) # check to make sure the submodule's path isn't pointing to a relative # path outside the expected cache base check_abs = os.path.abspath(submodule_cache_dir) check_common = os.path.commonprefix((submodule_cache_dir, check_abs)) if check_abs != check_common: err('unable to process submodule pathed outside of bare repository' ) verbose('submodule expected base: {}', check_common) verbose('submodule absolute path: {}', check_abs) return False # fetch/cache the submodule repository if not _fetch_submodule(opts, submodule_path, submodule_cache_dir, submodule_revision, submodule_url): return False # if a revision is not provided, extract the HEAD from the cache if not submodule_revision: submodule_revision = GIT.extract_submodule_revision( submodule_cache_dir) if not submodule_revision: return False # process nested submodules if not _fetch_submodules(opts, submodule_cache_dir, submodule_revision): return False return True
def _fetch_srcs(opts, cache_dir, revision, desc=None, refspecs=None): """ invokes a git fetch call of the configured origin into a bare repository With a provided cache directory (``cache_dir``; bare repository), fetch the contents of a configured origin into the directory. The fetch call will use a restricted depth, unless configured otherwise. In the event a target revision cannot be found (if provided), an unshallow fetch will be made. This call may be invoked without a revision provided -- specifically, this can occur for submodule configurations which do not have a specific revision explicitly set. Args: opts: fetch options cache_dir: the bare repository to fetch into revision: expected revision desired from the repository desc (optional): description to use for error message refspecs (optional): additional refspecs to add to the fetch call Returns: ``True`` if the fetch was successful; ``False`` otherwise """ git_dir = '--git-dir=' + cache_dir if not desc: desc = 'repository: {}'.format(opts.name) log('fetching most recent sources') prepared_fetch_cmd = [ git_dir, 'fetch', '--progress', '--prune', 'origin', ] # limit fetch depth target_depth = 1 if opts._git_depth is not None: target_depth = opts._git_depth limited_fetch = (target_depth and 'releng.git.no_depth' not in opts._quirks) depth_cmds = [ '--depth', str(target_depth), ] # if a revision is provided, first attempt to do a revision-specific fetch quick_fetch = 'releng.git.no_quick_fetch' not in opts._quirks if revision and quick_fetch: ls_cmd = [ 'ls-remote', '--exit-code', 'origin', ] debug('checking if tag exists on remote') if GIT.execute(ls_cmd + ['--tags', 'refs/tags/{}'.format(revision)], cwd=cache_dir, quiet=True): debug('attempting a tag reference fetch operation') fetch_cmd = list(prepared_fetch_cmd) fetch_cmd.append('+refs/tags/{0}:refs/tags/{0}'.format(revision)) if limited_fetch: fetch_cmd.extend(depth_cmds) if GIT.execute(fetch_cmd, cwd=cache_dir): debug('found the reference') return True debug('checking if reference exists on remote') if GIT.execute(ls_cmd + ['--heads', 'refs/heads/{}'.format(revision)], cwd=cache_dir, quiet=True): debug('attempting a head reference fetch operation') fetch_cmd = list(prepared_fetch_cmd) fetch_cmd.append( '+refs/heads/{0}:refs/remotes/origin/{0}'.format(revision)) if limited_fetch: fetch_cmd.extend(depth_cmds) if GIT.execute(fetch_cmd, cwd=cache_dir): debug('found the reference') return True # fetch standard (and configured) refspecs std_refspecs = [ '+refs/heads/*:refs/remotes/origin/*', '+refs/tags/*:refs/tags/*', ] prepared_fetch_cmd.extend(std_refspecs) # allow fetching addition references if configured (e.g. pull requests) if refspecs: for ref in refspecs: prepared_fetch_cmd.append( '+refs/{0}:refs/remotes/origin/{0}'.format(ref)) fetch_cmd = list(prepared_fetch_cmd) if limited_fetch: fetch_cmd.extend(depth_cmds) if not GIT.execute(fetch_cmd, cwd=cache_dir): err('unable to fetch branches/tags from remote repository') return False if revision: verbose('verifying target revision exists') exists_state = revision_exists(git_dir, revision) if exists_state in REVISION_EXISTS: pass elif (exists_state == GitExistsType.MISSING_HASH and limited_fetch and opts._git_depth is None): warn('failed to find hash on depth-limited fetch; fetching all...') fetch_cmd = list(prepared_fetch_cmd) fetch_cmd.append('--unshallow') if not GIT.execute(fetch_cmd, cwd=cache_dir): err('unable to unshallow fetch state') return False if revision_exists(git_dir, revision) not in REVISION_EXISTS: err( 'unable to find matching revision in {}\n' ' (revision: {})', desc, revision) return False else: err('unable to find matching revision in {}\n' 'revision: {})', desc, revision) return False return True
def stage(engine, pkg): """ handles the extraction stage for a package With a provided engine and package instance, the extraction stage will be processed. Args: engine: the engine pkg: the package being extracted Returns: ``True`` if the extraction stage is completed; ``False`` otherwise """ # packages flagged for local sources do not have an extraction stage if pkg.local_srcs: return True # skip packages flagged not to extract if pkg.no_extraction: return True note('extracting {}...', pkg.name) sys.stdout.flush() extract_opts = RelengExtractOptions() replicate_package_attribs(extract_opts, pkg) extract_opts.cache_dir = pkg.cache_dir extract_opts.cache_file = pkg.cache_file extract_opts.ext = pkg.ext_modifiers extract_opts.name = pkg.name extract_opts.revision = pkg.revision extract_opts.strip_count = pkg.strip_count extract_opts.version = pkg.version extract_opts._extract_override = engine.opts.extract_override extract_opts._quirks = engine.opts.quirks if os.path.exists(pkg.build_dir): warn('build directory exists before extraction; removing') if not path_remove(pkg.build_dir): err('unable to cleanup build directory: ' + pkg.build_dir) return False # prepare and step into the a newly created working directory # # An extractor will take the contents of an archive, cache directory or # other fetched content and populate the "work" directory. On successful # extraction (or moving resources), the work directory will be moved to the # package's respective build directory. out_dir = engine.opts.out_dir with generate_temp_dir(out_dir) as work_dir: with interim_working_dir(work_dir): extract_opts.work_dir = work_dir extracter = None hash_exclude = [] extract_types = engine.registry.extract_types if pkg.extract_type and pkg.extract_type in extract_types: def _(opts): return engine.registry.extract_types[pkg.vcs_type].extract( pkg.vcs_type, opts) extracter = _ elif pkg.vcs_type in extract_types: extracter = extract_types[pkg.vcs_type].extract elif pkg.vcs_type == VcsType.GIT: extracter = extract_git elif pkg.vcs_type == VcsType.HG: extracter = extract_mercurial elif os.path.isfile(pkg.cache_file): cache_basename = os.path.basename(pkg.cache_file) hash_exclude.append(cache_basename) extracter = extract_archive if not extracter: err('extract type is not implemented: {}', pkg.vcs_type) return False # perform the extract request extracted = extracter(extract_opts) if not extracted: return False result = verify_hashes(pkg.hash_file, work_dir, hash_exclude) if result == HashResult.VERIFIED: pass elif result == HashResult.BAD_PATH: if not pkg.is_internal: warn('missing hash file for package: ' + pkg.name) elif result == HashResult.EMPTY: if not pkg.is_internal: verbose('hash file for package is empty: ' + pkg.name) elif result in (HashResult.BAD_FORMAT, HashResult.MISMATCH, HashResult.MISSING_LISTED, HashResult.UNSUPPORTED): return False else: err('invalid extract operation (internal error; ' 'hash-check failure: {})', result) return False debug('extraction successful; moving sources into package output ' 'directory: ' + pkg.build_dir) shutil.move(work_dir, pkg.build_dir) return True
def fetch(opts): """ support fetching from bzr sources With provided fetch options (``RelengFetchOptions``), the fetch stage will be processed. Args: opts: fetch options Returns: ``True`` if the fetch stage is completed; ``False`` otherwise """ assert opts cache_file = opts.cache_file name = opts.name revision = opts.revision site = opts.site if not BZR.exists(): err('unable to fetch package; bzr is not installed') return None note('fetching {}...', name) sys.stdout.flush() cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir)) if not ensure_dir_exists(cache_dir): return None export_opts = [ 'export', cache_file, site, '--format=tgz', '--root=' + name, '--revision=' + revision, ] # some environments may have issue export bzr sources due to certificate # issues; this quirk allows injecting certifi-provided certificates for # all bzr exports if 'releng.bzr.certifi' in opts._quirks: global CERTIFI_MISSING_WARNED if certifi: verbose('performing bzr fetch with certifi certificates') pkg_site = certifi.where() export_opts.append('-Ossl.ca_certs=' + pkg_site) elif not CERTIFI_MISSING_WARNED: CERTIFI_MISSING_WARNED = True warn('''\ unable to perform bzr fetch with certifi certificates A quirk has been enabled to export bzr images using certifi certificates; however, certifi is not installed on this system. ''') log('exporting sources') if not BZR.execute(export_opts, poll=True): err('unable to export module') return None return cache_file
def stage(engine, pkg, script_env): """ handles the patching stage for a package With a provided engine and package instance, the patching stage will be processed. Args: engine: the engine pkg: the package being patched script_env: script environment information Returns: ``True`` if the patching stage is completed; ``False`` otherwise """ if pkg.is_internal: # packages flagged for local sources do not have a patch stage if pkg.local_srcs: return True # internal packages in development mode that specify a development # revision will not perform the patch stage if engine.opts.devmode and pkg.has_devmode_option: return True note('patching {}...', pkg.name) sys.stdout.flush() if pkg.build_subdir: build_dir = pkg.build_subdir else: build_dir = pkg.build_dir patch_script_filename = '{}-{}'.format(pkg.name, PATCH_SCRIPT) patch_script = os.path.join(pkg.def_dir, patch_script_filename) if os.path.isfile(patch_script): try: run_path(patch_script, init_globals=script_env) verbose('patch script executed: ' + patch_script) except Exception as e: err('error running patch script: \n' ' {}', patch_script, e) return False # find all patches in the package's folder, sort and apply each patch_glob = os.path.join(pkg.def_dir, '*.patch') patches = glob(patch_glob) if patches: patches = sorted(patches) if not PATCH.exists(): err('unable to apply patches; patch is not installed') return False for patch in patches: print('({})'.format(os.path.basename(patch))) if not PATCH.execute([ '--batch', '--forward', '--ignore-whitespace', '--input={}'.format(patch), '--strip=1', ], cwd=build_dir): err('failed to apply patch') return False return True
def check(self, quiet=False): """ check for the existence of required tools for the loaded package set For each loaded package, a series of required host tools will be checked and a caller will be notified whether or not anything is missing. Args: quiet (optional): whether or not to suppress output (defaults to ``False``) Returns: ``True`` is all known required tools exists; ``False`` otherwise """ missing = set() pkg_types = set() python_interpreters = set() vcs_types = set() # package-defined requirements check for pkg in self.pkgs: pkg_types.add(pkg.type) vcs_types.add(pkg.vcs_type) if pkg.type == PackageType.AUTOTOOLS: if pkg.autotools_autoreconf: if AUTORECONF.exists(): self._verbose_exists(AUTORECONF) else: missing.add(AUTORECONF.tool) elif pkg.type == PackageType.PYTHON: if pkg.python_interpreter: python_tool = PythonTool(pkg.python_interpreter) else: python_tool = PYTHON python_interpreters.add(python_tool) if PackageType.AUTOTOOLS in pkg_types: if MAKE.exists(): self._verbose_exists(MAKE) else: missing.add(MAKE.tool) if PackageType.CMAKE in pkg_types: if CMAKE.exists(): self._verbose_exists(CMAKE) else: missing.add(CMAKE.tool) if PackageType.PYTHON in pkg_types: for interpreter in python_interpreters: if interpreter.exists(): self._verbose_exists(interpreter) else: missing.add(interpreter.tool) if VcsType.BZR in vcs_types: if BZR.exists(): self._verbose_exists(BZR) else: missing.add(BZR.tool) if VcsType.CVS in vcs_types: if CVS.exists(): self._verbose_exists(CVS) else: missing.add(CVS.tool) if VcsType.GIT in vcs_types: if GIT.exists(): self._verbose_exists(GIT) else: missing.add(GIT.tool) if VcsType.HG in vcs_types: if HG.exists(): self._verbose_exists(HG) else: missing.add(HG.tool) if VcsType.RSYNC in vcs_types: if RSYNC.exists(): self._verbose_exists(RSYNC) else: missing.add(RSYNC.tool) if VcsType.SCP in vcs_types: if SCP.exists(): self._verbose_exists(SCP) else: missing.add(SCP.tool) if VcsType.SVN in vcs_types: if SVN.exists(): self._verbose_exists(SVN) else: missing.add(SVN.tool) # project-provided tools check for tool in self.tools: if which(tool): verbose('prerequisite exists: ' + tool) else: missing.add(tool) if missing and not quiet: sorted_missing = list(missing) sorted_missing.sort() msg = 'missing the following host tools for this project:' msg += '\n' msg += '\n' for entry in sorted_missing: msg += ' ' + entry + '\n' err(msg) return len(missing) == 0
def load(self, names): """ load one or more packages from the provided collection of names Attempts to load and return a series of ordered package instances using the collection of names provided. Each name will be used to find a package definition on the system. Package scripts are found, loaded and parsed. Packages with dependencies will have their dependent packages loaded as well (either from the explicitly from the ``names`` or implicitly from the package's configuration file). The returned package list will be an ordered package list based on configured dependencies outlined in the user's package definitions. When package dependencies do not play a role in the required order of the releng process, a first-configured first-returned approach is used. Args: names: the names of packages to load Returns: returns an ordered list of packages to use Raises: RelengToolInvalidPackageConfiguration: when an error has been detected loading the package """ pkgs = OrderedDict() final_deps = {} # cycle through all pending packages until the complete list is known names_left = list(names) while names_left: name = names_left.pop(0) # attempt to load the package from a user defined external directory pkg = None for pkg_dir in self.opts.extern_pkg_dirs: pkg_script = os.path.join(pkg_dir, name, name) pkg_script, pkg_script_exists = opt_file(pkg_script) if pkg_script_exists: pkg, env, deps = self.load_package(name, pkg_script) # if a package location has not been found, finally check the # default package directory if not pkg: pkg_script = os.path.join(self.opts.default_pkg_dir, name, name) pkg_script, _ = opt_file(pkg_script) pkg, env, deps = self.load_package(name, pkg_script) pkgs[pkg.name] = pkg for dep in deps: # if this is an unknown package and is not in out current list, # append it to the list of names to process if dep == name: raise RelengToolCyclicPackageDependency({ 'pkg_name': name, }) elif dep not in pkgs: if dep not in names_left: verbose('adding implicitly defined package: {}', dep) names_left.append(dep) if pkg not in final_deps: final_deps[pkg] = [] final_deps[pkg].append(dep) else: pkg.deps.append(pkgs[dep]) extend_script_env(self.script_env, env) # for packages which have a dependency but have not been bound yet, # bind the dependencies now for pkg, deps in final_deps.items(): for dep in deps: assert pkgs[dep] pkg.deps.append(pkgs[dep]) debug('sorting packages...') def fetch_deps(pkg): return pkg.deps sorter = TopologicalSorter(fetch_deps) sorted_pkgs = [] for pkg in pkgs.values(): sorted_pkgs = sorter.sort(pkg) if sorted_pkgs is None: raise RelengToolCyclicPackageDependency({ 'pkg_name': name, }) debug('sorted packages)') for pkg in sorted_pkgs: debug(' {}', pkg.name) return sorted_pkgs
def main(): """ mainline The mainline for the releng tool. Returns: the exit code """ retval = 1 try: parser = argparse.ArgumentParser(prog='releng-tool', add_help=False, usage=usage()) parser.add_argument('--assets-dir') parser.add_argument('--cache-dir') parser.add_argument('--config') parser.add_argument('--debug', action='store_true') parser.add_argument('--development', '-D', action='store_true') parser.add_argument('--dl-dir') parser.add_argument('--force', '-F', action='store_true') parser.add_argument('--help', '-h', action='store_true') parser.add_argument('--help-quirks', action='store_true') parser.add_argument('--images-dir') parser.add_argument('--jobs', '-j', default=0, type=type_nonnegativeint) parser.add_argument('--local-sources', '-L', nargs='?', action='append') parser.add_argument('--nocolorout', action='store_true') parser.add_argument('--out-dir') parser.add_argument('--root-dir') parser.add_argument('--quirk', action='append') parser.add_argument('--verbose', '-V', action='store_true') parser.add_argument('--version', action='version', version='%(prog)s ' + releng_version) known_args = sys.argv[1:] forward_args = [] idx = known_args.index('--') if '--' in known_args else -1 if idx != -1: forward_args = known_args[idx + 1:] known_args = known_args[:idx] args, unknown_args = parser.parse_known_args(known_args) if args.help: print(usage()) sys.exit(0) if args.help_quirks: print(usage_quirks()) sys.exit(0) # force verbose messages if debugging is enabled if args.debug: args.verbose = True releng_log_configuration(args.debug, args.nocolorout, args.verbose) # toggle on ansi colors by default for commands if not args.nocolorout: os.environ['CLICOLOR_FORCE'] = '1' # support character sequences (for color output on win32 cmd) if sys.platform == 'win32': enable_ansi_win32() verbose('releng-tool {}', releng_version) debug('({})', __file__) # extract additional argument information: # - pull the action value # - pull "exec" command (if applicable) # - key-value entries to be injected into the running # script/working environment new_args, unknown_args = process_args(unknown_args) args.action = new_args['action'] args.action_exec = new_args['exec'] args.injected_kv = new_args['entries'] # register any injected entry into the working environment right away for k, v in args.injected_kv.items(): os.environ[k] = v if unknown_args: warn('unknown arguments: {}', ' '.join(unknown_args)) if forward_args: debug('forwarded arguments: {}', ' '.join(forward_args)) # warn if the *nix-based system is running as root; ill-formed projects # may attempt to modify the local system's root if sys.platform != 'win32': if os.geteuid() == 0: # pylint: disable=E1101 if 'RELENG_IGNORE_RUNNING_AS_ROOT' not in os.environ: # attempt to check if we are in a container; if so, ignore # generating a warning -- we will check if kernel threads # are running on pid2; if not, it is most likely that we # are in a container environment; checks for a container # do not have to be perfect here, only to try to help # improve a user's experience (suppressing this warning # when not running on a typical host setup) try: with open('/proc/2/status') as f: inside_container = 'kthreadd' not in f.read() except IOError: inside_container = True if not inside_container: warn('running as root; this may be unsafe') # prepare engine options opts = RelengEngineOptions(args=args, forward_args=forward_args) # create and start the engine engine = RelengEngine(opts) try: if engine.run(): retval = 0 except RelengToolException as e: err(e) except KeyboardInterrupt: print('') return retval
def load_package(self, name, script): """ load a package definition Attempts to load a package definition of a given ``name`` from the provided ``script`` location. The script will be examine for required and optional configuration keys. On a successful execution/parsing, a package object will be returned along with other meta information. On error, ``None`` types are returned. Args: name: the package name script: the package script to load Returns: returns a tuple of three (3) containing the package instance, the extracted environment/globals from the package script and a list of known package dependencies Raises: RelengToolInvalidPackageConfiguration: when an error has been detected loading the package """ verbose('loading package: {}', name) debug('script {}', script) opts = self.opts if not os.path.isfile(script): raise RelengToolMissingPackageScript({ 'pkg_name': name, 'script': script, }) pkg_def_dir = os.path.abspath(os.path.join(script, os.pardir)) self.script_env['PKG_DEFDIR'] = pkg_def_dir try: env = run_script(script, self.script_env, catch=False) except Exception as e: raise RelengToolInvalidPackageScript({ 'description': str(e), 'script': script, 'traceback': traceback.format_exc(), }) self._active_package = name self._active_env = env # prepare helper expand values expand_extra = {} # version/revision extraction first # # Attempt to check the version first since it will be the most commonly # used package field -- rather initially fail on a simple field first # (for new packages and/or developers) than breaking on a possibly more # complex field below. Note that the version field is optional, in cases # where a package type does not need a version entry (e.g. sites which # do not require a version value for fetching or there is not revision # value to use instead). # # Note that when in development mode, the development-mode revision # (if any is set) needs to be checked as well. This value may override # the package's version value. # version pkg_version = self._fetch(Rpk.VERSION) if not pkg_version: pkg_version = '' pkg_version_key = pkg_key(name, Rpk.VERSION) expand_extra[pkg_version_key] = pkg_version # development mode revision pkg_has_devmode_option = False pkg_devmode_revision = self._fetch(Rpk.DEVMODE_REVISION, allow_expand=True, expand_extra=expand_extra) if pkg_devmode_revision: pkg_has_devmode_option = True if opts.revision_override and name in opts.revision_override: pkg_devmode_revision = opts.revision_override[name] if opts.devmode: pkg_version = pkg_devmode_revision expand_extra[pkg_version_key] = pkg_version # revision if opts.revision_override and name in opts.revision_override: pkg_revision = opts.revision_override[name] else: pkg_revision = self._fetch(Rpk.REVISION, allow_expand=True, expand_extra=expand_extra) if opts.devmode and pkg_devmode_revision: pkg_revision = pkg_devmode_revision elif not pkg_revision: pkg_revision = pkg_version # site / vcs-site detection # # After extracted required version information, the site / VCS type # needs to be checked next. This will allow the manage to early detect # if a version/revision field is required, and fail early if we have # not detected one from above. # site if opts.sites_override and name in opts.sites_override: # Site overriding is permitted to help in scenarios where a builder # is unable to acquire a package's source from the defined site. # This includes firewall settings or a desire to use a mirrored # source when experiencing network connectivity issues. pkg_site = opts.sites_override[name] else: pkg_site = self._fetch(Rpk.SITE, allow_expand=True, expand_extra=expand_extra) # On Windows, if a file site is provided, ensure the path value is # converted to a posix-styled path, to prevent issues with `urlopen` # being provided an unescaped path string if sys.platform == 'win32' and \ pkg_site and pkg_site.startswith('file://'): pkg_site = pkg_site[len('file://'):] abs_site = os.path.isabs(pkg_site) pkg_site = pkg_site.replace(os.sep, posixpath.sep) if abs_site: pkg_site = '/' + pkg_site pkg_site = 'file://' + pkg_site # vcs-type pkg_vcs_type = None pkg_vcs_type_raw = self._fetch(Rpk.VCS_TYPE) if pkg_vcs_type_raw: pkg_vcs_type_raw = pkg_vcs_type_raw.lower() if pkg_vcs_type_raw in VcsType: pkg_vcs_type = pkg_vcs_type_raw elif pkg_vcs_type_raw in self.registry.fetch_types: pkg_vcs_type = pkg_vcs_type_raw else: raise RelengToolUnknownVcsType({ 'pkg_name': name, 'pkg_key': pkg_key(name, Rpk.VCS_TYPE), }) if not pkg_vcs_type: if pkg_site: site_lc = pkg_site.lower() if site_lc.startswith('bzr+'): pkg_site = pkg_site[4:] pkg_vcs_type = VcsType.BZR elif site_lc.startswith('cvs+'): pkg_site = pkg_site[4:] pkg_vcs_type = VcsType.CVS elif site_lc.startswith(( ':ext:', ':extssh:', ':gserver:', ':kserver:', ':pserver:', )): pkg_vcs_type = VcsType.CVS elif site_lc.startswith('git+'): pkg_site = pkg_site[4:] pkg_vcs_type = VcsType.GIT elif site_lc.endswith('.git'): pkg_vcs_type = VcsType.GIT elif site_lc.startswith('hg+'): pkg_site = pkg_site[3:] pkg_vcs_type = VcsType.HG elif site_lc.startswith('rsync+'): pkg_site = pkg_site[6:] pkg_vcs_type = VcsType.RSYNC elif site_lc.startswith('scp+'): pkg_site = pkg_site[4:] pkg_vcs_type = VcsType.SCP elif site_lc.startswith('svn+'): pkg_site = pkg_site[4:] pkg_vcs_type = VcsType.SVN elif site_lc == 'local': pkg_vcs_type = VcsType.LOCAL else: pkg_vcs_type = VcsType.URL else: pkg_vcs_type = VcsType.NONE if pkg_vcs_type == VcsType.LOCAL: warn('package using local content: {}', name) # check if the detected vcs type needs a revision, and fail if we do # not have one if not pkg_revision and pkg_vcs_type in ( VcsType.BZR, VcsType.CVS, VcsType.GIT, VcsType.HG, VcsType.SVN, ): raise RelengToolMissingPackageRevision({ 'pkg_name': name, 'pkg_key1': pkg_key(name, Rpk.VERSION), 'pkg_key2': pkg_key(name, Rpk.REVISION), 'vcs_type': pkg_vcs_type, }) # archive extraction strip count pkg_strip_count = self._fetch(Rpk.STRIP_COUNT, default=DEFAULT_STRIP_COUNT) # build subdirectory pkg_build_subdir = self._fetch(Rpk.BUILD_SUBDIR) # dependencies deps = self._fetch(Rpk.DEPS, default=[]) # ignore cache pkg_devmode_ignore_cache = self._fetch(Rpk.DEVMODE_IGNORE_CACHE) # extension (override) pkg_filename_ext = self._fetch(Rpk.EXTENSION) # extract type pkg_extract_type = self._fetch(Rpk.EXTRACT_TYPE) if pkg_extract_type: pkg_extract_type = pkg_extract_type.lower() if pkg_extract_type not in self.registry.extract_types: raise RelengToolUnknownExtractType({ 'pkg_name': name, 'pkg_key': pkg_key(name, Rpk.EXTRACT_TYPE), }) # is-external pkg_is_external = self._fetch(Rpk.EXTERNAL) # is-internal pkg_is_internal = self._fetch(Rpk.INTERNAL) # no extraction pkg_no_extraction = self._fetch(Rpk.NO_EXTRACTION) # skip any remote configuration pkg_skip_remote_config = self._fetch(Rpk.SKIP_REMOTE_CONFIG) # skip any remote scripts pkg_skip_remote_scripts = self._fetch(Rpk.SKIP_REMOTE_SCRIPTS) # type pkg_type = None pkg_type_raw = self._fetch(Rpk.TYPE) if pkg_type_raw: pkg_type_raw = pkg_type_raw.lower() if pkg_type_raw in PackageType: pkg_type = pkg_type_raw elif pkg_type_raw in self.registry.package_types: pkg_type = pkg_type_raw else: raise RelengToolUnknownPackageType({ 'pkg_name': name, 'pkg_key': pkg_key(name, Rpk.TYPE), }) if not pkg_type: pkg_type = PackageType.SCRIPT # ###################################################################### # git configuration options for a repository pkg_git_config = self._fetch(Rpk.GIT_CONFIG) # git-depth pkg_git_depth = self._fetch(Rpk.GIT_DEPTH) # git-refspecs pkg_git_refspecs = self._fetch(Rpk.GIT_REFSPECS) # git-submodules pkg_git_submodules = self._fetch(Rpk.GIT_SUBMODULES) # git-verify pkg_git_verify_revision = self._fetch(Rpk.GIT_VERIFY_REVISION) # ###################################################################### # checks if pkg_is_external is not None and pkg_is_internal is not None: if pkg_is_external == pkg_is_internal: raise RelengToolConflictingConfiguration({ 'pkg_name': name, 'pkg_key1': pkg_key(name, Rpk.EXTERNAL), 'pkg_key2': pkg_key(name, Rpk.INTERNAL), 'desc': 'package flagged as external and internal', }) elif pkg_is_external is not None: pkg_is_internal = not pkg_is_external elif pkg_is_internal is not None: pass elif opts.default_internal_pkgs: pkg_is_internal = True else: pkg_is_internal = False # check a site is defined for vcs types which require it if not pkg_site and pkg_vcs_type in ( VcsType.BZR, VcsType.CVS, VcsType.GIT, VcsType.HG, VcsType.RSYNC, VcsType.SCP, VcsType.SVN, VcsType.URL, ): raise RelengToolMissingPackageSite({ 'pkg_name': name, 'pkg_key': pkg_key(name, Rpk.SITE), 'vcs_type': pkg_vcs_type, }) # list of support dvcs types SUPPORTED_DVCS = [ VcsType.GIT, VcsType.HG, ] is_pkg_dvcs = (pkg_vcs_type in SUPPORTED_DVCS) # find possible extension for a cache file # # non-dvcs's will be always gzip-tar'ed. if pkg_vcs_type in ( VcsType.BZR, VcsType.CVS, VcsType.RSYNC, VcsType.SVN, ): cache_ext = 'tgz' # dvcs's will not have an extension type elif is_pkg_dvcs: cache_ext = None # non-vcs type does not have an extension elif pkg_vcs_type in (VcsType.LOCAL, VcsType.NONE): cache_ext = None else: cache_ext = None url_parts = urlparse(pkg_site) if opts.cache_ext_transform: # Allow a configuration to override the target cache file's # extension based on the package's site path (for unique path # scenarios). cache_ext = opts.cache_ext_transform(url_parts.path) if not cache_ext: if pkg_filename_ext: cache_ext = pkg_filename_ext else: basename = os.path.basename(url_parts.path) __, cache_ext = interpret_stem_extension(basename) # prepare package container and directory locations # # The container folder for a package will typically be a combination of # a package's name plus version. If no version is set, the container # will be only use the package's name. We try to use the version entry # when possible to help manage multiple versions of output (e.g. to # avoid conflicts when bumping versions). # # When the version value is used, we will attempt to cleanup/minimize # the version to help provide the container a more "sane" path. For # instance, if a version references a path-styled branch names (e.g. # `bugfix/my-bug`, we want to avoid promoting a container name which # can result in a sub-directory being made (e.g. `pkg-bugfix/my-bug/`). if pkg_version: pkg_nv = '{}-{}'.format( name, ''.join(x if (x.isalnum() or x in '-._') else '_' for x in pkg_version)) else: pkg_nv = name pkg_build_output_dir = os.path.join(opts.build_dir, pkg_nv) if pkg_vcs_type == VcsType.LOCAL: pkg_build_dir = pkg_def_dir else: pkg_build_dir = pkg_build_output_dir # check if an internal package is configured to point to a local # directory for sources pkg_local_srcs = False if pkg_is_internal and opts.local_srcs: # specific package name reference in the local sources; either is # set to the path to use, or is set to `None` to indicate at this # package should not be retrieved locally if name in opts.local_srcs: if opts.local_srcs[name]: pkg_build_dir = opts.local_srcs[name] pkg_local_srcs = True # check if the "global" local sources path exists; either set to # a specific path, or set to `None` to indicate that it will use # the parent path based off the root directory elif GBL_LSRCS in opts.local_srcs: if opts.local_srcs[GBL_LSRCS]: container_dir = opts.local_srcs[GBL_LSRCS] else: container_dir = os.path.dirname(opts.root_dir) pkg_build_dir = os.path.join(container_dir, name) pkg_local_srcs = True if pkg_build_dir == opts.root_dir: raise RelengToolConflictingLocalSrcsPath({ 'pkg_name': name, 'root': opts.root_dir, 'path': pkg_build_dir, }) if pkg_build_subdir: pkg_build_subdir = os.path.join(pkg_build_dir, pkg_build_subdir) cache_dir = os.path.join(opts.dl_dir, name) if cache_ext: pkg_cache_file = os.path.join(cache_dir, pkg_nv + '.' + cache_ext) else: pkg_cache_file = os.path.join(cache_dir, pkg_nv) # Select sources (like CMake-based projects) may wish to be using # out-of-source tree builds. For supported project types, adjust the # build output directory to a sub-folder of the originally assumed # output folder. if pkg_type == PackageType.CMAKE: pkg_build_output_dir = os.path.join(pkg_build_output_dir, 'releng-output') # determine the build tree for a package # # A build tree (introduced for the libfoo-exec action), tracks the # directory where build commands would typically be executed for a # package on a host system. In most cases, this will be set to the # same path as `pkg_build_dir` (or the sub-directory, if provided); # however, some package types may have a better working directory # for build commands. For example, CMake projects will generate a # build package in an out-of-source directory (e.g. # `pkg_build_output_dir`), which is a better make to issue commands # such as "cmake --build .". if pkg_type == PackageType.CMAKE: pkg_build_tree = pkg_build_output_dir elif pkg_build_subdir: pkg_build_tree = pkg_build_subdir else: pkg_build_tree = pkg_build_dir # determine the package directory for this package # # Typically, a package's "cache directory" will be stored in the output # folder's "cache/<pkg-name>" path. However, having package-name driven # cache folder targets does not provide an easy way to manage sharing # caches between projects if they share the same content (either the # same site or sharing submodules). Cache targets for packages will be # stored in a database and can be used here to decide if a package's # cache will actually be stored in a different container. pkg_cache_dir = os.path.join(opts.cache_dir, name) if is_pkg_dvcs: ckey = pkg_cache_key(pkg_site) pkg_cache_dirname = name # if the default cache directory exists, always prioritize it (and # force update the cache location) if os.path.exists(pkg_cache_dir): self._dvcs_cache[name] = name # if the cache content is stored in another container, use it elif ckey in self._dvcs_cache: pkg_cache_dirname = self._dvcs_cache[ckey] verbose('alternative cache path for package: {} -> {}', name, pkg_cache_dirname) # track ckey entry to point to our cache container # # This package's "ckey" will be used to cache the target folder # being used for this package, so other packages with matching site # values could use it. In the rare case that the "ckey" entry # already exists but is pointing to another folder that our target # one, leave it as is (assume ownership of key is managed by another # package). if ckey not in self._dvcs_cache: self._dvcs_cache[ckey] = pkg_cache_dirname # adjust the cache directory and save any new cache changes pkg_cache_dir = os.path.join(opts.cache_dir, pkg_cache_dirname) self._save_dvcs_cache() # (commons) pkg = RelengPackage(name, pkg_version) pkg.asc_file = os.path.join(pkg_def_dir, name + '.asc') pkg.build_dir = pkg_build_dir pkg.build_output_dir = pkg_build_output_dir pkg.build_subdir = pkg_build_subdir pkg.build_tree = pkg_build_tree pkg.cache_dir = pkg_cache_dir pkg.cache_file = pkg_cache_file pkg.def_dir = pkg_def_dir pkg.devmode_ignore_cache = pkg_devmode_ignore_cache pkg.extract_type = pkg_extract_type pkg.git_config = pkg_git_config pkg.git_depth = pkg_git_depth pkg.git_refspecs = pkg_git_refspecs pkg.git_submodules = pkg_git_submodules pkg.git_verify_revision = pkg_git_verify_revision pkg.has_devmode_option = pkg_has_devmode_option pkg.hash_file = os.path.join(pkg_def_dir, name + '.hash') pkg.is_internal = pkg_is_internal pkg.local_srcs = pkg_local_srcs pkg.no_extraction = pkg_no_extraction pkg.revision = pkg_revision pkg.site = pkg_site pkg.skip_remote_config = pkg_skip_remote_config pkg.skip_remote_scripts = pkg_skip_remote_scripts pkg.strip_count = pkg_strip_count pkg.type = pkg_type pkg.vcs_type = pkg_vcs_type self._apply_postinit_options(pkg) # (additional environment helpers) for env in (os.environ, env): env[pkg_key(name, 'BUILD_DIR')] = pkg_build_dir env[pkg_key(name, 'BUILD_OUTPUT_DIR')] = pkg_build_output_dir env[pkg_key(name, 'DEFDIR')] = pkg_def_dir env[pkg_key(name, 'NAME')] = name env[pkg_key(name, 'REVISION')] = pkg_revision os.environ[pkg_key(name, Rpk.VERSION)] = pkg_version # (internals) prefix = '.releng_tool-stage-' outdir = pkg.build_output_dir pkg._ff_bootstrap = os.path.join(outdir, prefix + 'bootstrap') pkg._ff_build = os.path.join(outdir, prefix + 'build') pkg._ff_configure = os.path.join(outdir, prefix + 'configure') pkg._ff_extract = os.path.join(outdir, prefix + 'extract') pkg._ff_install = os.path.join(outdir, prefix + 'install') pkg._ff_license = os.path.join(outdir, prefix + 'license') pkg._ff_patch = os.path.join(outdir, prefix + 'patch') pkg._ff_post = os.path.join(outdir, prefix + 'post') # dump package attributes if running in a debug mode if opts.debug: info = {} for key, value in pkg.__dict__.items(): if not key.startswith('_'): info[key] = value debug( '''package-data: {} ============================== {} ==============================''', name, pprint.pformat(info)) return pkg, env, deps
def load(self, name, ignore=True): """ load the provided extension into the registry Attempts to load an extension with the provided name value. If an extension which is already loaded in the registry is provided, the request to load the specific extension is ignored. If an extension could not be loaded, a warning is generated and this method will return ``False``. Args: name: name of the extension to load ignore (optional): ignore if the detected extension could not be loaded (default: True) Returns: whether or not the extension was loaded in the registry """ # ignore if extension is already loaded if name in self.extension: return True loaded = False debug('attempting to load extension: {}', name) try: try: plugin = import_module(name) except RelengModuleNotFoundError: # python 2.7 may not be able to load from a nested path; try # searching through each package (if a nested module) if sys.version_info >= (3, 0) or '.' not in name: raise # split the module into parts and for each part, check to see # if it's a package directory; if so, keep going until the last # namespace package ext_parts = name.split('.') path = None for part in ext_parts[:-1]: file, pathname, desc = imp.find_module(part, path) if desc[-1] != imp.PKG_DIRECTORY: raise ImportError(name) pkg = imp.load_module(part, file, pathname, desc) path = pkg.__path__ # with the path of the last namespace package found, find the # desired module in this path last_part = ext_parts[-1] file, pathname, desc = imp.find_module(last_part, path) plugin = imp.load_module(last_part, file, pathname, desc) if hasattr(plugin, 'releng_setup'): if not ignore: plugin.releng_setup(self) loaded = True else: try: plugin.releng_setup(self) loaded = True except RelengInvalidSetupException as e: warn( 'extension is not supported ' 'due to an invalid setup: {}\n' ' ({})', name, e) except RelengVersionNotSupportedException as e: warn( 'extension is not supported ' 'with this version: {}\n' ' ({})', name, e) if loaded: self.extension.append(name) verbose('loaded extension: {}', name) loaded = True else: warn('extension does not have a setup method: {}', name) except RelengModuleNotFoundError: warn('unable to find extension: {}', name) return loaded