def install(opts):
"""
support installation cmake projects
With provided installation options (``RelengInstallOptions``), the
installation stage will be processed.
Args:
opts: installation options
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
if not CMAKE.exists():
err('unable to install package; cmake is not installed')
return False
# check if the no-install flag is set
if opts._cmake_noinstall:
verbose('configured to skip install stage for cmake')
return True
# default definitions
cmake_defs = {}
if opts.install_defs:
cmake_defs.update(expand(opts.install_defs))
# default options
cmake_opts = {
# build RelWithDebInfo (when using multi-configuration projects)
'--config': 'RelWithDebInfo',
# default install using the install target
'--target': 'install',
}
if opts.install_opts:
cmake_opts.update(expand(opts.install_opts))
# argument building
cmake_args = [
'--build',
opts.build_output_dir,
]
cmake_args.extend(prepare_definitions(cmake_defs, '-D'))
cmake_args.extend(prepare_arguments(cmake_opts))
# prepare environment for installation request; an environment dictionary is
# always needed to apply a custom DESTDIR during each install request
env = expand(opts.install_env)
if not env:
env = {}
# install to each destination
for dest_dir in opts.dest_dirs:
env['DESTDIR'] = dest_dir
if not CMAKE.execute(cmake_args, env=env):
err('failed to install cmake project: {}', opts.name)
return False
return True
def platform_exit(msg=None, code=None):
"""
exit out of the releng-tool process
Provides a convenience method to help invoke a system exit call without
needing to explicitly use ``sys``. A caller can provide a message to
indicate the reason for the exit. The provide message will output to
standard error. The exit code, if not explicit set, will vary on other
arguments. If a message is provided to this call, the default exit code will
be ``1``. If no message is provided, the default exit code will be ``0``.
In any case, if the caller explicitly sets a code value, the provided code
value will be used.
An example when using in the context of script helpers is as follows:
.. code-block:: python
releng_exit('there was an error performing this task')
Args:
msg (optional): error message to print
code (optional): exit code; defaults to 0 if no message or defaults to 1
if a message is set
Raises:
SystemExit: always raised
"""
if msg:
err(msg)
if code is None:
code = 1
elif code is None:
code = 0
sys.exit(code)
def _validate_cache(cache_dir):
"""
validate an existing cache directory to fetch on
A fetch operation may occur on an existing cache directory, typically when
a force-fetch or a configured revision has changed. This call helps
validate the existing cache directory (from a bad state such as a corrupted
repository). If a cache directory does exist,
Args:
cache_dir: the cache/bare repository to fetch into
Returns:
a 2-tuple (if a cache directory exists; and if validation failed)
"""
git_dir = '--git-dir=' + cache_dir
bad_validation = False
has_cache = False
if os.path.isdir(cache_dir):
log('cache directory detected; validating')
if GIT.execute([git_dir, 'rev-parse'], cwd=cache_dir, quiet=True):
debug('cache directory validated')
has_cache = True
else:
log('cache directory has errors; will re-downloaded')
if not path_remove(cache_dir):
err(
'unable to cleanup cache folder for package\n'
' (cache folder: {})', cache_dir)
bad_validation = True
return has_cache, bad_validation
def extract(opts):
"""
support extraction (checkout) of a git cache into a build directory
With provided extraction options (``RelengExtractOptions``), the extraction
stage will be processed. A Git extraction process will populate a working
tree based off the cached Git tree acquired from the fetch stage.
Args:
opts: the extraction options
Returns:
``True`` if the extraction stage is completed; ``False`` otherwise
"""
assert opts
cache_dir = opts.cache_dir
revision = opts.revision
work_dir = opts.work_dir
if not GIT.exists():
err('unable to extract package; git is not installed')
return None
# extract the package
if not _workdir_extract(cache_dir, work_dir, revision):
return False
# extract submodules (if configured to do so)
if opts._git_submodules:
if not _process_submodules(opts, work_dir):
return False
return True
def extract_submodule_revision(self, bare_dir):
"""
extract a submodule revision
Attempts to extract the HEAD reference of a submodule based off a
provided bare Git repository. This is to help support processing Git
submodules which do not have a branch/version explicitly set for module,
which is required for (at least) recursive submodule processing.
Args:
bare_dir: the bare repository
Returns:
the revision; ``None`` when a revision cannot be extracted
"""
rv, ref = self.execute_rv('--git-dir=' + bare_dir, 'show-ref', '--head')
if rv != 0:
err('failed to extract a submodule revision')
return None
# a `--head` fetch may fetch more than one reference; extract the first
# entry and remove any known ref prefix from it
revision = ref.split(None, 2)[1]
if revision.startswith('refs/heads/'):
revision = revision[len('refs/heads/'):]
elif revision.startswith('refs/remotes/origin/'):
revision = revision[len('refs/remotes/origin/'):]
return revision
def fetch(opts):
"""
support fetching from scp sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
site = opts.site
work_dir = opts.work_dir
if not SCP.exists():
err('unable to fetch package; scp is not installed')
return None
note('fetching {}...', name)
sys.stdout.flush()
if not SCP.execute(['-o', 'BatchMode yes', site, cache_file],
cwd=work_dir):
err('unable to secure-copied file from target')
return None
log('successfully secure-copied file from target')
return cache_file
def extract(opts):
"""
support extraction (checkout) of a mercurial cache into a build directory
With provided extraction options (``RelengExtractOptions``), the extraction
stage will be processed. A Mercurial extraction process will populate a
working tree based off the cached Mercurial repository acquired from the
fetch stage.
Args:
opts: the extraction options
Returns:
``True`` if the extraction stage is completed; ``False`` otherwise
"""
assert opts
cache_dir = opts.cache_dir
revision = opts.revision
work_dir = opts.work_dir
if not HG.exists():
err('unable to extract package; mercurial (hg) is not installed')
return None
log('checking out target revision into work tree')
if not HG.execute(
['--verbose', 'clone', '--rev', revision, cache_dir, work_dir],
cwd=work_dir):
err('unable to checkout revision')
return False
return True
def _sync_git_origin(cache_dir, site):
"""
synchronize an origin site to a git configuration
Ensures the configured site is set as the origin of the repository. This is
to help handle scenarios where a package's site has changed while content is
already cached.
Args:
cache_dir: the cache/bare repository
site: the site that should be set
Returns:
``True`` if the site is synchronized; ``False`` otherwise
"""
git_dir = '--git-dir=' + cache_dir
# silently try to add origin first, to lazily handle a missing case
GIT.execute([git_dir, 'remote', 'add', 'origin', site],
cwd=cache_dir,
quiet=True)
if not GIT.execute([git_dir, 'remote', 'set-url', 'origin', site],
cwd=cache_dir):
err('unable to ensure origin is set on repository cache')
return False
return True
def build(opts):
"""
support building python projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if opts._python_interpreter:
python_tool = PythonTool(opts._python_interpreter,
env_include=PYTHON_EXTEND_ENV)
else:
python_tool = PYTHON
if not python_tool.exists():
err('unable to build package; python is not installed')
return False
# definitions
python_defs = {}
if opts.build_defs:
python_defs.update(expand(opts.build_defs))
# default options
python_opts = {}
if opts.build_opts:
python_opts.update(expand(opts.build_opts))
# default environment
path1 = python_tool.path(sysroot=opts.staging_dir, prefix=opts.prefix)
path2 = python_tool.path(sysroot=opts.target_dir, prefix=opts.prefix)
env = {'PYTHONPATH': path1 + os.pathsep + path2}
# apply package-specific environment options
if opts.build_env:
env.update(expand(opts.build_env))
# argument building
python_args = [
'setup.py',
# ignore user's pydistutils.cfg
'--no-user-cfg',
# invoke the build operation
'build',
]
python_args.extend(prepare_definitions(python_defs))
python_args.extend(prepare_arguments(python_opts))
if not python_tool.execute(python_args, env=env):
err('failed to build python project: {}', opts.name)
return False
return True
def build(opts):
"""
support building cmake projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if not CMAKE.exists():
err('unable to build package; cmake is not installed')
return False
# definitions
cmake_defs = {}
if opts.build_defs:
cmake_defs.update(expand(opts.build_defs))
# options
cmake_opts = {
# build RelWithDebInfo (when using multi-configuration projects)
'--config': 'RelWithDebInfo',
}
if opts.build_opts:
cmake_opts.update(expand(opts.build_opts))
# argument building
cmake_args = [
# tell cmake to invoke build process in the output directory
'--build',
opts.build_output_dir,
]
cmake_args.extend(prepare_definitions(cmake_defs, '-D'))
cmake_args.extend(prepare_arguments(cmake_opts))
# enable specific number of parallel jobs is set
#
# https://cmake.org/cmake/help/v3.12/manual/cmake.1.html#build-tool-mode
if 'releng.cmake.disable_parallel_option' not in opts._quirks:
if opts.jobsconf != 1 and opts.jobs > 1:
cmake_args.append('--parallel')
cmake_args.append(str(opts.jobs))
else:
verbose('cmake parallel jobs disabled by quirk')
if not CMAKE.execute(cmake_args, env=expand(opts.build_env)):
err('failed to build cmake project: {}', opts.name)
return False
return True
def _execute(self,
args=None,
cwd=None,
quiet=False,
env=None,
poll=False,
capture=None):
"""
execute the host tool with the provided arguments (if any)
Runs the host tool described by ``args`` until completion.
Args:
args (optional): the list of arguments for the tool
cwd (optional): working directory to use
quiet (optional): whether or not to suppress output
env (optional): environment variables to include
poll (optional): force polling stdin/stdout for output data
capture (optional): list to capture output into
Returns:
the return code of the execution request
"""
if not self.exists():
return 1
if args and not is_sequence_not_string(args):
err('invalid argument type provided into execute (should be list): '
+ str(args))
return 1
final_env = None
if self.include or self.sanitize or env:
final_env = os.environ.copy()
if self.sanitize:
for key in self.sanitize:
final_env.pop(key, None)
if self.include:
final_env.update(self.include)
if env:
final_env.update(env)
final_args = [self.tool]
if args:
final_args.extend(args)
return _execute(final_args,
cwd=cwd,
env=final_env,
quiet=quiet,
critical=False,
poll=poll,
capture=capture)
def fetch(opts):
"""
support fetching from svn sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
revision = opts.revision
site = opts.site
work_dir = opts.work_dir
if not SVN.exists():
err('unable to fetch package; svn is not installed')
return None
note('fetching {}...'.format(name))
sys.stdout.flush()
log('checking out sources')
if not SVN.execute(['checkout', '-r', revision, site, work_dir],
cwd=work_dir):
err('unable to checkout module')
return None
log('caching sources')
def svn_exclude(file):
if file.endswith('.svn'):
return True
return False
# ensure cache file's directory exists
cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir))
if not ensure_dir_exists(cache_dir):
return None
with tarfile.open(cache_file, 'w:gz') as tar:
tar.add(work_dir, arcname=name, exclude=svn_exclude)
return cache_file
def process_file_flag(flag, file, quiet=False):
"""
process a file flag event
Will either write a file flag configuration event or attempt to read a file
flag state. If the ``flag`` option is set to ``True``, this process event
will assume that this instance is attempting to configure a file flag (on)
state and generate the target file flag on the system. If the flag option is
set to ``False``, the file's existence will be checked to reflect whether or
not the flag is considered enabled.
Args:
flag: the flag option to used; ``None`` to check flag state
file: the filename
quiet: suppression of any error messages to standard out
Returns:
``FileFlag.EXISTS`` if the flag is enabled; ``FileFlag.NO_EXIST`` if the
flag is not enabled; ``FileFlag.CONFIGURED`` if the flag was
configured as requested; ``FileFlag.NOT_CONFIGURED`` if the flag
could not be configured as requested
"""
if flag:
# When checking if the file flag exists, attempt to update the access/
# modified times. For the case where may experience issues creating the
# file flag themselves (permission errors, etc.), fallback on just the
# existence of the file flag to still be considered as configured.
if touch(file):
rv = FileFlag.CONFIGURED
else:
if os.path.isfile(file):
rv = FileFlag.CONFIGURED
else:
rv = FileFlag.NOT_CONFIGURED
if not quiet:
err('unable to configure file flag: {}', file)
elif flag is None and os.path.isfile(file):
rv = FileFlag.EXISTS
else:
rv = FileFlag.NO_EXIST
return rv
def build(opts):
"""
support building autotools projects
With provided build options (``RelengBuildOptions``), the build stage will
be processed.
Args:
opts: build options
Returns:
``True`` if the building stage is completed; ``False`` otherwise
"""
if not MAKE.exists():
err('unable to build package; make is not installed')
return False
# definitions
autotools_defs = {}
if opts.build_defs:
autotools_defs.update(expand(opts.build_defs))
# default options
autotools_opts = {}
if opts.build_opts:
autotools_opts.update(expand(opts.build_opts))
# argument building
autotools_args = []
autotools_args.extend(prepare_definitions(autotools_defs))
autotools_args.extend(prepare_arguments(autotools_opts))
if opts.jobs > 1:
autotools_args.append('--jobs')
autotools_args.append(str(opts.jobs))
if not MAKE.execute(autotools_args, env=expand(opts.build_env)):
err('failed to build autotools project: {}', opts.name)
return False
return True
def path_remove(path, quiet=False):
"""
remove the provided path
Attempts to remove the provided path if it exists. The path value can either
be a directory or a specific file. If the provided path does not exist, this
method has no effect. In the event that a file or directory could not be
removed due to an error other than unable to be found, an error message will
be output to standard error (unless ``quiet`` is set to ``True``).
An example when using in the context of script helpers is as follows:
.. code-block:: python
releng_remove('my-file')
# (or)
releng_remove('my-directory/')
Args:
path: the path to remove
quiet (optional): whether or not to suppress output
Returns:
``True`` if the path was removed or does not exist; ``False`` if the
path could not be removed from the system
"""
if not os.path.exists(path):
return True
try:
if os.path.isdir(path) and not os.path.islink(path):
_path_remove_dir(path)
else:
_path_remove_file(path)
except OSError as e:
if e.errno != errno.ENOENT:
if not quiet:
err('unable to remove path: {}\n' ' {}', path, e)
return False
return True
def ensure_dir_exists(dir_, quiet=False, critical=False):
"""
ensure the provided directory exists
Attempts to create the provided directory. If the directory already exists,
this method has no effect. If the directory does not exist and could not be
created, this method will return ``False``. Also, if an error has been
detected, an error message will be output to standard error (unless
``quiet`` is set to ``True``).
An example when using in the context of script helpers is as follows:
.. code-block:: python
if releng_mkdir('my-directory'):
print('directory was created')
else:
print('directory was not created')
Args:
dir_: the directory
quiet (optional): whether or not to suppress output (defaults to
``False``)
critical (optional): whether or not to stop execution on failure
(defaults to ``False``)
Returns:
``True`` if the directory exists; ``False`` if the directory could not
be created
"""
try:
os.makedirs(dir_)
except OSError as e:
if e.errno != errno.EEXIST or not os.path.isdir(dir_):
if not quiet:
err('unable to create directory: {}\n' ' {}', dir_, e)
if critical:
sys.exit(-1)
return False
return True
def _create_bare_git_repo(cache_dir):
"""
create a bare git repository
This call will build a bare Git repository in the provided cache
directory. If the repository could not be created, an error message
will be generated and this method will return ``False``.
Args:
cache_dir: the cache/bare repository
Returns:
``True`` if the repository could be created; ``False`` otherwise
"""
git_dir = '--git-dir=' + cache_dir
if GIT.execute([git_dir, 'init', '--bare', '--quiet'], cwd=cache_dir):
return True
err('unable to initialize bare git repository')
return False
def run_script(script, globals_, subject=None, catch=True):
"""
execute the provided script and provide the resulting globals module
With the provided ``script`` file, execute the code and return the resulting
global module based on the execution results. This invoke is just a wrapper
call for ``run_path`` but with improved formatting for user feedback when
invoked in various stages of a releng-tool run. The provided ``globals``
will be passed into the ``run_path`` call.
When an issue occurs invoking the provided script, an error messaged is
output to standard error. This includes an error message (tailored, if
provided, by a ``subject`` value), the captured exception message and a
stack trace. This call will return ``None`` when an error is detected.
Args:
script: the script
globals_: dictionary to pre-populate script's globals
subject (optional): subject value to enhance a final error message
catch (optional): whether or not to catch any exceptions
Returns:
resulting globals module; ``None`` if an execution error occurs
"""
if not catch:
result = run_path(script, init_globals=globals_)
else:
try:
result = run_path(script, init_globals=globals_)
except Exception as e:
err('{}\n'
'error running {}{}script: {}\n'
' {}', traceback.format_exc(), subject, subject and ' ',
script, e)
return None
return result
def _stage_license(self, pkg):
"""
process license files for a specific package processing
If a package contains one or more files containing licenses information,
this information will be populated in the package's license folder.
Args:
pkg: the package being processed
Returns:
``True`` if the license information was copied; ``False`` if these
license information could not be copied
"""
# skip if package has no license files
if not pkg.license_files:
if pkg.license and not pkg.is_internal and not pkg.no_extraction:
warn('package defines no license files: ' + pkg.name)
return True
# ensure package-specific license directory exists
pkg_license_dir = os.path.join(self.opts.license_dir, pkg.nv)
if not ensure_dir_exists(pkg_license_dir):
return False
# copy over each license files
for file in pkg.license_files:
src = os.path.join(pkg.build_dir, file)
dst = os.path.join(pkg_license_dir, file)
if not path_copy(src, dst, critical=False):
err('unable to copy license information: ' + pkg.name)
return False
return True
def _sync_git_configuration(opts):
"""
ensure the git configuration is properly synchronized with this repository
This call ensures that various Git configuration options are properly
synchronized with the cached Git repository. This includes:
- Ensuring the configured site is set as the origin of the repository. This
is to help handle scenarios where a package's site has changed while
content is already cached.
- Ensure various `git config` options are set, if specific repository
options need to be set (e.g. overriding `core.autocrlf`).
Args:
opts: fetch options
Returns:
``True`` if the configuration has been synchronized; ``False`` otherwise
"""
cache_dir = opts.cache_dir
git_dir = '--git-dir=' + cache_dir
site = opts.site
if not _sync_git_origin(cache_dir, site):
return False
# apply repository-specific configurations
if opts._git_config:
for key, val in opts._git_config.items():
if not GIT.execute([git_dir, 'config', key, val], cwd=cache_dir):
err('unable to apply configuration entry "{}" with value "{}"',
key, val)
return False
return True
def configure(opts):
"""
support configuration for autotools projects
With provided configuration options (``RelengConfigureOptions``), the
configuration stage will be processed.
Args:
opts: configuration options
Returns:
``True`` if the configuration stage is completed; ``False`` otherwise
"""
# check if autoreconf
if opts._autotools_autoreconf:
verbose('configured to run autoreconf')
if not AUTORECONF.exists():
err('unable to configure package; autoreconf is not installed')
return False
if not AUTORECONF.execute(['--verbose']):
err('failed to prepare autotools project (autoreconf): {}',
opts.name)
return False
# definitions
autotools_defs = {
'--prefix': opts.prefix,
'--exec-prefix': opts.prefix,
}
if opts.conf_defs:
autotools_defs.update(expand(opts.conf_defs))
# default options
autotools_opts = {}
if opts.conf_opts:
autotools_opts.update(expand(opts.conf_opts))
# argument building
autotools_args = []
autotools_args.extend(prepare_definitions(autotools_defs))
autotools_args.extend(prepare_arguments(autotools_opts))
if not execute(['./configure'] + autotools_args,
env_update=expand(opts.conf_env),
critical=False):
err('failed to prepare autotools project (configure): {}', opts.name)
return False
return True
def path_copy(src, dst, quiet=False, critical=True, dst_dir=None):
"""
copy a file or directory into a target file or directory
This call will attempt to copy a provided file or directory, defined by
``src`` into a destination file or directory defined by ``dst``. If ``src``
is a file, then ``dst`` is considered to be a file or directory; if ``src``
is a directory, ``dst`` is considered a target directory. If a target
directory or target file's directory does not exist, it will be
automatically created. In the event that a file or directory could not be
copied, an error message will be output to standard error (unless ``quiet``
is set to ``True``). If ``critical`` is set to ``True`` and the specified
file/directory could not be copied for any reason, this call will issue a
system exit (``SystemExit``).
An example when using in the context of script helpers is as follows:
.. code-block:: python
# (stage)
# my-file
releng_copy('my-file', 'my-file2')
# (stage)
# my-file
# my-file2
releng_copy('my-file', 'my-directory/')
# (stage)
# my-directory/my-file
# my-file
# my-file2
releng_copy('my-directory/', 'my-directory2/')
# (stage)
# my-directory/my-file
# my-directory2/my-file
# my-file
# my-file2
Args:
src: the source directory or file
dst: the destination directory or file\\* (\\*if ``src`` is a file)
quiet (optional): whether or not to suppress output
critical (optional): whether or not to stop execution on failure
dst_dir (optional): force hint that the destination is a directory
Returns:
``True`` if the copy has completed with no error; ``False`` if the copy
has failed
Raises:
SystemExit: if the copy operation fails with ``critical=True``
"""
success = False
errmsg = None
try:
if os.path.isfile(src):
attempt_copy = True
if dst_dir:
base_dir = dst
else:
base_dir = os.path.dirname(dst)
if not os.path.isdir(base_dir):
attempt_copy = ensure_dir_exists(base_dir, quiet=quiet)
if attempt_copy:
if os.path.isdir(dst):
dst = os.path.join(dst, os.path.basename(src))
if os.path.islink(src):
target = os.readlink(src)
if os.path.islink(dst) or os.path.isfile(dst):
path_remove(dst)
os.symlink(target, dst)
else:
_copyfile(src, dst)
_copystat(src, dst)
success = True
elif os.path.exists(src):
if src == dst:
errmsg = "'{!s}' and '{!s}' " \
"are the same folder".format(src, dst)
elif _copy_tree(src, dst, quiet=quiet, critical=critical):
success = True
else:
errmsg = 'source does not exist: {}'.format(src)
except (IOError, ShutilError) as e:
errmsg = str(e)
if not quiet and errmsg:
err('unable to copy source contents to target location\n'
' {}', errmsg)
if not success and critical:
sys.exit(-1)
return success
def require_version(version, quiet=False, critical=True):
"""
perform a required-version check
Enables a caller to explicitly check for a required releng-tool version.
Invoking this function with a dotted-separated ``version`` string, the
string will be parsed and compared with the running releng-tool version.
If the required version is met, this method will have no effect. In the
event that the required version is not met, the exception ``SystemExit``
will be raised if the critical flag is set; otherwise this call will
return ``False``.
An example when using in the context of script helpers is as follows:
.. code-block:: python
# ensure we are using releng-tool v1
releng_require_version('1.0.0')
Args:
version: dotted-separated version string
quiet (optional): whether or not to suppress output
critical (optional): whether or not to stop execution on failure
Returns:
``True`` if the version check is met; ``False`` if the version check
has failed
Raises:
SystemExit: if the version check fails with ``critical=True``
"""
rv = True
if version:
requested = version.split('.')
current = releng_version.split('.')
rv = requested <= current
if not rv:
if not quiet:
args = {
'detected': releng_version,
'required': version,
}
err('''
required releng-tool version check has failed
This project has indicated a required minimum version of releng-tool to
be installed on this system; however, an older version has been
detected:
(required) {required}
(detected) {detected}
Please update to a more recent version:
https://docs.releng.io/install/
'''.strip().format(**args))
if critical:
sys.exit(-1)
return rv
def fetch(opts):
"""
support fetching from cvs sources
With provided fetch options (``RelengFetchOptions``), the fetch stage will
be processed.
Args:
opts: fetch options
Returns:
``True`` if the fetch stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
name = opts.name
revision = opts.revision
site = opts.site
work_dir = opts.work_dir
cache_basename = os.path.basename(cache_file)
cache_stem, __ = interpret_stem_extension(cache_basename)
if not CVS.exists():
err('unable to fetch package; cvs is not installed')
return None
note('fetching {}...', name)
sys.stdout.flush()
try:
cvsroot, module = site.rsplit(' ', 1)
except ValueError:
err('''\
improper cvs site defined
The provided CVS site does not define both the CVSROOT as well as the target
module to checkout. For example:
:pserver:[email protected]:/var/lib/cvsroot mymodule
Site: {}''', site)
return None
log('checking out sources')
if not CVS.execute(['-d', cvsroot, 'checkout', '-d', cache_stem,
'-r', revision, module], cwd=work_dir):
err('unable to checkout module')
return None
cvs_module_dir = os.path.join(work_dir, cache_stem)
if not os.path.exists(cvs_module_dir):
err('no sources available for the provided revision')
return None
log('caching sources')
def cvs_filter(info):
if info.name.endswith('CVS'):
return None
return info
cache_dir = os.path.abspath(os.path.join(cache_file, os.pardir))
if not ensure_dir_exists(cache_dir):
return None
with tarfile.open(cache_file, 'w:gz') as tar:
tar.add(cvs_module_dir, arcname=cache_stem, filter=cvs_filter)
return cache_file
def extract(opts):
"""
support extraction of an archive into a build directory
With provided extraction options (``RelengExtractOptions``), the extraction
stage will be processed. The archive's extension will be used in attempt to
finding a matching tool/implementation which can be used to extract the
contents of the file. In the event that the method of extraction cannot be
determined, it will be assumed that the file is in fact not extractable.
Files which are not extracted are just copied into the build directly (e.g.
single resource files).
Args:
opts: the extraction options
Returns:
``True`` if the extraction stage is completed; ``False`` otherwise
"""
assert opts
cache_file = opts.cache_file
strip_count = opts.strip_count
work_dir = opts.work_dir
cache_basename = os.path.basename(cache_file)
__, cache_ext = interpret_stem_extension(cache_basename)
is_extractable = False
if cache_ext:
cache_ext = cache_ext.lower()
# if the user defines a tool override for this extension type, use
# whatever the user wants to use (passing the file and directory to
# extract to)
extract_override = getattr(opts, '_extract_override', None)
if extract_override and cache_ext in extract_override:
is_extractable = True
tool_cmd = extract_override[cache_ext].format(file=cache_file,
dir=work_dir)
if not execute(tool_cmd.split(), cwd=work_dir, critical=False):
err('unable to extract with tool override\n'
' (command: {})', tool_cmd)
return None
# attempt to extract the (compressed) tar archive with the host's
# tar tool; if it does not exist, we'll fallback to using python's
# internal implementation (tarfile)
elif cache_ext.startswith(TAR_SUPPORTED):
is_extractable = True
# before attempting to use an external tar command, only allow
# using it if the `force-local` option is available whenever a
# colon character is provided, to prevent tar from assuming the
# path is a remote target
needs_force_local = False
if ':' in cache_file:
needs_force_local = True
has_extracted = False
if TAR.exists() and (TAR.force_local or not needs_force_local):
tar_args = [
'--extract',
'--file=' + cache_file,
'--strip-components={}'.format(strip_count),
'--verbose',
]
if needs_force_local:
tar_args.append('--force-local')
if TAR.execute(tar_args, cwd=work_dir):
has_extracted = True
else:
warn('unable to extract archive with host tar; '
'will use fallback')
if not has_extracted:
try:
def tar_extract(members, strip_count):
for member in members:
# strip members from package defined count
if strip_count > 0:
np = os.path.normpath(member.name)
parts = np.split(os.path.sep, strip_count)
if len(parts) <= strip_count:
continue
member.name = parts[-1]
# notify the user of the target member to extract
print(member.name)
yield member
with tarfile.open(cache_file, 'r') as tar:
tar.extractall(path=work_dir,
members=tar_extract(tar, strip_count))
except Exception as e:
err(
'unable to extract tar file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
# extract a zip-extension cache file using python's internal
# implementation (zipfile)
elif cache_ext == 'zip':
is_extractable = True
try:
with ZipFile(cache_file, 'r') as zip_:
for member in zip_.namelist():
# strip members from package defined count
member_s = member
if strip_count > 0:
np = os.path.normpath(member_s)
parts = np.split(os.path.sep, strip_count)
if len(parts) <= strip_count:
continue
member_s = parts[-1]
dest = os.path.join(work_dir, member_s)
# notify the user of the target member to extract
print(member)
# if this is a directory entry, ensure the directory
# exists for the destination
if not os.path.basename(member):
ensure_dir_exists(dest)
else:
# always ensure the container directory for a file
# exists before attempting to extract a member into
# it, as not all processed zip files may process
# a directory entry (to be created) ahead of time
ensure_dir_exists(os.path.dirname(dest))
with zip_.open(member) as s, open(dest, 'wb') as f:
shutil.copyfileobj(s, f)
except Exception as e:
err(
'unable to extract zip file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
if not is_extractable:
debug('file not considered extractable: ' + cache_file)
try:
shutil.copy2(cache_file, work_dir)
except IOError as e:
err(
'unable to copy over cache file\n'
' {}\n'
' (file: {})\n'
' (target: {})', e, cache_file, work_dir)
return False
return True
def check(self, quiet=False):
"""
check for the existence of required tools for the loaded package set
For each loaded package, a series of required host tools will be checked
and a caller will be notified whether or not anything is missing.
Args:
quiet (optional): whether or not to suppress output (defaults to
``False``)
Returns:
``True`` is all known required tools exists; ``False`` otherwise
"""
missing = set()
pkg_types = set()
python_interpreters = set()
vcs_types = set()
# package-defined requirements check
for pkg in self.pkgs:
pkg_types.add(pkg.type)
vcs_types.add(pkg.vcs_type)
if pkg.type == PackageType.AUTOTOOLS:
if pkg.autotools_autoreconf:
if AUTORECONF.exists():
self._verbose_exists(AUTORECONF)
else:
missing.add(AUTORECONF.tool)
elif pkg.type == PackageType.PYTHON:
if pkg.python_interpreter:
python_tool = PythonTool(pkg.python_interpreter)
else:
python_tool = PYTHON
python_interpreters.add(python_tool)
if PackageType.AUTOTOOLS in pkg_types:
if MAKE.exists():
self._verbose_exists(MAKE)
else:
missing.add(MAKE.tool)
if PackageType.CMAKE in pkg_types:
if CMAKE.exists():
self._verbose_exists(CMAKE)
else:
missing.add(CMAKE.tool)
if PackageType.PYTHON in pkg_types:
for interpreter in python_interpreters:
if interpreter.exists():
self._verbose_exists(interpreter)
else:
missing.add(interpreter.tool)
if VcsType.BZR in vcs_types:
if BZR.exists():
self._verbose_exists(BZR)
else:
missing.add(BZR.tool)
if VcsType.CVS in vcs_types:
if CVS.exists():
self._verbose_exists(CVS)
else:
missing.add(CVS.tool)
if VcsType.GIT in vcs_types:
if GIT.exists():
self._verbose_exists(GIT)
else:
missing.add(GIT.tool)
if VcsType.HG in vcs_types:
if HG.exists():
self._verbose_exists(HG)
else:
missing.add(HG.tool)
if VcsType.RSYNC in vcs_types:
if RSYNC.exists():
self._verbose_exists(RSYNC)
else:
missing.add(RSYNC.tool)
if VcsType.SCP in vcs_types:
if SCP.exists():
self._verbose_exists(SCP)
else:
missing.add(SCP.tool)
if VcsType.SVN in vcs_types:
if SVN.exists():
self._verbose_exists(SVN)
else:
missing.add(SVN.tool)
# project-provided tools check
for tool in self.tools:
if which(tool):
verbose('prerequisite exists: ' + tool)
else:
missing.add(tool)
if missing and not quiet:
sorted_missing = list(missing)
sorted_missing.sort()
msg = 'missing the following host tools for this project:'
msg += '\n'
msg += '\n'
for entry in sorted_missing:
msg += ' ' + entry + '\n'
err(msg)
return len(missing) == 0
def initialize_sample(opts):
"""
initialize a sample project
Generates a sample provided in the root directory to help new users or new
project get started.
Args:
opts: options for this run
Returns:
``True`` if the sample project could be initialized; ``False`` if an
issue has occurred generating the sample project
"""
root_dir = opts.root_dir
if not ensure_dir_exists(root_dir):
return False
if os.listdir(root_dir):
err('unable to initialize sample project is non-empty directory')
return False
sample_dir = os.path.join(root_dir, 'package', 'sample')
success = True
if ensure_dir_exists(sample_dir):
# sample project
sample_defs = os.path.join(root_dir, 'package', 'sample', 'sample')
try:
with open(sample_defs, 'w') as f:
f.write('''\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
SAMPLE_DEPENDENCIES = []
SAMPLE_LICENSE = ['<license name>']
SAMPLE_LICENSE_FILES = ['<license file>']
SAMPLE_SITE = '<location for sources>'
SAMPLE_TYPE = '<package-type>'
SAMPLE_VERSION = '<package-version>'
''')
verbose('written sample file')
except IOError as e:
err('unable to generate a sample file')
verbose(str(e))
success = False
else:
success = False
# .gitignore
try:
project_gitignore = os.path.join(root_dir,
'.gitignore') # (assumption)
with open(project_gitignore, 'w') as f:
f.write('''\
# releng-tool
/cache/
/dl/
/output/
.releng-flag-*
''')
verbose('written .gitignore file')
except IOError as e:
err('unable to generate a .gitignore file')
verbose(str(e))
success = False
# releng project
try:
project_defs = os.path.join(root_dir, 'releng')
with open(project_defs, 'w') as f:
f.write('''\
#!/usr/bin/env python
# -*- coding: utf-8 -*-
packages = [
'sample',
]
''')
verbose('written releng file')
except IOError as e:
err('unable to generate a releng file')
verbose(str(e))
success = False
if success:
log('initialized empty releng-tool project')
else:
warn('partially initialized a releng-tool project')
return success
def stage(engine, pkg, script_env):
"""
handles the configuration stage for a package
With a provided engine and package instance, the configuration stage will be
processed.
Args:
engine: the engine
pkg: the package being configured
script_env: script environment information
Returns:
``True`` if the configuration stage is completed; ``False`` otherwise
"""
note('configuring {}...', pkg.name)
sys.stdout.flush()
# ignore configuration step for types which do not have one
if pkg.type == PackageType.PYTHON:
return True
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
pkg_install_type = NC(pkg.install_type, PackageInstallType.TARGET)
configure_opts = RelengConfigureOptions()
replicate_package_attribs(configure_opts, pkg)
configure_opts.build_dir = build_dir
configure_opts.build_output_dir = pkg.build_output_dir
configure_opts.conf_defs = pkg.conf_defs
configure_opts.conf_env = pkg.conf_env
configure_opts.conf_opts = pkg.conf_opts
configure_opts.def_dir = pkg.def_dir
configure_opts.env = script_env
configure_opts.ext = pkg.ext_modifiers
configure_opts.host_dir = engine.opts.host_dir
configure_opts.install_type = pkg_install_type
configure_opts.name = pkg.name
configure_opts.prefix = NC(pkg.prefix, engine.opts.sysroot_prefix)
configure_opts.staging_dir = engine.opts.staging_dir
configure_opts.symbols_dir = engine.opts.symbols_dir
configure_opts.target_dir = engine.opts.target_dir
configure_opts.version = pkg.version
configure_opts._quirks = engine.opts.quirks
# if package has a job-override value, use it over any global option
if pkg.fixed_jobs:
configure_opts.jobs = pkg.fixed_jobs
configure_opts.jobsconf = pkg.fixed_jobs
else:
configure_opts.jobs = engine.opts.jobs
configure_opts.jobsconf = engine.opts.jobsconf
configurer = None
if pkg.type in engine.registry.package_types:
def _(opts):
return engine.registry.package_types[pkg.type].configure(
pkg.type, opts)
configurer = _
elif pkg.type == PackageType.AUTOTOOLS:
configurer = conf_autotools
elif pkg.type == PackageType.CMAKE:
configurer = conf_cmake
elif pkg.type == PackageType.SCRIPT:
configurer = conf_script
if not configurer:
err('configurer type is not implemented: {}', pkg.type)
return False
with interim_working_dir(build_dir):
configured = configurer(configure_opts)
if not configured:
return False
return True
def stage(engine, pkg, script_env):
"""
handles the installation stage for a package
With a provided engine and package instance, the installation stage will be
processed.
Args:
engine: the engine
pkg: the package being built
script_env: script environment information
Returns:
``True`` if the installation stage is completed; ``False`` otherwise
"""
note('installing {}...', pkg.name)
sys.stdout.flush()
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
pkg_install_type = NC(pkg.install_type, PackageInstallType.TARGET)
if pkg_install_type == PackageInstallType.HOST:
dest_dirs = [engine.opts.host_dir]
elif pkg_install_type == PackageInstallType.IMAGES:
dest_dirs = [engine.opts.images_dir]
elif pkg_install_type == PackageInstallType.STAGING:
dest_dirs = [engine.opts.staging_dir]
elif pkg_install_type == PackageInstallType.STAGING_AND_TARGET:
dest_dirs = [engine.opts.staging_dir, engine.opts.target_dir]
else:
# default to target directory
dest_dirs = [engine.opts.target_dir]
install_opts = RelengInstallOptions()
replicate_package_attribs(install_opts, pkg)
install_opts.build_dir = build_dir
install_opts.build_output_dir = pkg.build_output_dir
install_opts.cache_file = pkg.cache_file
install_opts.def_dir = pkg.def_dir
install_opts.dest_dirs = dest_dirs
install_opts.env = script_env
install_opts.ext = pkg.ext_modifiers
install_opts.host_dir = engine.opts.host_dir
install_opts.images_dir = engine.opts.images_dir
install_opts.install_defs = pkg.install_defs
install_opts.install_env = pkg.install_env
install_opts.install_opts = pkg.install_opts
install_opts.install_type = pkg_install_type
install_opts.name = pkg.name
install_opts.prefix = NC(pkg.prefix, engine.opts.sysroot_prefix)
install_opts.staging_dir = engine.opts.staging_dir
install_opts.symbols_dir = engine.opts.symbols_dir
install_opts.target_dir = engine.opts.target_dir
install_opts.version = pkg.version
install_opts._quirks = engine.opts.quirks
installer = None
if pkg.type in engine.registry.package_types:
def _(opts):
return engine.registry.package_types[pkg.type].install(
pkg.type, opts)
installer = _
elif pkg.type == PackageType.AUTOTOOLS:
installer = install_autotools
elif pkg.type == PackageType.CMAKE:
installer = install_cmake
elif pkg.type == PackageType.PYTHON:
installer = install_python
elif pkg.type == PackageType.SCRIPT:
installer = install_script
if not installer:
err('installer type is not implemented: {}', pkg.type)
return False
with interim_working_dir(build_dir):
installed = installer(install_opts)
if not installed:
return False
return True
def stage(engine, pkg, script_env):
"""
handles the patching stage for a package
With a provided engine and package instance, the patching stage will be
processed.
Args:
engine: the engine
pkg: the package being patched
script_env: script environment information
Returns:
``True`` if the patching stage is completed; ``False`` otherwise
"""
if pkg.is_internal:
# packages flagged for local sources do not have a patch stage
if pkg.local_srcs:
return True
# internal packages in development mode that specify a development
# revision will not perform the patch stage
if engine.opts.devmode and pkg.has_devmode_option:
return True
note('patching {}...', pkg.name)
sys.stdout.flush()
if pkg.build_subdir:
build_dir = pkg.build_subdir
else:
build_dir = pkg.build_dir
patch_script_filename = '{}-{}'.format(pkg.name, PATCH_SCRIPT)
patch_script = os.path.join(pkg.def_dir, patch_script_filename)
if os.path.isfile(patch_script):
try:
run_path(patch_script, init_globals=script_env)
verbose('patch script executed: ' + patch_script)
except Exception as e:
err('error running patch script: \n' ' {}', patch_script, e)
return False
# find all patches in the package's folder, sort and apply each
patch_glob = os.path.join(pkg.def_dir, '*.patch')
patches = glob(patch_glob)
if patches:
patches = sorted(patches)
if not PATCH.exists():
err('unable to apply patches; patch is not installed')
return False
for patch in patches:
print('({})'.format(os.path.basename(patch)))
if not PATCH.execute([
'--batch',
'--forward',
'--ignore-whitespace',
'--input={}'.format(patch),
'--strip=1',
],
cwd=build_dir):
err('failed to apply patch')
return False
return True
