def validate(self, root_artifact):
root_arch = root_artifact.source.morphology['arch']
target_arch = arch
if root_arch != target_arch:
raise morphlib.Error('Target architecture is %s '
'but the system architecture is %s' %
(target_arch, root_arch))
def add_binary(self, binaries):
'''Add a binary file to the current repository.
Command line argument:
* `FILENAME...` is the binaries to be added to the repository.
This checks for the existence of a .gitfat file in the repository. If
there is one then a line is added to .gitattributes telling it that
the given binary should be handled by git-fat. If there is no .gitfat
file then it is created, with the rsync remote pointing at the correct
directory on the Trove host. A line is then added to .gitattributes to
say that the given binary should be handled by git-fat.
Example:
morph add-binary big_binary.tar.gz
'''
if not binaries:
raise morphlib.Error('add-binary must get at least one argument')
gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True)
gd.fat_init()
if not gd.has_fat():
self._make_gitfat(gd)
self._handle_binaries(binaries, gd)
logging.info('Staged binaries for commit')
def _validate_cross_morphology_references(self, srcpool):
'''Perform validation across all morphologies involved in the build'''
stratum_names = {}
for src in srcpool:
kind = src.morphology['kind']
# Verify that chunks pointed to by strata really are chunks, etc.
method_name = '_validate_cross_refs_for_%s' % kind
if hasattr(self, method_name):
logging.debug('Calling %s' % method_name)
getattr(self, method_name)(src, srcpool)
else:
logging.warning('No %s' % method_name)
# Verify stratum build-depends agree with the system's contents.
# It is permissible for a stratum to build-depend on a stratum that
# isn't specified in the target system morphology.
# Multiple references to the same stratum are permitted. This is
# handled by the SourcePool deduplicating added Sources.
# It is forbidden to have two different strata with the same name.
# Hence if a Stratum is defined in the System, and in a Stratum as
# a build-dependency, then they must both have the same Repository
# and Ref specified.
if src.morphology['kind'] == 'stratum':
name = src.name
if name in stratum_names:
raise morphlib.Error(
"Multiple strata produce a '%s' artifact: %s and %s" %
(name, stratum_names[name].filename, src.filename))
stratum_names[name] = src
def _expand(protocol, path):
if protocol == "git":
return "git://%s/%s/%%s" % (trove_host, path)
elif protocol == "ssh":
return "ssh://git@%s/%s/%%s" % (trove_host, path)
else:
raise morphlib.Error('Unknown protocol in trove_id: %s' % protocol)
def pull(self, args):
'''Pull changes to the current branch from a repository.
Command line arguments:
* `REMOTE` is the remote branch to pull from. By default this is the
branch being tracked by your current git branch (ie origin/master
for branch master)
This is a wrapper for the `git pull` command. It also deals with
pulling any binary files that have been added to the repository using
git-fat.
Example:
morph pull
'''
if len(args) > 1:
raise morphlib.Error('pull takes at most one argument')
gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True)
remote = gd.get_remote('origin')
if args:
branch = args[0]
remote.pull(branch)
else:
remote.pull()
if gd.has_fat():
gd.fat_init()
gd.fat_pull()
def get_host_architecture(): # pragma: no cover
'''Get the canonical Morph name for the host's architecture.'''
machine = os.uname()[-1]
table = {
'x86_64': 'x86_64',
'i386': 'x86_32',
'i486': 'x86_32',
'i586': 'x86_32',
'i686': 'x86_32',
'armv7l': 'armv7l',
'armv7b': 'armv7b',
'armv8l': 'armv8l',
'armv8b': 'armv8b',
'aarch64': 'armv8l64',
'aarch64b': 'armv8b64',
'mips': 'mips32',
'mips64': 'mips64',
'ppc64': 'ppc64'
}
if machine not in table:
raise morphlib.Error('Unknown host architecture %s' % machine)
if machine == 'armv7l' and has_hardware_fp():
return 'armv7lhf'
elif machine in ('mips', 'mips64'):
if determine_endianness() == 'big':
return table[machine] + 'b'
else:
return table[machine] + 'l'
return table[machine]
def push(self, args):
'''Push a branch to a remote repository.
Command line arguments:
* `REPO` is the repository to push your changes to.
* `TARGET` is the branch to push to the repository.
This is a wrapper for the `git push` command. It also deals with
pushing any binary files that have been added using git-fat.
Example:
morph push origin jrandom/new-feature
'''
if len(args) != 2:
raise morphlib.Error('push must get exactly two arguments')
gd = morphlib.gitdir.GitDirectory(os.getcwd(), search_for_root=True)
remote, branch = args
rs = morphlib.gitdir.RefSpec(branch)
gd.get_remote(remote).push(rs)
if gd.has_fat():
gd.fat_init()
gd.fat_push()
def init(self, args):
'''Initialize a workspace directory.
Command line argument:
* `DIR` is the directory to use as a workspace, and defaults to
the current directory.
This creates a workspace, either in the current working directory,
or if `DIR` is given, in that directory. If the directory doesn't
exist, it is created. If it does exist, it must be empty.
You need to run `morph init` to initialise a workspace, or none
of the other system branching tools will work: they all assume
an existing workspace. Note that a workspace only exists on your
machine, not on the git server.
Example:
morph init /src/workspace
cd /src/workspace
'''
if not args:
args = ['.']
elif len(args) > 1:
raise morphlib.Error('init must get at most one argument')
ws = morphlib.workspace.create(args[0])
self.app.status(msg='Initialized morph workspace', chatty=True)
def _parse_size_from_environment(self, env_var, default):
'''Parse a size from an environment variable.'''
size = os.environ.get(env_var, default)
if size is None:
return None
bytes = self._parse_size(size)
if bytes is None:
raise morphlib.Error('Cannot parse %s value %s' % (env_var, size))
return bytes
def _validate_architecture(self, root_artifact):
'''Perform the validation between root and target architectures.'''
root_arch = root_artifact.source.morphology['arch']
host_arch = morphlib.util.get_host_architecture()
if root_arch != host_arch:
raise morphlib.Error(
'Are you trying to cross-build? '
'Host architecture is %s but target is %s'
% (host_arch, root_arch))
def combine_aliases(app): # pragma: no cover
'''Create a full repo-alias set from the app's settings.
The standard 'baserock:' and 'upstream:' keyed URLs use whatever trove was
set as 'trove-host'.
Every trove listed in 'trove-ids' has its own repo-alias created in
addition to the defaults. We assume these require authenticated access, so
the keyed URL expansions for these troves are ssh:// URLs for both read and
write access. This can be overridden by the user if they calculate the full
repo-alias string and set it in their config manually.
'''
trove_host = app.settings['trove-host']
trove_ids = app.settings['trove-id']
repo_aliases = app.settings['repo-alias']
repo_pat = r'^(?P<prefix>[a-z][a-z0-9-]+)=(?P<pull>[^#]+)#(?P<push>[^#]+)$'
trove_pat = (r'^(?P<prefix>[a-z][a-z0-9-]+)=(?P<path>[^#]+)#'
'(?P<pull>[^#]+)#(?P<push>[^#]+)$')
alias_map = {}
def _expand(protocol, path):
if protocol == "git":
return "git://%s/%s/%%s" % (trove_host, path)
elif protocol == "ssh":
return "ssh://git@%s/%s/%%s" % (trove_host, path)
else:
raise morphlib.Error('Unknown protocol in trove_id: %s' % protocol)
if trove_host:
alias_map['baserock'] = "baserock=%s#%s" % (_expand(
'git', 'baserock'), _expand('ssh', 'baserock'))
alias_map['upstream'] = "upstream=%s#%s" % (_expand(
'git', 'delta'), _expand('ssh', 'delta'))
for trove_id in trove_ids:
m = re.match(trove_pat, trove_id)
if m:
alias_map[m.group('prefix')] = "%s=%s#%s" % (
m.group('prefix'), _expand(m.group('pull'),
m.group('path')),
_expand(m.group('push'), m.group('path')))
elif '=' not in trove_id and trove_id not in alias_map:
alias_map[trove_id] = "%s=%s#%s" % (trove_id,
_expand('ssh', trove_id),
_expand('ssh', trove_id))
for repo_alias in repo_aliases:
m = re.match(repo_pat, repo_alias)
if m:
alias_map[m.group('prefix')] = repo_alias
else:
raise morphlib.Error('Invalid repo-alias: %s' % repo_alias)
return alias_map.values()
def find_initramfs(self, temp_root):
'''Check whether the rootfs has an initramfs.
Uses the INITRAMFS_PATH option to locate it.
'''
if 'INITRAMFS_PATH' in os.environ:
initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH'])
if not os.path.exists(initramfs):
raise morphlib.Error('INITRAMFS_PATH specified, '
'but file does not exist')
return initramfs
return None
def _validate_has_non_bootstrap_chunks(srcpool):
stratum_sources = [src for src in srcpool
if src.morphology['kind'] == 'stratum']
# any will return true for an empty iterable, which will give
# a false positive when there are no strata.
# This is an error by itself, but the source of this error can
# be better diagnosed later, so we abort validating here.
if not stratum_sources:
return
if not any(spec.get('build-mode', 'staging') != 'bootstrap'
for src in stratum_sources
for spec in src.morphology['chunks']):
raise morphlib.Error('No non-bootstrap chunks found.')
def _validate_cross_refs_for_xxx(self, src, srcpool, specs, wanted):
for spec in specs:
repo_name = spec.get('repo') or src.repo_name
ref = spec.get('ref') or src.original_ref
filename = morphlib.util.sanitise_morphology_path(
spec.get('morph', spec.get('name')))
logging.debug('Validating cross ref to %s:%s:%s' %
(repo_name, ref, filename))
for other in srcpool.lookup(repo_name, ref, filename):
if other.morphology['kind'] != wanted:
raise morphlib.Error(
'%s %s references %s:%s:%s which is a %s, '
'instead of a %s' %
(src.morphology['kind'], src.name, repo_name, ref,
filename, other.morphology['kind'], wanted))
def build(self, repo_name, ref, filename, original_ref=None):
'''Initiate a distributed build on a controller'''
distbuild.add_crash_conditions(self.app.settings['crash-condition'])
if self.addr == '':
raise morphlib.Error(
'Need address of controller to run a distbuild')
self.app.status(msg='Starting distributed build')
loop = distbuild.MainLoop()
args = [repo_name, ref, filename, original_ref or ref]
cm = distbuild.InitiatorConnectionMachine(
self.app, self.addr, self.port, distbuild.Initiator,
[self.app] + args, self.RECONNECT_INTERVAL, self.MAX_RETRIES)
loop.add_state_machine(cm)
loop.run()
def _validate_architecture(self, root_artifact):
'''Perform the validation between root and target architectures.'''
root_arch = root_artifact.source.morphology['arch']
host_arch = morphlib.util.get_host_architecture()
if root_arch == host_arch:
return
# Since the armv8 instruction set is nearly entirely armv7 compatible,
# and since the incompatibilities are appropriately trapped in the
# kernel, we can safely run any armv7 toolchain natively on armv8.
if host_arch == 'armv8l' and root_arch in ('armv7l', 'armv7lhf'):
return
if host_arch == 'armv8b' and root_arch in ('armv7b', 'armv7bhf'):
return
raise morphlib.Error(
'Are you trying to cross-build? Host architecture is %s but '
'target is %s' % (host_arch, root_arch))
def get_host_architecture(): # pragma: no cover
'''Get the canonical Morph name for the host's architecture.'''
machine = os.uname()[-1]
table = {
'x86_64': 'x86_64',
'i386': 'x86_32',
'i486': 'x86_32',
'i586': 'x86_32',
'i686': 'x86_32',
'armv7l': 'armv7l',
'armv7b': 'armv7b',
'ppc64': 'ppc64'
}
if machine not in table:
raise morphlib.Error('Unknown host architecture %s' % machine)
if machine == 'armv7l' and has_hardware_fp():
return 'armv7lhf'
return table[machine]
def sanitise_morphology_path(morph_field, morph_kind, belongs_to='None'):
'''This function receives the name or the morph field of one morphology
and returns the path of the morphology depending on the name, kind and
if it belongs to other morphologies.
'''
# Dictionary which match morphology's kind and morphology's
# directory in definitions.git
morph_dir = { 'chunk': 'chunks', 'stratum': 'strata',
'system':'systems', 'cluster': 'clusters'}
# For chunks morphologies we need to know to which stratums
# belongs this chunk.
if morph_kind == 'chunk':
if belongs_to == '':
raise morphlib.Error('Chunk morphologies need the stratum name'
'to create the path. Please add the stratum'
'which belongs this morphology')
# Get the name of the chunk which we assume is at the end
# of the morph file
if '/' in morph_field:
morph_field = os.path.basename(morph_field)
# Add the stratum name to the chunk name
morph_field = os.path.join(belongs_to, morph_field)
# Reset the kind to stratum because chunk contains stratum
# name in its path.
morph_kind = 'stratum'
# Add the morphology path to the morph field.
if not morph_field.startswith(morph_dir[morph_kind]):
morph_field = os.path.join(morph_dir[morph_kind], morph_field)
# Add the morphology suffix if the morphology.
if not morph_field.endswith('.morph'):
morph_field = morph_field + '.morph'
return morph_field
def check_disk_available(tmp_path, tmp_min_size, cache_path,
cache_min_size): # pragma: no cover
# if both are on the same filesystem, assume they share a storage pool,
# so the sum of the two sizes needs to be available
# TODO: if we need to do this on any more than 2 paths
# extend it to take a [(path, min)]
tmp_min_size, cache_min_size = unify_space_requirements(
tmp_path, tmp_min_size, cache_path, cache_min_size)
tmp_size, cache_size = map(get_bytes_free_in_path, (tmp_path, cache_path))
errors = []
for path, min in [(tmp_path, tmp_min_size), (cache_path, cache_min_size)]:
free = get_bytes_free_in_path(path)
if free < min:
errors.append('\t%(path)s requires %(min)d bytes free, '
'has %(free)d' % locals())
if not errors:
return
raise morphlib.Error('Insufficient space on disk:\n' + '\n'.join(errors) +
'\n'
'Please run `morph gc`. If the problem persists '
'increase the disk size, manually clean up some '
'space or reduce the disk space required by the '
'tempdir-min-space and cachedir-min-space '
'configuration options.')
def deploy_system(self, build_command, deploy_tempdir, root_repo_dir,
build_repo, ref, system, env_vars, deployment_filter,
parent_location):
sys_ids = set(system['deploy'].iterkeys())
if deployment_filter and not \
any(sys_id in deployment_filter for sys_id in sys_ids):
return
old_status_prefix = self.app.status_prefix
system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph'])
self.app.status_prefix = system_status_prefix
try:
# Find the artifact to build
morph = morphlib.util.sanitise_morphology_path(system['morph'])
srcpool = build_command.create_source_pool(build_repo, ref, morph)
artifact = build_command.resolve_artifacts(srcpool)
deploy_defaults = system.get('deploy-defaults', {})
for system_id, deploy_params in system['deploy'].iteritems():
if not system_id in deployment_filter and deployment_filter:
continue
deployment_status_prefix = '%s[%s]' % (system_status_prefix,
system_id)
self.app.status_prefix = deployment_status_prefix
try:
user_env = morphlib.util.parse_environment_pairs(
os.environ, [
pair[len(system_id) + 1:]
for pair in env_vars if pair.startswith(system_id)
])
final_env = dict(deploy_defaults.items() +
deploy_params.items() + user_env.items())
is_upgrade = ('yes'
if self.app.settings['upgrade'] else 'no')
final_env['UPGRADE'] = is_upgrade
deployment_type = final_env.pop('type', None)
if not deployment_type:
raise morphlib.Error('"type" is undefined '
'for system "%s"' % system_id)
location = final_env.pop('location', None)
if not location:
raise morphlib.Error('"location" is undefined '
'for system "%s"' % system_id)
morphlib.util.sanitize_environment(final_env)
self.check_deploy(root_repo_dir, ref, deployment_type,
location, final_env)
system_tree = self.setup_deploy(build_command,
deploy_tempdir,
root_repo_dir, ref,
artifact, deployment_type,
location, final_env)
for subsystem in system.get('subsystems', []):
self.deploy_system(build_command,
deploy_tempdir,
root_repo_dir,
build_repo,
ref,
subsystem,
env_vars, [],
parent_location=system_tree)
if parent_location:
deploy_location = os.path.join(parent_location,
location.lstrip('/'))
else:
deploy_location = location
self.run_deploy_commands(deploy_tempdir, final_env,
artifact, root_repo_dir, ref,
deployment_type, system_tree,
deploy_location)
finally:
self.app.status_prefix = system_status_prefix
finally:
self.app.status_prefix = old_status_prefix
def cross_bootstrap(self, args):
'''Cross-bootstrap a system from a different architecture.'''
# A brief overview of this process: the goal is to native build as much
# of the system as possible because that's easier, but in order to do
# so we need at least 'build-essential'. 'morph cross-bootstrap' will
# cross-build any bootstrap-mode chunks in the given system and
# will then prepare a large rootfs tarball which, when booted, will
# build the rest of the chunks in the system using the cross-built
# build-essential.
#
# This approach saves us from having to run Morph, Git, Python, Perl,
# or anything else complex and difficult to cross-build on the target
# until it is bootstrapped. The user of this command needs to provide
# a kernel and handle booting the system themselves (the generated
# tarball contains a /bin/sh that can be used as 'init'.
#
# This function is a variant of the BuildCommand() class in morphlib.
# To do: make it work on a system branch instead of repo/ref/morph
# triplet.
if len(args) < 4:
raise cliapp.AppException(
'cross-bootstrap requires 4 arguments: target archicture, and '
'repo, ref and and name of the system morphology')
arch = args[0]
root_repo, ref, system_name = args[1:4]
if arch not in morphlib.valid_archs:
raise morphlib.Error('Unsupported architecture "%s"' % arch)
# Get system artifact
build_env = morphlib.buildenvironment.BuildEnvironment(
self.app.settings, arch)
build_command = morphlib.buildcommand.BuildCommand(self.app, build_env)
morph_name = morphlib.util.sanitise_morphology_path(system_name)
srcpool = build_command.create_source_pool(root_repo, ref, morph_name)
# FIXME: this is a quick fix in order to get it working for
# Baserock 13 release, it is not a reasonable fix
def validate(self, root_artifact):
root_arch = root_artifact.source.morphology['arch']
target_arch = arch
if root_arch != target_arch:
raise morphlib.Error('Target architecture is %s '
'but the system architecture is %s' %
(target_arch, root_arch))
morphlib.buildcommand.BuildCommand._validate_architecture = validate
system_artifact = build_command.resolve_artifacts(srcpool)
# Calculate build order
# This is basically a hacked version of BuildCommand.build_in_order()
sources = build_command.get_ordered_sources(system_artifact.walk())
cross_sources = []
native_sources = []
for s in sources:
if s.morphology['kind'] == 'chunk':
if s.build_mode == 'bootstrap':
cross_sources.append(s)
else:
native_sources.append(s)
if len(cross_sources) == 0:
raise morphlib.Error(
'Nothing to cross-compile. Only chunks built in \'bootstrap\' '
'mode can be cross-compiled.')
for s in cross_sources:
build_command.cache_or_build_source(s, build_env)
for s in native_sources:
build_command.fetch_sources(s)
# Install those to the output tarball ...
self.app.status(msg='Building final bootstrap system image')
system_artifact.source.cross_sources = cross_sources
system_artifact.source.native_sources = native_sources
staging_area = build_command.create_staging_area(build_env,
use_chroot=False)
builder = BootstrapSystemBuilder(self.app, staging_area,
build_command.lac, build_command.rac,
system_artifact.source,
build_command.lrc, 1, False)
builder.build_and_cache()
self.app.status(
msg='Bootstrap tarball for %(name)s is cached at %(cachepath)s',
name=system_artifact.name,
cachepath=build_command.lac.artifact_filename(system_artifact))
def check_time(self):
# Check that the current time is not far in the past.
if time.localtime(time.time()).tm_year < 2012:
raise morphlib.Error(
'System time is far in the past, please set your system clock')
def dump(*args, **kwargs):
raise morphlib.Error('YAML not available')
def _validate_root_kind(root_artifact):
root_kind = root_artifact.source.morphology['kind']
if root_kind != 'system':
raise morphlib.Error(
'Building a %s directly is not supported' % root_kind)
