def kraft_build(ctx, fast):
logger.debug("Building %s..." % ctx.workdir)
try:
project = Project.from_config(
ctx.workdir,
config.load(
config.find(ctx.workdir, None, ctx.env)
)
)
except KraftError as e:
logger.error(str(e))
sys.exit(1)
if not project.is_configured():
if click.confirm('It appears you have not configured your application. Would you like to do this now?', default=True):
project.configure()
n_proc = None
if fast:
# This simply set the `-j` flag which signals to make to use all cores.
n_proc = ""
project.build(n_proc=n_proc)
def download(cls,
manifest=None,
localdir=None,
version=None,
override_existing=False,
**kwargs):
try:
repo = GitRepo(localdir)
except (InvalidGitRepositoryError, NoSuchPathError):
repo = GitRepo.init(localdir)
if manifest.git is not None:
try:
repo.create_remote('origin', manifest.git)
except GitCommandError as e:
pass
try:
if sys.stdout.isatty():
repo.remotes.origin.fetch(progress=GitProgressBar(
label="%s@%s" % (str(manifest), version.version)))
else:
for fetch_info in repo.remotes.origin.fetch():
logger.debug(
"Updated %s %s to %s" %
(manifest.git, fetch_info.ref, fetch_info.commit))
# self.last_checked = datetime.now()
except (GitCommandError, AttributeError) as e:
logger.error("Could not fetch %s: %s" % (manifest.git, str(e)))
if version.git_sha is not None:
repo.git.checkout(version.git_sha)
def checkout(ctx, self, version=None, retry=False):
"""Checkout a version of the repository."""
if ctx.dont_checkout:
return
if version is None:
version = self.version
if self._type == RepositoryType.ARCH:
return
elif self._type == RepositoryType.PLAT and self._source == UNIKRAFT_CORE:
return
elif version is not None:
try:
repo = GitRepo(self._localdir)
except (NoSuchPathError, InvalidGitRepositoryError):
logger.debug("Attempting to checkout %s before update!" % self)
# Allow one retry
if retry is False:
self.update()
self.checkout(version, True)
return
try:
# If this throws an exception, it means we have never checked
# out the repository before.
commit_hash = str(repo.head.commit)
# Determine if the repository has already been checked out at
# this version
if commit_hash.startswith(version) \
or version in self._known_versions.keys() and self._known_versions[version] == commit_hash:
logger.debug("%s already at %s" % (self._name, version))
return
except ValueError as e:
pass
logger.debug("Checking-out %s@%s..." % (self._name, version))
# First simply attempting what was specified
try:
repo.git.checkout(version)
except GitCommandError as e1:
# Don't try well-known branches with well-known RELEASE prefix:
if version != BRANCH_MASTER and version != BRANCH_STAGING:
try:
repo.git.checkout('RELEASE-%s' % version)
except GitCommandError as e2:
if not ctx.ignore_checkout_errors:
logger.error("Could not checkout %s@%s: %s" %
(self._name, version, str(e2)))
sys.exit(1)
elif not ctx.ignore_checkout_errors:
logger.error("Could not checkout %s@%s: %s" %
(self._name, version, str(e1)))
sys.exit(1)
def kraft_download_via_manifest(ctx, workdir=None, manifest=None,
equality=None, version=None, use_git=False,
skip_verify=False):
"""
"""
threads = list()
def kraft_download_component_thread(localdir=None, manifest=None,
equality=ManifestVersionEquality.EQ,
version=None, use_git=False,
skip_verify=False,
override_existing=False):
with ctx:
kraft_download_component(
localdir=localdir,
manifest=manifest,
equality=equality,
version=version,
use_git=use_git,
skip_verify=skip_verify,
override_existing=override_existing
)
if workdir is None:
localdir = manifest.localdir
elif manifest.type == ComponentType.CORE:
localdir = os.path.join(workdir, manifest.type.workdir)
else:
localdir = os.path.join(workdir, manifest.type.workdir, manifest.name)
thread = ErrorPropagatingThread(
target=kraft_download_component_thread,
kwargs={
'localdir': localdir,
'manifest': manifest,
'equality': equality,
'version': version,
'use_git': use_git,
'skip_verify': skip_verify
}
)
threads.append((manifest, thread))
thread.start()
for manifest, thread in threads:
try:
thread.join()
except Exception as e:
logger.error("Error pulling manifest: %s " % e)
if ctx.obj.verbose:
import traceback
logger.error(traceback.format_exc())
if sys.stdout.isatty():
flush()
def clean(ctx, proper):
"""
Cleans build files.
"""
logger.debug("Cleaning %s..." % ctx.workdir)
try:
project = Project.from_config(
ctx.workdir, config.load(config.find(ctx.workdir, None, ctx.env)))
except KraftError as e:
logger.error(str(e))
sys.exit(1)
project.clean(proper=proper)
def kraft_update(ctx):
origins = ctx.obj.settings.get(KRAFTRC_LIST_ORIGINS)
if origins is None or len(origins) == 0:
logger.error(
"No source origins available. Please see: kraft list add --help")
sys.exit(1)
try:
for origin in origins:
manifest = ctx.obj.cache.get(origin)
if manifest is None:
manifest = Manifest(manifest=origin)
threads, items = kraft_update_from_source_threads(origin)
for thread in threads:
thread.join()
# Check thread's return value
while not items.empty():
result = items.get()
if result is not None:
manifest.add_item(result)
logger.info("Found %s/%s via %s..." %
(click.style(result.type.shortname, fg="blue"),
click.style(result.name,
fg="blue"), manifest.manifest))
ctx.obj.cache.save(origin, manifest)
except RateLimitExceededException:
logger.error("".join([
"GitHub rate limit exceeded. You can tell kraft to use a ",
"personal access token by setting the UK_KRAFT_GITHUB_TOKEN ",
"environmental variable."
]))
except Exception as e:
logger.critical(str(e))
if ctx.obj.verbose:
import traceback
logger.critical(traceback.format_exc())
sys.exit(1)
def update(self):
"""Update this particular repository."""
repo = None
try:
repo = GitRepo(self._localdir)
# Not a repository? No problem, let's clone it:
except (InvalidGitRepositoryError, NoSuchPathError) as e:
repo = GitRepo.init(self._localdir)
repo.create_remote('origin', self._source)
try:
for fetch_info in repo.remotes.origin.fetch():
logger.debug("Updated %s %s to %s" %
(self._source, fetch_info.ref, fetch_info.commit))
self._last_checked = datetime.now()
except (GitCommandError, AttributeError) as e:
logger.error("Could not fetch %s: %s" % (self._source, str(e)))
def execute(cmd="", env={}, dry_run=False):
if type(cmd) is list:
cmd = " ".join(cmd)
logger.debug("Running: %s" % cmd)
if not dry_run:
popen = subprocess.Popen(
cmd,
shell=True,
stdout=subprocess.PIPE,
env=merge_dicts(os.environ, env)
)
for line in popen.stdout:
logger.info(line.strip().decode('ascii'))
popen.stdout.close()
return_code = popen.wait()
if return_code is not None and int(return_code) > 0:
logger.error("Command '%s' returned %d" % (cmd, return_code))
sys.exit(return_code)
def kraft_run(ctx, plat, arch, initrd, background, paused, gdb, dbg,
virtio_nic, bridge, interface, dry_run, args, memory,
cpu_sockets, cpu_cores):
"""
Starts the unikraft application once it has been successfully
built.
"""
try:
project = Project.from_config(
ctx.workdir, config.load(config.find(ctx.workdir, None, ctx.env)))
except KraftError as e:
logger.error(str(e))
sys.exit(1)
target_platform = None
for platform in project.platforms.all():
if plat == platform.name:
target_platform = platform
if target_platform is None:
logger.error('Application platform not configured or set')
sys.exit(1)
target_architecture = None
for architecture in project.architectures.all():
if arch == architecture.name:
target_architecture = architecture
if target_architecture is None:
logger.error('Application architecture not configured or set')
sys.exit(1)
unikernel = UNIKERNEL_IMAGE_FORMAT % (ctx.workdir, project.name,
target_platform.name,
target_architecture.name)
if not os.path.exists(unikernel):
logger.error('Could not find unikernel: %s' % unikernel)
logger.info('Have you tried running `kraft build`?')
sys.exit(1)
executor = target_platform.repository.executor
executor.architecture = target_architecture.name
executor.use_debug = dbg
if initrd:
executor.add_initrd(initrd)
if virtio_nic:
executor.add_virtio_nic(virtio_nic)
if bridge:
executor.add_bridge(bridge)
if interface:
executor.add_interface(interface)
if gdb:
executor.open_gdb(gdb)
if memory:
executor.set_memory(memory)
if cpu_sockets:
executor.set_cpu_sockets(cpu_sockets)
if cpu_cores:
executor.set_cpu_cores(cpu_cores)
try:
executor.unikernel = unikernel
executor.execute(
extra_args=args,
background=background,
paused=paused,
dry_run=dry_run,
)
except ExecutorError as e:
logger.error("Cannot execute: %s" % e)
sys.exit(1)
def cmd_lib_bump(ctx,
lib=None,
version=None,
bump_all=False,
force_version=False,
fast_forward=False,
build=False):
"""
Update an existing Unikraft library's source origin version (experimental).
If --fast-forward is specified, the library will be upgraded to the latest
determined version. Otherwise, and by default, the bump will increment by
the smallest possible version.
"""
kraft_list_preflight()
try:
if bump_all:
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, item in manifest.items():
if item.type != ComponentType.LIB.shortname:
continue
if not (ctx.obj.assume_yes
or click.confirm("Bump %s?" % item.name)):
continue
kraft_download_via_manifest(manifest=item, use_git=True)
kraft_lib_bump(workdir=item.localdir,
force_version=force_version,
fast_forward=fast_forward,
build=build)
elif lib is not None and os.path.isdir(lib):
kraft_lib_bump(
workdir=lib,
version=version,
force_version=force_version,
fast_forward=fast_forward,
)
elif lib is not None:
manifests = maniest_from_name(lib)
for manifest in manifests:
if manifest.type != ComponentType.LIB.shortname:
continue
if len(manifests) > 0 and version is not None:
logger.warn("Ignoring --version flag")
for manifest in manifests:
kraft_download_via_manifest(manifest=manifest, use_git=True)
kraft_lib_bump(
workdir=manifest.localdir,
version=version if len(manifests) == 1 else None,
force_version=force_version,
fast_forward=fast_forward,
build=build)
else:
kraft_lib_bump(workdir=os.getcwd(),
version=version,
force_version=force_version,
fast_forward=fast_forward,
build=build)
except Exception as e:
logger.error(e)
if ctx.obj.verbose:
import traceback
logger.error(traceback.format_exc())
sys.exit(1)
def kraft_configure(ctx, target_plat, target_arch, force_configure, menuconfig):
"""
Populates the local .config with the default values for the target application.
"""
logger.debug("Configuring %s..." % ctx.workdir)
try:
project = Project.from_config(
ctx.workdir,
config.load(
config.find(ctx.workdir, None, ctx.env)
)
)
except KraftError as e:
logger.error(str(e))
sys.exit(1)
if project.is_configured() and force_configure is False and menuconfig is False:
if click.confirm('%s is already configured, would you like to overwrite configuration?' % ctx.workdir):
# It should be safe to set this now
force_configure = True
else:
logger.error('Cancelling!')
sys.exit(1)
# Check if we have used "--arch" before. This saves the user from having to
# re-type it. This means omission uses the settings.
if target_arch is None and len(project.architectures.all()) > 1 and ctx.settings.get(KRAFTCONF_PREFERRED_ARCHITECTURE):
target_arch = ctx.settings.get(KRAFTCONF_PREFERRED_ARCHITECTURE)
elif target_arch is None and len(project.architectures.all()) == 1:
for arch in project.architectures.all():
target_arch = arch.name
if target_arch is not None and ctx.settings.get(KRAFTCONF_PREFERRED_ARCHITECTURE) is None:
ctx.settings.set(KRAFTCONF_PREFERRED_ARCHITECTURE, target_arch)
# Check if we have used "--plat" before. This saves the user from having to
# re-type it. This means omission uses the settings.
if target_plat is None and len(project.platforms.all()) > 1 and ctx.settings.get(KRAFTCONF_PREFERRED_PLATFORM):
target_plat = ctx.settings.get(KRAFTCONF_PREFERRED_PLATFORM)
elif target_plat is None and len(project.platforms.all()) == 1:
for plat in project.platforms.all():
target_plat = plat.name
if target_plat is not None and ctx.settings.get(KRAFTCONF_PREFERRED_PLATFORM) is None:
ctx.settings.set(KRAFTCONF_PREFERRED_PLATFORM, target_plat)
if menuconfig:
project.menuconfig()
else:
try:
project.configure(
target_arch=target_arch,
target_plat=target_plat
)
except KraftError as e:
logger.error(str(e))
sys.exit(1)
def kraft_init(ctx, name, target_plat, target_arch, template_app, version,
force_create):
if name is None:
name = os.path.basename(os.getcwd())
# Pre-flight check determines if we are trying to work with nothing
if ctx.cache.is_stale() and click.confirm(
'kraft caches are out-of-date. Would you like to update?'):
update()
# Check if the directory is non-empty and prompt for validation
if utils.is_dir_empty(ctx.workdir) is False and force_create is False:
if click.confirm(
'%s is a non-empty directory, would you like to continue?' %
ctx.workdir):
# It should be safe to set this now
force_create = True
else:
logger.error('Cancelling!')
sys.exit(1)
# If we are using a template application, we can simply copy from the source
# repository
if template_app is not None:
apps = {}
for repo in ctx.cache.all():
repo = ctx.cache.get(repo)
if repo.type is RepositoryType.APP:
apps[repo.name] = repo
if template_app not in apps.keys():
logger.error('Template application not found: %s' % template_app)
logger.error('Supported templates: %s' % ', '.join(apps.keys()))
sys.exit(1)
app = apps[template_app]
if version and version not in app.known_versions.keys():
logger.error('Unknown version \'%s\' for app: %s' %
(version, template_app))
sys.exit(1)
app.checkout(version)
utils.recursively_copy(
app.localdir,
ctx.workdir,
overwrite=force_create,
ignore=['.git', 'build', '.config', '.config.old', '.config.orig'])
logger.info('Initialized new unikraft application \'%s\' in %s' %
(name, ctx.workdir))
# If no application is provided, we can initialize a template by dumping
# a YAML file
else:
# Determine the version of unikraft that we should be using
if version is None:
# This simply sets the "source" to the unikraft core repository which,
# once parsed through the internal cache, should pop out the latest
# version.
unikraft_source = UNIKRAFT_CORE
unikraft_version = None
else:
unikraft_source, unikraft_version = interpolate_source_version(
source=version, repository_type=RepositoryType.CORE)
try:
core = Core.from_source_string(source=unikraft_source,
version=unikraft_version)
preferred_arch = ctx.settings.get(KRAFTCONF_PREFERRED_ARCHITECTURE)
if target_arch is None:
if preferred_arch:
target_arch = preferred_arch
else:
logger.error("Please provide an architecture.")
sys.exit(1)
arch_source, arch_version = interpolate_source_version(
source=target_arch, repository_type=RepositoryType.ARCH)
archs = Architectures([])
archs.add(
target_arch,
Architecture.from_source_string(
name=target_arch,
source=arch_source,
), {})
preferred_plat = ctx.settings.get(KRAFTCONF_PREFERRED_PLATFORM)
if target_plat is None:
if preferred_plat:
target_plat = preferred_plat
else:
logger.error("Please provide a platform.")
sys.exit(1)
plat_source, plat_version = interpolate_source_version(
source=target_plat, repository_type=RepositoryType.PLAT)
plats = Platforms([])
plats.add(
target_plat,
Platform.from_source_string(
name=target_plat,
source=plat_source,
), {})
logger.info("Using %s..." % core)
project = Project(
path=ctx.workdir,
name=name,
core=core,
architectures=archs,
platforms=plats,
)
project.init(force_create=force_create)
logger.info('Initialized new unikraft application \'%s\' in %s' %
(name, ctx.workdir))
except KraftError as e:
logger.error(str(e))
sys.exit(1)
def cmd_list(ctx, show_installed=False, show_core=False, show_plats=False,
show_libs=False, show_apps=False, show_local=False, paginate=False,
this=False, this_set=None, return_json=False):
"""
Retrieves lists of available architectures, platforms, libraries and
applications supported by unikraft. Use this command if you wish to
determine (and then later select) the possible targets for your unikraft
application.
By default, this subcommand will list all possible targets.
"""
if ctx.invoked_subcommand is None:
kraft_list_preflight()
show_archs = False
# If no flags are set, show everything
if (show_core is False
and show_archs is False
and show_plats is False
and show_libs is False
and show_apps is False):
show_core = show_archs = show_plats = show_libs = show_apps = True
# Populate a matrix with all relevant columns and rows for each
# component.
components = {}
data = []
data_json = {}
if this or this_set is not None:
workdir = os.getcwd()
if this_set is not None:
workdir = this_set
try:
app = Application.from_workdir(workdir)
for manifest in app.manifests:
if manifest.type.shortname not in components:
components[manifest.type.shortname] = []
components[manifest.type.shortname].append(manifest)
except KraftError as e:
logger.error(str(e))
sys.exit(1)
else:
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, item in manifest.items():
if item.type.shortname not in components:
components[item.type.shortname] = []
components[item.type.shortname].append(item)
for type, member in ComponentType.__members__.items():
columns = [
click.style(member.plural.upper(), fg='white'),
click.style('VERSION ', fg='white'),
click.style('RELEASED', fg='white'),
click.style('LAST CHECKED', fg='white')
]
if show_local:
columns.append(click.style('LOCATION', fg='white'))
rows = []
components_showing = 0
if member.shortname in components and (
(show_core and member is ComponentType.CORE) or
(show_archs and member is ComponentType.ARCH) or
(show_plats and member is ComponentType.PLAT) or
(show_libs and member is ComponentType.LIB) or
(show_apps and member is ComponentType.APP)):
rows = components[member.shortname]
# if len(rows) > 0:
data.append(columns)
for row in rows:
installed = False
install_error = False
localdir = row.localdir
if os.path.isdir(localdir):
installed = True
if len(os.listdir(localdir)) == 0:
install_error = True
logger.warn("%s directory is empty: %s " % (
row.name, localdir
))
latest_release = None
if UNIKRAFT_RELEASE_STABLE in row.dists.keys():
latest_release = row.dists[UNIKRAFT_RELEASE_STABLE].latest
elif UNIKRAFT_RELEASE_STAGING in row.dists.keys():
latest_release = row.dists[UNIKRAFT_RELEASE_STAGING].latest
if return_json:
if member.plural not in data_json:
data_json[member.plural] = []
row_json = row.__getstate__()
if not show_installed or (installed and show_installed):
data_json[member.plural].append(row_json)
components_showing += 1
else:
line = [
click.style(row.name, fg='yellow' if install_error else 'green' if installed else 'red'), # noqa: E501
click.style(latest_release.version if latest_release is not None else "", fg='white'), # noqa: E501
click.style(prettydate(latest_release.timestamp) if latest_release is not None else "", fg='white'), # noqa: E501
click.style(prettydate(row.last_checked), fg='white'),
]
if show_local:
line.append(click.style(localdir if installed else '', fg='white')) # noqa: E501
if not show_installed or (installed and show_installed):
data.append(line)
components_showing += 1
# Delete component headers with no rows
if components_showing == 0 and len(data) > 0:
del data[-1]
# Line break
elif len(rows) > 0:
data.append([click.style("", fg='white')] * len(columns))
if return_json:
click.echo(json.dumps(data_json))
else:
output = pretty_columns(data)
if len(data) == 0:
logger.info("Nothing to show")
elif paginate:
click.echo_via_pager(output)
else:
click.echo(output[:-1])
def kraft_list_pull(ctx, name=None, workdir=None, use_git=False,
pull_dependencies=False, skip_verify=False, appdir=None,
skip_app=False, force_pull=False):
"""
Pull a particular component from a known manifest. This will retrieve
the contents to either the automatically determined directory or to an
alternative working directory.
Args:
name (str): The name of the component(s) to pull. This can be the full
qualifier, e.g.: lib/python3==0.4, partial, or the minimum: python3.
workdir (str): The path to save the component(s).
use_git (bool): Whether to use git to retrieve the components.
pull_dependencies (bool): If an application is specified in name, this
will signal to pull the listed libraries for this.
appdir (str): Used in conjunction with pull_dependencies and used to
specify the application from which the dependencies are determined
and then pulled.
"""
manifests = list()
names = list()
if isinstance(name, tuple):
names = list(name)
elif name is not None:
names.append(name)
not_found = list()
if isinstance(name, tuple):
not_found = list(name)
elif name is not None:
not_found.append(name)
# Pull the dependencies for the application at workdir or cwd
if (pull_dependencies and
(len(names) == 0 or (appdir is not None and len(names) == 1))):
app = Application.from_workdir(
appdir if appdir is not None
else workdir if workdir is not None
else os.getcwd(),
force_pull
)
for component in app.components:
if component.manifest is not None:
manifests.append((
component.manifest,
ManifestVersionEquality.EQ,
component.version.version
))
# Pull the provided named components
else:
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, manifest in manifest.items():
if len(names) == 0:
manifests.append((manifest, 0, None))
else:
for fullname in names:
type, name, eq, version = \
break_component_naming_format(fullname)
if (type is None or
(type is not None
and type == manifest.type)) \
and manifest.name == name:
manifests.append((manifest, eq, version))
# Accomodate for multi-type names
if fullname in not_found:
not_found.remove(fullname)
for name in not_found:
logger.warn("Could not find manifest: %s" % name)
if len(manifests) == 0:
logger.error("No manifests to download")
sys.exit(1)
for manifest in manifests:
if skip_app and manifest[0].type == ComponentType.APP:
continue
kraft_download_via_manifest(
workdir=workdir,
manifest=manifest[0],
equality=manifest[1],
version=manifest[2],
use_git=use_git,
skip_verify=skip_verify
)
if pull_dependencies and len(names) > 0:
for manifest in manifests:
if manifest[0].type == ComponentType.APP:
kraft_list_pull(
appdir=manifest[0].localdir,
workdir=workdir,
use_git=use_git,
pull_dependencies=True,
skip_verify=skip_verify
)
def cmd_list_show(ctx, return_json=False, name=None):
"""
Show the details of a component in a remote repository.
"""
kraft_list_preflight()
components = list()
type, name, _, _ = break_component_naming_format(name)
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, component in manifest.items():
if (type is None or
(type is not None
and type == component.type)) \
and component.name == name:
components.append(component)
if len(components) == 0:
logger.error("Unknown component name: %s" % name)
sys.exit(1)
if return_json:
data_json = []
for _, component in enumerate(components):
data_json.append(component.__getstate__())
click.echo(json.dumps(data_json))
else:
for i, component in enumerate(components):
# print seperator
if len(components) > 1 and i > 0 and not return_json:
click.echo("---")
table = list()
table.append(['name', component.name])
table.append(['type', component.type.shortname])
description = ""
if component.description is not None:
description = component.description
desc = textwrap.wrap(description, LIST_DESC_WIDTH)
for i, line in enumerate(desc):
table.append(['description' if i == 0 else '', line])
for i, dist in enumerate(component.dists):
dist = component.dists[dist]
table.append([('distributions' if len(component.dists) > 1 else
'distribution') if i == 0 else '',
'%s@%s' % (dist.name, dist.latest.version)])
if component.git is not None:
table.append(['git', component.git])
if component.manifest is not None:
table.append(['manifest', component.manifest])
table.append(['last checked', prettydate(component.last_checked)])
localdir = component.localdir
if os.path.isdir(localdir) and len(os.listdir(localdir)) != 0:
table.append(['located at', localdir])
for i, data in enumerate(table):
table[i] = [
click.style(data[0] + ':' if len(data[0]) > 0 else '',
fg="white"), data[1]
]
# print and remove last new line
click.echo(pretty_columns(table)[:-1])