def remove_lib(ctx, self, lib=None, purge=False):
if lib is None or str(lib) == "":
logger.warn("No library to remove")
return False
elif isinstance(lib, six.string_types):
_, name, _, _ = break_component_naming_format(lib)
manifests = maniest_from_name(name)
if len(manifests) == 0:
logger.warn("Unknown library: %s" % lib)
return False
for manifest in manifests:
if manifest.type != ComponentType.LIB:
continue
self.config.libraries.remove(name, purge)
break
if purge and os.path.exists(manifest.localdir):
logger.info("Purging lib/%s..." % name)
shutil.rmtree(manifest.localdir)
self.save_yaml()
return True
def add_lib(ctx, self, lib=None):
if lib is None or str(lib) == "":
logger.warn("No library to add")
return False
elif isinstance(lib, six.string_types):
_, name, _, version = break_component_naming_format(lib)
manifests = maniest_from_name(name)
if len(manifests) == 0:
logger.warn("Unknown library: %s" % lib)
return False
for manifest in manifests:
if manifest.type != ComponentType.LIB:
continue
self.config.libraries.add(
Library(
name=name,
version=version,
manifest=manifest,
))
self.save_yaml()
return True
def from_workdir(ctx,
cls,
workdir=None,
force_init=False,
use_versions=[]):
if workdir is None:
workdir = ctx.obj.workdir
config = None
try:
config = load_config(find_config(workdir, None, ctx.obj.env))
except KraftFileNotFound:
pass
# Dynamically update the configuration specification based on version
# overrides provided by use_versions
for use in use_versions:
_type, name, _, version = break_component_naming_format(use)
if _type is ComponentType.CORE:
config.unikraft.version = version
for k, target in enumerate(config.targets.all()):
if _type is ComponentType.ARCH or _type is None:
if target.architecture.name == name:
_type = ComponentType.ARCH
target.architecture.version = version
config.targets.set(k, target)
break
if _type is ComponentType.PLAT or _type is None:
if target.platform.name == name:
_type = ComponentType.PLAT
target.platform.version = version
config.targets.set(k, target)
break
if _type is ComponentType.LIB or _type is None:
for k, lib in enumerate(config.libraries.all()):
if lib.name == name:
_type = ComponentType.LIB
lib.version = version
config.libraries.set(k, lib)
break
return cls(
config=config,
localdir=workdir,
ignore_version=force_init,
)
def maniest_from_name(ctx, name=None):
from kraft.types import break_component_naming_format
components = list()
if name is None:
return components
type, name, _, _ = break_component_naming_format(name)
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, component in manifest.items():
if (type is None or \
(type is not None \
and type == component.type)) \
and component.name == name:
components.append(component)
return components
def get_component_from_git_repo(ctx, origin=None):
if origin is None:
raise ValueError("expected origin")
# TODO: There should be a work around to fix this import loop cycle
from kraft.manifest import ManifestItem
from kraft.manifest import ManifestItemVersion
from kraft.manifest import ManifestItemDistribution
from .types import ListProviderType
# This is a best-effort guess at the type and name of the git repository
# using the path to determine if it's namespaced.
uri = urlparse(origin)
pathparts = uri.path.split('/')
if len(pathparts) >= 2:
potential_typename = '/'.join(pathparts[-2:])
_type, _name, _, _ = break_component_naming_format(potential_typename)
elif len(pathparts) == 1:
_type, _name, _, _ = break_component_naming_format(uri.path)
if _type is None:
raise ValueError("".join([
"Cannot determine the type of the repository: %s\n\n",
"Please ensure it is of the naming convention <type>-<name> or ",
"that it is namespaced in a directory <type>/<name>."
]) % origin)
localdir = None
if os.path.exists(origin):
localdir = origin
repo = GitRepo(origin)
item = ManifestItem(provider=ListProviderType.GIT,
name=_name,
type=_type.shortname,
dist=UNIKRAFT_RELEASE_STABLE,
git=origin,
manifest=origin,
localdir=localdir)
stable = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STABLE)
staging = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STAGING)
for version in repo.tags:
commit = repo.commit(version)
# interpret the tag name for symbolic distributions
ref = GIT_UNIKRAFT_TAG_PATTERN.match(str(version))
if ref is not None:
version = ref.group(1)
stable.add_version(
ManifestItemVersion(git_sha=str(commit),
version=version,
timestamp=datetime.fromtimestamp(
int(commit.committed_date))))
# Only create the stable distribution if versions exist within
if len(stable.versions) > 0:
item.add_distribution(stable)
for ref in repo.git.branch('-a').split('\n'):
# skip fast forwards
if "->" in ref or "detached" in ref:
continue
branch = ref.strip().replace("*", "")
branch = branch.strip().replace("remotes/", "")
branch = branch.strip().replace("origin/", "")
if branch in UNIKRAFT_RELEASE_STABLE_VARIATIONS:
continue # we've done this one seperately
# Add this commit to the staging branch (this usually happens when
# commits have been applied on top of a HEAD)
if ref.strip() == "":
dist = staging
# Add the branch as a distribution
else:
dist = ManifestItemDistribution(name=branch, )
if branch in item.dists.keys():
dist = item.dists[branch]
# Add the latest commit to that branch as the only version
commit = repo.commit(branch)
dist.add_version(
ManifestItemVersion(git_sha=str(commit),
version=str(commit)[:7],
timestamp=datetime.fromtimestamp(
int(commit.committed_date))))
item.add_distribution(dist)
return item
def get_component_from_github(ctx, origin=None, org=None, repo=None):
if origin is None:
raise ValueError("expected origin")
elif org is None:
raise ValueError("expected org")
elif repo is None:
raise ValueError("expected repo")
# TODO: There should be a work around to fix this import loop cycle
from kraft.manifest import ManifestItem
from kraft.manifest import ManifestItemVersion
from kraft.manifest import ManifestItemDistribution
from .types import ListProviderType
if isinstance(repo, str):
if ".git" in repo:
repo = repo.split(".")[0]
github_api = Github(ctx.obj.env.get('UK_KRAFT_GITHUB_TOKEN', None))
repo = github_api.get_repo("%s/%s" % (org, repo))
if repo is None or not isinstance(repo, Repository):
raise TypeError("repo expected Repository")
# Ensure repository matches expression
if "*" in origin:
uri = urlparse(origin)
github_org = uri.path.split('/')[1]
github_repo = uri.path.split('/')[2]
if "*" in github_org:
raise ValueError(
"cannot use wildcard in GitHub organisation names")
regex = fnmatch.translate(github_repo)
reobj = re.compile(regex)
match = reobj.match(repo.name)
if match is None:
return
_type, _name, _, _ = break_component_naming_format(repo.name)
item = ManifestItem(
provider=ListProviderType.GITHUB,
name=_name,
description=repo.description,
type=_type.shortname,
dist=UNIKRAFT_RELEASE_STABLE,
git=repo.git_url,
manifest=origin,
)
for branch in repo.get_branches():
if branch.name == UNIKRAFT_RELEASE_STABLE:
dist = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STABLE)
tags = repo.get_tags()
releases = repo.get_releases()
did_add_version = False
if tags.totalCount > 0:
for tag in tags:
_version = tag.name
# interpret the tag name for symbolic distributions
ref = GIT_UNIKRAFT_TAG_PATTERN.match(tag.name)
if ref is not None:
_version = ref.group(1)
did_add_version = True
dist.add_version(
ManifestItemVersion(
git_sha=tag.name,
version=_version,
timestamp=repo.pushed_at,
tarball=GITHUB_TARBALL %
(repo.owner.login, repo.name, tag.name),
))
if releases.totalCount > 0:
for release in releases:
# Skip draft releases
if release.draft:
continue
_version = release.tag_name
# interpret the tag name for symbolic distributions
ref = GIT_UNIKRAFT_TAG_PATTERN.match(release.tag_name)
if ref is not None:
_version = ref.group(1)
did_add_version = True
dist.add_version(
ManifestItemVersion(
git_sha=release.tag_name,
version=_version,
timestamp=release.published_at,
tarball=GITHUB_TARBALL %
(repo.owner.login, repo.name, release.tag_name),
))
if did_add_version is False:
dist.add_version(
ManifestItemVersion(
git_sha=branch.commit.sha,
version=branch.commit.sha[:7],
timestamp=repo.pushed_at,
tarball=GITHUB_TARBALL %
(repo.owner.login, repo.name, branch.commit.sha),
))
else:
dist = ManifestItemDistribution(name=branch.name, )
dist.add_version(
ManifestItemVersion(
git_sha=branch.commit.sha,
version=branch.commit.sha[:7],
timestamp=repo.pushed_at,
tarball=GITHUB_TARBALL %
(repo.owner.login, repo.name, branch.commit.sha),
))
item.add_distribution(dist)
return item
def load_config(config_details, use_versions=[]): # noqa: C901
"""Load the configuration from a working directory and a list of
configuration files. Files are loaded in order, and merged on top
of each other to create the final configuration.
Return a fully interpolated, extended and validated configuration.
"""
processed_files = [
process_kraftfile(config_file, config_details.environment)
for config_file in config_details.config_files
]
config_details = config_details._replace(config_files=processed_files)
main_file = config_details.config_files[0]
if main_file.config is None:
raise CannotReadKraftfile(main_file.filename)
name = load_mapping(config_details.config_files, 'get_name', 'name',
config_details.working_dir)
if name is None or len(name) == 0:
name = get_project_name(config_details.working_dir, None,
config_details.environment)
unikraft = load_mapping(config_details.config_files, 'get_unikraft',
'unikraft', config_details.working_dir)
arguments = load_mapping(config_details.config_files, 'get_arguments',
'arguments', config_details.working_dir)
before = load_mapping(config_details.config_files, 'get_before', 'before',
config_details.working_dir)
after = load_mapping(config_details.config_files, 'get_after', 'after',
config_details.working_dir)
targets = load_mapping(config_details.config_files, 'get_targets',
'targets', config_details.working_dir)
libraries = load_mapping(config_details.config_files, 'get_libraries',
'libraries', config_details.working_dir)
volumes = load_mapping(config_details.config_files, 'get_volumes',
'volumes', config_details.working_dir)
networks = load_mapping(config_details.config_files, 'get_networks',
'networks', config_details.working_dir)
# Dynamically update the configuration specification based on version
# overrides provided by use_versions
for use in use_versions:
_type, name, _, version = break_component_naming_format(use)
if _type is ComponentType.CORE:
unikraft['version'] = version
for i, conf in enumerate(targets):
if _type is ComponentType.ARCH or _type is None:
if 'architecture' in conf and \
conf['architecture']['name'] == name:
targets[i]['architecture']['version'] = version
if _type is ComponentType.PLAT or _type is None:
if 'platform' in conf and \
conf['platform']['name'] == name:
targets[i]['platform']['version'] = version
if _type is ComponentType.LIB or _type is None:
for lib, conf in libraries.items():
if lib == name:
libraries[lib]['version'] = version
core = Unikraft(**unikraft)
return Config(
specification=main_file.version,
name=name,
arguments=arguments,
before=before,
after=after,
unikraft=core,
targets=TargetManager(targets, core),
libraries=LibraryManager(libraries),
volumes=VolumeManager(volumes),
networks=NetworkManager(networks),
)
def kraft_app_init(ctx,
appdir=None,
name=None,
plat=None,
arch=None,
template_app=None,
force_init=False,
pull_dependencies=False,
dumps_local=False,
create_makefile=False):
"""
"""
if appdir is None:
raise ValueError("Cannot initialize application at unset directory")
# If we are using a template application, we can simply copy from the source
# repository
if template_app is not None:
app_manifest = None
_, template_name, _, version = break_component_naming_format(
template_app)
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, item in manifest.items():
if item.name == template_name and item.type == ComponentType.APP:
app_manifest = item
if app_manifest is None:
raise UnknownApplicationTemplateName(template_app)
if version is not None:
version = app_manifest.get_version(version)
if version is None:
raise UnknownVersionError(version, app_manifest)
else:
version = app_manifest.get_version(UNIKRAFT_RELEASE_STABLE)
kraft_download_component(localdir=appdir,
manifest=app_manifest,
version=version.version)
if pull_dependencies or dumps_local:
workdir = None
if dumps_local:
workdir = os.path.join(appdir, UNIKRAFT_WORKDIR)
kraft_list_pull(
name=str(app_manifest),
appdir=appdir,
workdir=workdir,
pull_dependencies=True,
skip_app=True,
force_pull=force_init,
)
app = Application.from_workdir(appdir)
# If no application is provided, we can initialize a template by dumping
# a YAML file
else:
unikraft = ctx.obj.cache.find_item_by_name(type="core",
name="unikraft")
unikraft.download()
app = Application(name=name,
unikraft=unikraft,
architectures=[arch],
platforms=[plat],
localdir=appdir,
ignore_version=force_init)
app.name = name
app.init(create_makefile=create_makefile)
logger.info('Initialized new unikraft application: %s' % appdir)
def kraft_list_pull(ctx, name=None, workdir=None, use_git=False,
pull_dependencies=False, skip_verify=False, appdir=None,
skip_app=False, force_pull=False):
"""
Pull a particular component from a known manifest. This will retrieve
the contents to either the automatically determined directory or to an
alternative working directory.
Args:
name (str): The name of the component(s) to pull. This can be the full
qualifier, e.g.: lib/python3==0.4, partial, or the minimum: python3.
workdir (str): The path to save the component(s).
use_git (bool): Whether to use git to retrieve the components.
pull_dependencies (bool): If an application is specified in name, this
will signal to pull the listed libraries for this.
appdir (str): Used in conjunction with pull_dependencies and used to
specify the application from which the dependencies are determined
and then pulled.
"""
manifests = list()
names = list()
if isinstance(name, tuple):
names = list(name)
elif name is not None:
names.append(name)
not_found = list()
if isinstance(name, tuple):
not_found = list(name)
elif name is not None:
not_found.append(name)
# Pull the dependencies for the application at workdir or cwd
if (pull_dependencies and
(len(names) == 0 or (appdir is not None and len(names) == 1))):
app = Application.from_workdir(
appdir if appdir is not None
else workdir if workdir is not None
else os.getcwd(),
force_pull
)
for component in app.components:
if component.manifest is not None:
manifests.append((
component.manifest,
ManifestVersionEquality.EQ,
component.version.version
))
# Pull the provided named components
else:
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, manifest in manifest.items():
if len(names) == 0:
manifests.append((manifest, 0, None))
else:
for fullname in names:
type, name, eq, version = \
break_component_naming_format(fullname)
if (type is None or
(type is not None
and type == manifest.type)) \
and manifest.name == name:
manifests.append((manifest, eq, version))
# Accomodate for multi-type names
if fullname in not_found:
not_found.remove(fullname)
for name in not_found:
logger.warn("Could not find manifest: %s" % name)
if len(manifests) == 0:
logger.error("No manifests to download")
sys.exit(1)
for manifest in manifests:
if skip_app and manifest[0].type == ComponentType.APP:
continue
kraft_download_via_manifest(
workdir=workdir,
manifest=manifest[0],
equality=manifest[1],
version=manifest[2],
use_git=use_git,
skip_verify=skip_verify
)
if pull_dependencies and len(names) > 0:
for manifest in manifests:
if manifest[0].type == ComponentType.APP:
kraft_list_pull(
appdir=manifest[0].localdir,
workdir=workdir,
use_git=use_git,
pull_dependencies=True,
skip_verify=skip_verify
)
def cmd_list_show(ctx, return_json=False, name=None):
"""
Show the details of a component in a remote repository.
"""
kraft_list_preflight()
components = list()
type, name, _, _ = break_component_naming_format(name)
for manifest_origin in ctx.obj.cache.all():
manifest = ctx.obj.cache.get(manifest_origin)
for _, component in manifest.items():
if (type is None or
(type is not None
and type == component.type)) \
and component.name == name:
components.append(component)
if len(components) == 0:
logger.error("Unknown component name: %s" % name)
sys.exit(1)
if return_json:
data_json = []
for _, component in enumerate(components):
data_json.append(component.__getstate__())
click.echo(json.dumps(data_json))
else:
for i, component in enumerate(components):
# print seperator
if len(components) > 1 and i > 0 and not return_json:
click.echo("---")
table = list()
table.append(['name', component.name])
table.append(['type', component.type.shortname])
description = ""
if component.description is not None:
description = component.description
desc = textwrap.wrap(description, LIST_DESC_WIDTH)
for i, line in enumerate(desc):
table.append(['description' if i == 0 else '', line])
for i, dist in enumerate(component.dists):
dist = component.dists[dist]
table.append([('distributions' if len(component.dists) > 1 else
'distribution') if i == 0 else '',
'%s@%s' % (dist.name, dist.latest.version)])
if component.git is not None:
table.append(['git', component.git])
if component.manifest is not None:
table.append(['manifest', component.manifest])
table.append(['last checked', prettydate(component.last_checked)])
localdir = component.localdir
if os.path.isdir(localdir) and len(os.listdir(localdir)) != 0:
table.append(['located at', localdir])
for i, data in enumerate(table):
table[i] = [
click.style(data[0] + ':' if len(data[0]) > 0 else '',
fg="white"), data[1]
]
# print and remove last new line
click.echo(pretty_columns(table)[:-1])
