def remove_lib(ctx, self, lib=None, purge=False): if lib is None or str(lib) == "": logger.warn("No library to remove") return False elif isinstance(lib, six.string_types): _, name, _, _ = break_component_naming_format(lib) manifests = maniest_from_name(name) if len(manifests) == 0: logger.warn("Unknown library: %s" % lib) return False for manifest in manifests: if manifest.type != ComponentType.LIB: continue self.config.libraries.remove(name, purge) break if purge and os.path.exists(manifest.localdir): logger.info("Purging lib/%s..." % name) shutil.rmtree(manifest.localdir) self.save_yaml() return True
def add_lib(ctx, self, lib=None): if lib is None or str(lib) == "": logger.warn("No library to add") return False elif isinstance(lib, six.string_types): _, name, _, version = break_component_naming_format(lib) manifests = maniest_from_name(name) if len(manifests) == 0: logger.warn("Unknown library: %s" % lib) return False for manifest in manifests: if manifest.type != ComponentType.LIB: continue self.config.libraries.add( Library( name=name, version=version, manifest=manifest, )) self.save_yaml() return True
def from_workdir(ctx, cls, workdir=None, force_init=False, use_versions=[]): if workdir is None: workdir = ctx.obj.workdir config = None try: config = load_config(find_config(workdir, None, ctx.obj.env)) except KraftFileNotFound: pass # Dynamically update the configuration specification based on version # overrides provided by use_versions for use in use_versions: _type, name, _, version = break_component_naming_format(use) if _type is ComponentType.CORE: config.unikraft.version = version for k, target in enumerate(config.targets.all()): if _type is ComponentType.ARCH or _type is None: if target.architecture.name == name: _type = ComponentType.ARCH target.architecture.version = version config.targets.set(k, target) break if _type is ComponentType.PLAT or _type is None: if target.platform.name == name: _type = ComponentType.PLAT target.platform.version = version config.targets.set(k, target) break if _type is ComponentType.LIB or _type is None: for k, lib in enumerate(config.libraries.all()): if lib.name == name: _type = ComponentType.LIB lib.version = version config.libraries.set(k, lib) break return cls( config=config, localdir=workdir, ignore_version=force_init, )
def maniest_from_name(ctx, name=None): from kraft.types import break_component_naming_format components = list() if name is None: return components type, name, _, _ = break_component_naming_format(name) for manifest_origin in ctx.obj.cache.all(): manifest = ctx.obj.cache.get(manifest_origin) for _, component in manifest.items(): if (type is None or \ (type is not None \ and type == component.type)) \ and component.name == name: components.append(component) return components
def get_component_from_git_repo(ctx, origin=None): if origin is None: raise ValueError("expected origin") # TODO: There should be a work around to fix this import loop cycle from kraft.manifest import ManifestItem from kraft.manifest import ManifestItemVersion from kraft.manifest import ManifestItemDistribution from .types import ListProviderType # This is a best-effort guess at the type and name of the git repository # using the path to determine if it's namespaced. uri = urlparse(origin) pathparts = uri.path.split('/') if len(pathparts) >= 2: potential_typename = '/'.join(pathparts[-2:]) _type, _name, _, _ = break_component_naming_format(potential_typename) elif len(pathparts) == 1: _type, _name, _, _ = break_component_naming_format(uri.path) if _type is None: raise ValueError("".join([ "Cannot determine the type of the repository: %s\n\n", "Please ensure it is of the naming convention <type>-<name> or ", "that it is namespaced in a directory <type>/<name>." ]) % origin) localdir = None if os.path.exists(origin): localdir = origin repo = GitRepo(origin) item = ManifestItem(provider=ListProviderType.GIT, name=_name, type=_type.shortname, dist=UNIKRAFT_RELEASE_STABLE, git=origin, manifest=origin, localdir=localdir) stable = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STABLE) staging = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STAGING) for version in repo.tags: commit = repo.commit(version) # interpret the tag name for symbolic distributions ref = GIT_UNIKRAFT_TAG_PATTERN.match(str(version)) if ref is not None: version = ref.group(1) stable.add_version( ManifestItemVersion(git_sha=str(commit), version=version, timestamp=datetime.fromtimestamp( int(commit.committed_date)))) # Only create the stable distribution if versions exist within if len(stable.versions) > 0: item.add_distribution(stable) for ref in repo.git.branch('-a').split('\n'): # skip fast forwards if "->" in ref or "detached" in ref: continue branch = ref.strip().replace("*", "") branch = branch.strip().replace("remotes/", "") branch = branch.strip().replace("origin/", "") if branch in UNIKRAFT_RELEASE_STABLE_VARIATIONS: continue # we've done this one seperately # Add this commit to the staging branch (this usually happens when # commits have been applied on top of a HEAD) if ref.strip() == "": dist = staging # Add the branch as a distribution else: dist = ManifestItemDistribution(name=branch, ) if branch in item.dists.keys(): dist = item.dists[branch] # Add the latest commit to that branch as the only version commit = repo.commit(branch) dist.add_version( ManifestItemVersion(git_sha=str(commit), version=str(commit)[:7], timestamp=datetime.fromtimestamp( int(commit.committed_date)))) item.add_distribution(dist) return item
def get_component_from_github(ctx, origin=None, org=None, repo=None): if origin is None: raise ValueError("expected origin") elif org is None: raise ValueError("expected org") elif repo is None: raise ValueError("expected repo") # TODO: There should be a work around to fix this import loop cycle from kraft.manifest import ManifestItem from kraft.manifest import ManifestItemVersion from kraft.manifest import ManifestItemDistribution from .types import ListProviderType if isinstance(repo, str): if ".git" in repo: repo = repo.split(".")[0] github_api = Github(ctx.obj.env.get('UK_KRAFT_GITHUB_TOKEN', None)) repo = github_api.get_repo("%s/%s" % (org, repo)) if repo is None or not isinstance(repo, Repository): raise TypeError("repo expected Repository") # Ensure repository matches expression if "*" in origin: uri = urlparse(origin) github_org = uri.path.split('/')[1] github_repo = uri.path.split('/')[2] if "*" in github_org: raise ValueError( "cannot use wildcard in GitHub organisation names") regex = fnmatch.translate(github_repo) reobj = re.compile(regex) match = reobj.match(repo.name) if match is None: return _type, _name, _, _ = break_component_naming_format(repo.name) item = ManifestItem( provider=ListProviderType.GITHUB, name=_name, description=repo.description, type=_type.shortname, dist=UNIKRAFT_RELEASE_STABLE, git=repo.git_url, manifest=origin, ) for branch in repo.get_branches(): if branch.name == UNIKRAFT_RELEASE_STABLE: dist = ManifestItemDistribution(name=UNIKRAFT_RELEASE_STABLE) tags = repo.get_tags() releases = repo.get_releases() did_add_version = False if tags.totalCount > 0: for tag in tags: _version = tag.name # interpret the tag name for symbolic distributions ref = GIT_UNIKRAFT_TAG_PATTERN.match(tag.name) if ref is not None: _version = ref.group(1) did_add_version = True dist.add_version( ManifestItemVersion( git_sha=tag.name, version=_version, timestamp=repo.pushed_at, tarball=GITHUB_TARBALL % (repo.owner.login, repo.name, tag.name), )) if releases.totalCount > 0: for release in releases: # Skip draft releases if release.draft: continue _version = release.tag_name # interpret the tag name for symbolic distributions ref = GIT_UNIKRAFT_TAG_PATTERN.match(release.tag_name) if ref is not None: _version = ref.group(1) did_add_version = True dist.add_version( ManifestItemVersion( git_sha=release.tag_name, version=_version, timestamp=release.published_at, tarball=GITHUB_TARBALL % (repo.owner.login, repo.name, release.tag_name), )) if did_add_version is False: dist.add_version( ManifestItemVersion( git_sha=branch.commit.sha, version=branch.commit.sha[:7], timestamp=repo.pushed_at, tarball=GITHUB_TARBALL % (repo.owner.login, repo.name, branch.commit.sha), )) else: dist = ManifestItemDistribution(name=branch.name, ) dist.add_version( ManifestItemVersion( git_sha=branch.commit.sha, version=branch.commit.sha[:7], timestamp=repo.pushed_at, tarball=GITHUB_TARBALL % (repo.owner.login, repo.name, branch.commit.sha), )) item.add_distribution(dist) return item
def load_config(config_details, use_versions=[]): # noqa: C901 """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top of each other to create the final configuration. Return a fully interpolated, extended and validated configuration. """ processed_files = [ process_kraftfile(config_file, config_details.environment) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) main_file = config_details.config_files[0] if main_file.config is None: raise CannotReadKraftfile(main_file.filename) name = load_mapping(config_details.config_files, 'get_name', 'name', config_details.working_dir) if name is None or len(name) == 0: name = get_project_name(config_details.working_dir, None, config_details.environment) unikraft = load_mapping(config_details.config_files, 'get_unikraft', 'unikraft', config_details.working_dir) arguments = load_mapping(config_details.config_files, 'get_arguments', 'arguments', config_details.working_dir) before = load_mapping(config_details.config_files, 'get_before', 'before', config_details.working_dir) after = load_mapping(config_details.config_files, 'get_after', 'after', config_details.working_dir) targets = load_mapping(config_details.config_files, 'get_targets', 'targets', config_details.working_dir) libraries = load_mapping(config_details.config_files, 'get_libraries', 'libraries', config_details.working_dir) volumes = load_mapping(config_details.config_files, 'get_volumes', 'volumes', config_details.working_dir) networks = load_mapping(config_details.config_files, 'get_networks', 'networks', config_details.working_dir) # Dynamically update the configuration specification based on version # overrides provided by use_versions for use in use_versions: _type, name, _, version = break_component_naming_format(use) if _type is ComponentType.CORE: unikraft['version'] = version for i, conf in enumerate(targets): if _type is ComponentType.ARCH or _type is None: if 'architecture' in conf and \ conf['architecture']['name'] == name: targets[i]['architecture']['version'] = version if _type is ComponentType.PLAT or _type is None: if 'platform' in conf and \ conf['platform']['name'] == name: targets[i]['platform']['version'] = version if _type is ComponentType.LIB or _type is None: for lib, conf in libraries.items(): if lib == name: libraries[lib]['version'] = version core = Unikraft(**unikraft) return Config( specification=main_file.version, name=name, arguments=arguments, before=before, after=after, unikraft=core, targets=TargetManager(targets, core), libraries=LibraryManager(libraries), volumes=VolumeManager(volumes), networks=NetworkManager(networks), )
def kraft_app_init(ctx, appdir=None, name=None, plat=None, arch=None, template_app=None, force_init=False, pull_dependencies=False, dumps_local=False, create_makefile=False): """ """ if appdir is None: raise ValueError("Cannot initialize application at unset directory") # If we are using a template application, we can simply copy from the source # repository if template_app is not None: app_manifest = None _, template_name, _, version = break_component_naming_format( template_app) for manifest_origin in ctx.obj.cache.all(): manifest = ctx.obj.cache.get(manifest_origin) for _, item in manifest.items(): if item.name == template_name and item.type == ComponentType.APP: app_manifest = item if app_manifest is None: raise UnknownApplicationTemplateName(template_app) if version is not None: version = app_manifest.get_version(version) if version is None: raise UnknownVersionError(version, app_manifest) else: version = app_manifest.get_version(UNIKRAFT_RELEASE_STABLE) kraft_download_component(localdir=appdir, manifest=app_manifest, version=version.version) if pull_dependencies or dumps_local: workdir = None if dumps_local: workdir = os.path.join(appdir, UNIKRAFT_WORKDIR) kraft_list_pull( name=str(app_manifest), appdir=appdir, workdir=workdir, pull_dependencies=True, skip_app=True, force_pull=force_init, ) app = Application.from_workdir(appdir) # If no application is provided, we can initialize a template by dumping # a YAML file else: unikraft = ctx.obj.cache.find_item_by_name(type="core", name="unikraft") unikraft.download() app = Application(name=name, unikraft=unikraft, architectures=[arch], platforms=[plat], localdir=appdir, ignore_version=force_init) app.name = name app.init(create_makefile=create_makefile) logger.info('Initialized new unikraft application: %s' % appdir)
def kraft_list_pull(ctx, name=None, workdir=None, use_git=False, pull_dependencies=False, skip_verify=False, appdir=None, skip_app=False, force_pull=False): """ Pull a particular component from a known manifest. This will retrieve the contents to either the automatically determined directory or to an alternative working directory. Args: name (str): The name of the component(s) to pull. This can be the full qualifier, e.g.: lib/python3==0.4, partial, or the minimum: python3. workdir (str): The path to save the component(s). use_git (bool): Whether to use git to retrieve the components. pull_dependencies (bool): If an application is specified in name, this will signal to pull the listed libraries for this. appdir (str): Used in conjunction with pull_dependencies and used to specify the application from which the dependencies are determined and then pulled. """ manifests = list() names = list() if isinstance(name, tuple): names = list(name) elif name is not None: names.append(name) not_found = list() if isinstance(name, tuple): not_found = list(name) elif name is not None: not_found.append(name) # Pull the dependencies for the application at workdir or cwd if (pull_dependencies and (len(names) == 0 or (appdir is not None and len(names) == 1))): app = Application.from_workdir( appdir if appdir is not None else workdir if workdir is not None else os.getcwd(), force_pull ) for component in app.components: if component.manifest is not None: manifests.append(( component.manifest, ManifestVersionEquality.EQ, component.version.version )) # Pull the provided named components else: for manifest_origin in ctx.obj.cache.all(): manifest = ctx.obj.cache.get(manifest_origin) for _, manifest in manifest.items(): if len(names) == 0: manifests.append((manifest, 0, None)) else: for fullname in names: type, name, eq, version = \ break_component_naming_format(fullname) if (type is None or (type is not None and type == manifest.type)) \ and manifest.name == name: manifests.append((manifest, eq, version)) # Accomodate for multi-type names if fullname in not_found: not_found.remove(fullname) for name in not_found: logger.warn("Could not find manifest: %s" % name) if len(manifests) == 0: logger.error("No manifests to download") sys.exit(1) for manifest in manifests: if skip_app and manifest[0].type == ComponentType.APP: continue kraft_download_via_manifest( workdir=workdir, manifest=manifest[0], equality=manifest[1], version=manifest[2], use_git=use_git, skip_verify=skip_verify ) if pull_dependencies and len(names) > 0: for manifest in manifests: if manifest[0].type == ComponentType.APP: kraft_list_pull( appdir=manifest[0].localdir, workdir=workdir, use_git=use_git, pull_dependencies=True, skip_verify=skip_verify )
def cmd_list_show(ctx, return_json=False, name=None): """ Show the details of a component in a remote repository. """ kraft_list_preflight() components = list() type, name, _, _ = break_component_naming_format(name) for manifest_origin in ctx.obj.cache.all(): manifest = ctx.obj.cache.get(manifest_origin) for _, component in manifest.items(): if (type is None or (type is not None and type == component.type)) \ and component.name == name: components.append(component) if len(components) == 0: logger.error("Unknown component name: %s" % name) sys.exit(1) if return_json: data_json = [] for _, component in enumerate(components): data_json.append(component.__getstate__()) click.echo(json.dumps(data_json)) else: for i, component in enumerate(components): # print seperator if len(components) > 1 and i > 0 and not return_json: click.echo("---") table = list() table.append(['name', component.name]) table.append(['type', component.type.shortname]) description = "" if component.description is not None: description = component.description desc = textwrap.wrap(description, LIST_DESC_WIDTH) for i, line in enumerate(desc): table.append(['description' if i == 0 else '', line]) for i, dist in enumerate(component.dists): dist = component.dists[dist] table.append([('distributions' if len(component.dists) > 1 else 'distribution') if i == 0 else '', '%s@%s' % (dist.name, dist.latest.version)]) if component.git is not None: table.append(['git', component.git]) if component.manifest is not None: table.append(['manifest', component.manifest]) table.append(['last checked', prettydate(component.last_checked)]) localdir = component.localdir if os.path.isdir(localdir) and len(os.listdir(localdir)) != 0: table.append(['located at', localdir]) for i, data in enumerate(table): table[i] = [ click.style(data[0] + ':' if len(data[0]) > 0 else '', fg="white"), data[1] ] # print and remove last new line click.echo(pretty_columns(table)[:-1])