def download(cls,
manifest=None,
localdir=None,
version=None,
override_existing=False,
**kwargs):
try:
repo = GitRepo(localdir)
except (InvalidGitRepositoryError, NoSuchPathError):
repo = GitRepo.init(localdir)
if manifest.git is not None:
try:
repo.create_remote('origin', manifest.git)
except GitCommandError as e:
pass
try:
if sys.stdout.isatty():
repo.remotes.origin.fetch(progress=GitProgressBar(
label="%s@%s" % (str(manifest), version.version)))
else:
for fetch_info in repo.remotes.origin.fetch():
logger.debug(
"Updated %s %s to %s" %
(manifest.git, fetch_info.ref, fetch_info.commit))
# self.last_checked = datetime.now()
except (GitCommandError, AttributeError) as e:
logger.error("Could not fetch %s: %s" % (manifest.git, str(e)))
if version.git_sha is not None:
repo.git.checkout(version.git_sha)
def init_env(self): # noqa: C901
"""
Determines whether the integrity of the kraft application, namely
determining whether the kraft application can run under the given
runtime environment.
"""
if 'UK_CACHEDIR' not in os.environ:
os.environ['UK_CACHEDIR'] = os.path.join(os.environ['HOME'],
UNIKRAFT_CACHEDIR)
if 'UK_WORKDIR' not in os.environ:
os.environ['UK_WORKDIR'] = os.path.join(os.environ['HOME'],
UNIKRAFT_WORKDIR)
if os.path.isdir(os.environ['UK_WORKDIR']) is False:
os.mkdir(os.environ['UK_WORKDIR'])
if 'UK_ROOT' not in os.environ:
os.environ['UK_ROOT'] = os.path.join(os.environ['UK_WORKDIR'],
UNIKRAFT_COREDIR)
if os.path.isdir(os.environ['UK_ROOT']) is False:
os.mkdir(os.environ['UK_ROOT'])
if 'UK_ARCHS' not in os.environ:
os.environ['UK_ARCHS'] = os.path.join(os.environ['UK_WORKDIR'],
UNIKRAFT_ARCHSDIR)
if os.path.isdir(os.environ['UK_ARCHS']) is False:
os.mkdir(os.environ['UK_ARCHS'])
if 'UK_PLATS' not in os.environ:
os.environ['UK_PLATS'] = os.path.join(os.environ['UK_WORKDIR'],
UNIKRAFT_PLATSDIR)
if os.path.isdir(os.environ['UK_PLATS']) is False:
os.mkdir(os.environ['UK_PLATS'])
if 'UK_LIBS' not in os.environ:
os.environ['UK_LIBS'] = os.path.join(os.environ['UK_WORKDIR'],
UNIKRAFT_LIBSDIR)
if os.path.isdir(os.environ['UK_LIBS']) is False:
os.mkdir(os.environ['UK_LIBS'])
if 'UK_APPS' not in os.environ:
os.environ['UK_APPS'] = os.path.join(os.environ['UK_WORKDIR'],
UNIKRAFT_APPSDIR)
if os.path.isdir(os.environ['UK_APPS']) is False:
os.mkdir(os.environ['UK_APPS'])
# Check if we have a build-time engine set
if 'UK_BUILD_ENGINE' not in os.environ:
os.environ['UK_BUILD_ENGINE'] = 'gcc'
if 'KRAFTRC' not in os.environ:
os.environ['KRAFTRC'] = os.path.join(os.environ['HOME'], KRAFTRC)
if os.path.exists(os.environ['KRAFTRC']) is False:
default_kraftrc = pkgutil.get_data(__name__, KRAFTRC)
logger.debug("Creating %s..." % os.environ['KRAFTRC'])
if not default_kraftrc:
Path(os.environ['KRAFTRC']).touch()
else:
kraftrc = open(os.environ['KRAFTRC'], "wb")
kraftrc.write(default_kraftrc)
kraftrc.close()
self._env = Environment.from_env_file(self._workdir, None)
self._cache = Cache(self.env)
self._settings = Settings(os.environ['KRAFTRC'])
def is_stale(self):
"""Determine if the list of remote repositories is stale. Return a
boolean value if at least one repository is marked as stale."""
logger.debug("Checking cache for staleness...")
if len(self.all()) == 0:
return True
# biggest_timeout = 0
# repos = self.repos()
# # If there is nothing cached, this is also stale
# if len(repos) == 0:
# return True
# for repo in repos:
# # If we have never checked, this is stale
# if repos[repo].last_checked is None:
# return True
# diff = (datetime.now() - repos[repo].last_checked).total_seconds()
# if diff > biggest_timeout:
# biggest_timeout = diff
# if biggest_timeout > STALE_TIMEOUT:
# return True
return False
def download(ctx, self, manifest=None, localdir=None, version=None,
override_existing=False):
if version.tarball is None:
logger.warn("Cannot download tarball, not in manifest")
return
remote = manifest.get_version(version.version).tarball
if remote.endswith(".tar.gz"):
ext = ".tar.gz"
else:
_, ext = os.path.splitext(remote)
local = os.path.join(
ctx.obj.env.get('UK_CACHEDIR', os.path.join(
ctx.obj.workdir,
UNIKRAFT_CACHEDIR)
),
"%s-%s%s" % (manifest.name, version.version, ext)
)
logger.debug("Downloading %s..." % remote)
t = TarballProgressBar(label="%s/%s@%s"
% (manifest.type.shortname, manifest.name, version.version)
)
urllib.request.urlretrieve(
remote,
filename=local,
reporthook=t.update_to,
data=None
)
def execute(cmd="", env={}, dry_run=False, use_logger=False):
if type(cmd) is list:
cmd = " ".join(cmd)
logger.debug("Running: %s" % cmd)
if not dry_run:
popen = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
env=merge_dicts(os.environ, env))
for line in popen.stdout:
line = line.strip().decode('ascii')
if use_logger:
logger.info(line)
else:
print(line)
popen.stdout.close()
return_code = popen.wait()
if return_code is not None and int(return_code) > 0:
return return_code
return 0
def kraft_build(ctx, fast):
logger.debug("Building %s..." % ctx.workdir)
try:
project = Project.from_config(
ctx.workdir,
config.load(
config.find(ctx.workdir, None, ctx.env)
)
)
except KraftError as e:
logger.error(str(e))
sys.exit(1)
if not project.is_configured():
if click.confirm('It appears you have not configured your application. Would you like to do this now?', default=True):
project.configure()
n_proc = None
if fast:
# This simply set the `-j` flag which signals to make to use all cores.
n_proc = ""
project.build(n_proc=n_proc)
def open_menuconfig(self):
"""
Run the make menuconfig target.
"""
cmd = self.make_raw('menuconfig')
logger.debug("Running:\n%s" % ' '.join(cmd))
subprocess.run(cmd)
def kraft_clean(ctx,
workdir=None,
proper=False,
dist=False,
progress=True,
libs=None):
"""
Cleans the build artifacts of a Unikraft unikernel.
"""
if workdir is None or os.path.exists(workdir) is False:
raise ValueError("working directory is empty: %s" % workdir)
logger.debug("Cleaning %s..." % workdir)
app = Application.from_workdir(workdir)
if progress:
if proper:
make_progressbar(app.make_raw(extra="properclean"))
elif dist:
make_progressbar(app.make_raw(extra="distclean"))
else:
if len(libs) is not None and len(libs) > 0:
for lib in list(libs):
make_progressbar(app.make_raw(extra="clean-lib%s" % lib))
else:
make_progressbar(app.make_raw(extra="clean"))
else:
app.clean(proper=proper, dist=dist)
def automount(self, dry_run=False):
for vol in self.volumes.all():
if vol.driver is VolumeDriver.VOL_INITRD:
self.add_initrd(vol.source)
if vol.driver is VolumeDriver.VOL_9PFS:
source = vol.source
# Extract tarball file systems
if not dry_run and vol.source.lower().endswith(
('.tgz', '.tar.gz', '.tar')):
source = tempfile.mkdtemp()
logger.debug('Extracting %s to %s...' %
(vol.source, source))
tarball = tarfile.open(vol.source)
tarball.extractall(source)
tarball.close()
self.add_virtio_9pfs(source)
if vol.driver is VolumeDriver.VOL_RAW:
self.add_virtio_raw(vol.source)
if vol.driver is VolumeDriver.VOL_QCOW2:
self.add_virtio_qcow2(vol.source)
def kraft_build(ctx,
workdir=None,
fetch=True,
prepare=True,
target=None,
fast=False,
force_build=False):
"""
"""
if workdir is None or os.path.exists(workdir) is False:
raise ValueError("working directory is empty: %s" % workdir)
logger.debug("Building %s..." % workdir)
app = Application.from_workdir(workdir, force_build)
if not app.is_configured():
if click.confirm(
'It appears you have not configured your application. Would you like to do this now?',
default=True): # noqa: E501
app.configure()
n_proc = None
if fast:
# This simply set the `-j` flag which signals to make to use all cores.
n_proc = 0
app.build(fetch=fetch, prepare=prepare, target=target, n_proc=n_proc)
def get(self, origin=None):
ret = None
if isinstance(origin, six.string_types) and origin in self._cache:
logger.debug("Retrieving %s from cache..." % origin)
with self._cache_lock:
ret = self._cache[origin]
return ret
def is_stale(self):
"""
Determine if the list of remote repositories is stale. Return a boolean
value if at least one repository is marked as stale.
"""
logger.debug("Checking cache for staleness...")
return True if len(self.all()) == 0 else False
def intrusively_determine_kconfig(self):
if self.is_downloaded:
config_uk = os.path.join(self.localdir, CONFIG_UK)
if os.path.exists(config_uk):
logger.debug("Reading: %s..." % config_uk)
return kconfiglib.Kconfig(filename=config_uk)
return None
def intrusively_determine_kconfig(self):
kconfig = None
config_uk = UK_CONFIG_FILE % self._localdir
if os.path.exists(config_uk):
logger.debug("Reading: %s..." % config_uk)
kconfig = kconfiglib.Kconfig(filename=config_uk)
return kconfig
def get(self, prop, default=None):
try:
result = dpath.util.get(self._settings, prop)
except KeyError:
logger.debug('Missed setting lookup: %s', prop)
result = None
return result
def delete_resource(resource):
if os.path.isfile(resource):
logger.debug("Removing file: %s" % resource)
os.remove(resource)
elif os.path.isdir(resource):
logger.debug("Removing directory: %s" % resource)
shutil.rmtree(resource)
def execute(
self, # noqa: C901
extra_args=None,
background=False,
paused=False,
dry_run=False):
logger.debug("Executing on KVM...")
self._cmd.extend(('-k', self.unikernel))
if background:
self._cmd.append('-x')
if paused:
self._cmd.append('-P')
if dry_run:
self._cmd.append('-D')
if extra_args:
self._cmd.extend(('-a', ' '.join(extra_args)))
self.automount(dry_run)
self.autoconnect(dry_run)
if self.architecture == "x86_64":
self._cmd.extend(('-t', 'x86pc'))
elif self.architecture == "arm64":
self._cmd.extend(('-t', 'arm64v'))
# if platform.machine() != self.architecture:
# self._cmd.append('-W')
if self.arguments:
self._cmd.extend(('-a', self.arguments))
cmd = [QEMU_GUEST]
cmd.extend(self._cmd)
for pre_up_cmd in self._pre_up:
util.execute(pre_up_cmd, dry_run=dry_run)
cmd = list(map(str, cmd))
logger.debug('Running: %s' % ' '.join(cmd))
if not dry_run:
process = subprocess.Popen(cmd)
try:
process.wait()
except KeyboardInterrupt:
try:
process.terminate()
except OSError:
pass
process.wait()
for post_down_cmd in self._post_down:
util.execute(post_down_cmd, dry_run=dry_run)
def save(self, origin, manifest):
if not isinstance(origin, six.string_types):
raise TypeError("origin is not string")
if not isinstance(manifest, Manifest):
raise TypeError("Invalid manifest")
with self._cache_lock:
logger.debug("Saving %s into cache..." % manifest)
self._cache[origin] = manifest
def kraft_build(ctx,
verbose=False,
workdir=None,
fetch=True,
prepare=True,
progress=True,
target=None,
fast=False,
force_build=False):
"""
"""
if workdir is None or os.path.exists(workdir) is False:
raise ValueError("working directory is empty: %s" % workdir)
logger.debug("Building %s..." % workdir)
app = Application.from_workdir(workdir, force_build)
if not app.is_configured():
if click.confirm(
'It appears you have not configured your application. Would you like to do this now?',
default=True): # noqa: E501
app.configure()
n_proc = None
if fast:
# This simply set the `-j` flag which signals to make to use all cores.
n_proc = 0
if fetch:
app.fetch()
if prepare:
app.prepare()
if progress:
return_code = make_progressbar(app.make_raw(verbose=verbose))
else:
return_code = app.build(target=target, n_proc=n_proc)
if return_code > 0:
sys.exit(return_code)
print("\nSuccessfully built unikernels:\n")
for target in app.binaries:
if not os.path.exists(target.binary):
continue
print(" => %s/%s" %
(UNIKRAFT_BUILDDIR, os.path.basename(target.binary)))
print(" => %s/%s (with symbols)" %
(UNIKRAFT_BUILDDIR, os.path.basename(target.binary_debug)))
print("\nTo instantiate, use: kraft run\n")
def save_yaml(self, file=None):
if file is None:
file = os.path.join(self.localdir, SUPPORTED_FILENAMES[0])
yaml = serialize_config(self.repr(), original=file)
logger.debug("Saving: %s" % file)
with open(file, 'w+') as f:
f.write(yaml)
def execute(cmd="", env={}, dry_run=False):
if type(cmd) is list:
cmd = " ".join(cmd)
logger.debug("Running: %s" % cmd)
if not dry_run:
cmd = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, env={**os.environ, **env})
for line in cmd.stdout:
logger.info(line.strip().decode('ascii'))
def kraft_clean(ctx, workdir=None, proper=False):
"""
Cleans the build artifacts of a Unikraft unikernel.
"""
if workdir is None or os.path.exists(workdir) is False:
raise ValueError("working directory is empty: %s" % workdir)
logger.debug("Cleaning %s..." % workdir)
app = Application.from_workdir(workdir)
app.clean(proper=proper)
def run(self, target=None, initrd=None, background=False, # noqa: C901
paused=False, gdb=4123, dbg=False, virtio_nic=None, bridge=None,
interface=None, dry_run=False, args=None, memory=64, cpu_sockets=1,
cpu_cores=1):
if target is None:
raise KraftError('Target not set')
elif target.binary is None:
raise KraftError('Target has not been compiled')
elif not os.path.exists(target.binary):
raise KraftError('Could not find unikernel: %s' % target.binary)
logger.debug("Running binary: %s" % target.binary)
runner = target.platform.runner
runner.use_debug = dbg
runner.architecture = target.architecture.name
if initrd:
runner.add_initrd(initrd)
if virtio_nic:
runner.add_virtio_nic(virtio_nic)
if bridge:
runner.add_bridge(bridge)
if interface:
runner.add_interface(interface)
if gdb:
runner.open_gdb(gdb)
if memory:
runner.set_memory(memory)
if cpu_sockets:
runner.set_cpu_sockets(cpu_sockets)
if cpu_cores:
runner.set_cpu_cores(cpu_cores)
runner.unikernel = target.binary
runner.execute(
extra_args=args,
background=background,
paused=paused,
dry_run=dry_run,
)
def find_config(base_dir, filenames, environment, override_dir=None):
if filenames == ['-']:
return ConfigDetails(
os.path.abspath(override_dir) if override_dir else os.getcwd(),
[KraftFile(None, yaml.safe_load(sys.stdin))], environment)
if filenames:
filenames = [os.path.join(base_dir, f) for f in filenames]
else:
filenames = get_default_config_files(base_dir)
logger.debug("Using configuration files: %s" % (",".join(filenames)))
return ConfigDetails(
override_dir if override_dir else os.path.dirname(filenames[0]),
[KraftFile.from_filename(f) for f in filenames], environment)
def execute(self,
extra_args=None,
background=False,
paused=False,
dry_run=False):
logger.debug("Executing on Xen...")
self._cmd.extend(('-k', self.unikernel))
if background:
self._cmd.append('-X')
if paused:
self._cmd.append('-P')
if dry_run:
self._cmd.append('-D')
if extra_args:
self._cmd.extend(('-a', ' '.join(extra_args)))
self.automount(dry_run)
self.autoconnect(dry_run)
if self.arguments:
self._cmd.extend(('-a', self.arguments))
cmd = [XEN_GUEST]
cmd.extend(self._cmd)
for pre_up_cmd in self._pre_up:
utils.execute(pre_up_cmd, dry_run=dry_run)
cmd = list(map(str, cmd))
logger.debug('Running: %s' % ' '.join(cmd))
if not dry_run:
process = subprocess.Popen(cmd)
try:
process.wait()
except KeyboardInterrupt:
try:
process.terminate()
except OSError:
pass
process.wait()
for post_down_cmd in self._post_down:
utils.execute(post_down_cmd, dry_run=dry_run)
def clean(ctx, proper):
"""
Cleans build files.
"""
logger.debug("Cleaning %s..." % ctx.workdir)
try:
project = Project.from_config(
ctx.workdir, config.load(config.find(ctx.workdir, None, ctx.env)))
except KraftError as e:
logger.error(str(e))
sys.exit(1)
project.clean(proper=proper)
def list_possible_mirrors(self):
extra = []
for lib in self.config.libraries.all():
if not lib.is_fetched:
for mirror in lib.origin_mirrors:
response = requests.head(mirror)
if response.status_code == 200:
extra.append(
lib.kname +
UNIKRAFT_LIB_MAKEFILE_URL_EXT +
"=" +
mirror
)
else:
logger.debug("Mirror down: %s" % mirror)
break
return extra
def kraft_prepare(ctx, workdir=None):
"""
"""
if workdir is None or os.path.exists(workdir) is False:
raise ValueError("working directory is empty: %s" % workdir)
logger.debug("Preparing %s..." % workdir)
app = Application.from_workdir(workdir)
if not app.is_configured():
if click.confirm(
'It appears you have not configured your application. Would you like to do this now?',
default=True): # noqa: E501
app.configure()
app.prepare()
def recursively_copy(src, dest, overwrite=False, ignore=None):
if os.path.basename(src) in ignore:
pass
elif os.path.isdir(src):
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(src)
for f in files:
recursively_copy(os.path.join(src, f), os.path.join(dest, f),
overwrite, ignore)
elif (os.path.exists(dest) and overwrite) or os.path.exists(dest) is False:
logger.debug('Copying %s => %s' % (src, dest))
try:
copyfile(src, dest)
except SameFileError:
pass
def make_progressbar(make=""):
if make is None or len(make) == 0:
return None
make_n = make[:]
make_n.append("-n")
logger.debug("Calculating how many files to build...")
logger.debug(" ".join(make_n))
all_make_commands = subprocess.check_output(make_n)
all_make_commands = all_make_commands.decode('utf-8').split('\n')
actual_make_commands = []
for i, command in enumerate(all_make_commands):
command = command.strip()
if command.startswith('make') \
or command == "" \
or command == ":" \
or "fixdep" in command:
pass
else:
actual_make_commands.append(command)
with std_out_err_redirect_tqdm() as orig_stdout:
logger.debug("Starting build...")
logger.debug(" ".join(make))
popen = subprocess.Popen(make, stdout=subprocess.PIPE, env=os.environ)
with tqdm(
total=len(actual_make_commands),
file=orig_stdout,
unit="file",
leave=False,
bar_format=
"{desc:<3}{percentage:3.0f}% {bar} {n_fmt}/{total_fmt} [{rate_fmt}{postfix}]",
dynamic_ncols=True) as t: # noqa: E125
for line in popen.stdout:
t.update()
line = line.strip().decode('ascii')
if line.startswith("make: Leaving directory") is False and \
line.startswith("make: Entering directory") is False:
print(line)
t.close()
popen.stdout.close()
return popen.wait()
return 0