def test_bytes_to_human():
assert utils.bytes_to_human(1) == '1.0 bytes'
assert utils.bytes_to_human(1024) == '1.0 KB'
assert utils.bytes_to_human(1024 * 1.5) == '1.5 KB'
assert utils.bytes_to_human(1024 * 1024 * 4.4) == '4.4 MB'
assert utils.bytes_to_human(1024 * 1024 * 1024 * 16.9) == '16.9 GB'
assert utils.bytes_to_human(1024 * 1024 * 1024 * 1024 * 1.06) == '1.1 TB'
assert utils.bytes_to_human(1024 * 1024 * 1024 * 1024 * 1024) == '1.0 PB'
assert utils.bytes_to_human(1024 * 1024 * 1024 * 1024 * 1024 * 1024 * 9) \
== '9.0 EB'
def build_package(self, system_filename=None, interpreter_version=None,
current_revision=None, env=None):
"""
Build a package
:param system_filename: Use given file as base system, if None empty
system image will be used.
:param interpreter_version: Use specific interpreter version, only used
for fresh packages.
:param current_revision: VCS revision id from current package.
"""
if interpreter_version:
self.interpreter_version = interpreter_version
log.info("Using forced interpreter version: "
"%s" % interpreter_version)
self.storage = load_handler(self.config.storage.handler,
self.config.storage.settings)
if system_filename and self.storage.exists(system_filename):
log.info("Starting package build using package "
"%s" % system_filename)
if current_revision:
log.info("VCS revision from current package: "
"%s" % current_revision)
self.current_revision = current_revision
else:
if system_filename:
log.warning("Requested base package file not found, using "
"empty system image")
self.envs['UPAAS_FRESH_PACKAGE'] = 'true'
system_filename = None
log.info("Starting package build using empty system image")
if not self.has_valid_os_image():
try:
self.bootstrap_os()
except exceptions.OSBootstrapError as e:
self.system_error("Error during os bootstrap: %s" % e)
except StorageError as e:
self.system_error("Error during uploading OS image: "
"%s" % e)
if not self.interpreter_version:
self.user_error("Unsupported interpreter version")
if env:
log.info("Additional ENV: %s" % ', '.join(env.keys()))
self.envs.update(env)
self.actions.update(self.parse_actions(self.metadata))
self.envs.update(self.parse_envs(self.metadata))
self.os_packages += self.parse_packages(self.metadata)
result = BuildResult()
result.parent = system_filename
result.interpreter_version = self.interpreter_version
# directory is encoded into string to prevent unicode errors
directory = tempfile.mkdtemp(dir=self.config.paths.workdir,
prefix="upaas_package_")
workdir = os.path.join(directory, "workdir")
chroot_homedir = self.config.apps.home
os.mkdir(workdir, 0o755)
log.info("Working directory created at '%s'" % workdir)
self.envs['HOME'] = chroot_homedir
if not self.unpack_os(directory, workdir,
system_filename=system_filename):
kill_and_remove_dir(directory)
self.system_error("Unpacking OS image failed")
log.info("OS image unpacked")
result.progress = 10
yield result
log.info("Using interpreter %s, version %s" % (
self.metadata.interpreter.type, self.interpreter_version))
if not self.run_actions(self.builder_action_names, workdir):
kill_and_remove_dir(directory)
self.system_error("System actions failed")
log.info("All builder actions executed")
result.progress = 20
yield result
if not self.install_packages(workdir, self.os_packages):
kill_and_remove_dir(directory)
self.user_error("Failed to install OS packages")
log.info("All packages installed")
result.progress = 35
yield result
if not self.run_actions(self.interpreter_action_names, workdir, '/'):
kill_and_remove_dir(directory)
self.system_error("Interpreter actions failed")
log.info("All interpreter actions executed")
result.progress = 40
yield result
# TODO if building fails up to this point, then we can try retry it
# on another builder (for a limited number of times)
if system_filename:
if not self.update(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Updating repository failed")
else:
if not self.clone(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Cloning repository failed")
log.info("Application repository ready")
result.progress = 45
yield result
result.vcs_revision = self.vcs_info(workdir, chroot_homedir)
result.progress = 46
yield result
if not self.write_files(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Creating files from metadata failed")
log.info("Created all files from metadata")
result.progress = 49
yield result
if not self.run_actions(self.app_action_names, workdir,
chroot_homedir):
self.user_error("Application actions failed")
kill_and_remove_dir(directory)
log.info("All application actions executed")
result.progress = 85
yield result
if not self.run_actions(self.finalize_action_names, workdir, '/'):
kill_and_remove_dir(directory)
self.system_error("Finalize actions failed")
log.info("All final actions executed")
result.progress = 88
yield result
if not self.chown_app_dir(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.system_error("Setting file ownership failed")
log.info("Owner of application directory updated")
result.progress = 89
yield result
if not self.umount_filesystems(workdir):
kill_and_remove_dir(directory)
self.system_error("Failed to unmount filesystems")
result.progress = 90
yield result
package_path = os.path.join(directory, "package")
if not tar.pack_tar(workdir, package_path):
kill_and_remove_dir(directory)
self.system_error("Creating package file failed")
result.bytes = os.path.getsize(package_path)
log.info("Application package created, "
"%s" % utils.bytes_to_human(result.bytes))
result.progress = 93
yield result
checksum = calculate_file_sha256(package_path)
log.info("Package checksum: %s" % checksum)
result.progress = 96
yield result
try:
self.storage.put(package_path, checksum)
except StorageError as e:
kill_and_remove_dir(directory)
self.system_error("Package upload failed: %s" % e)
kill_and_remove_dir(directory)
result.progress = 100
result.filename = checksum
result.checksum = checksum
yield result