def create_pkg_file(request):
workdir = empty_dir(request)
remote_path = "pkg.tar.gz"
local_path = os.path.join(workdir, remote_path)
d = tarfile.TarInfo('home')
d.type = tarfile.DIRTYPE
tar = tarfile.open(local_path, "w:gz")
tar.addfile(d)
tar.close()
upaas_config = load_main_config()
storage = load_handler(upaas_config.storage.handler,
upaas_config.storage.settings)
storage.put(local_path, remote_path)
def cleanup():
try:
storage.delete(remote_path)
except FileNotFound:
pass
request.addfinalizer(cleanup)
request.instance.storage = storage
request.instance.pkg_file_path = remote_path
def __init__(self, builder_config):
"""
:param builder_config: Builder configuration.
"""
self.config = builder_config
self.storage = load_handler(self.config.storage.handler,
self.config.storage.settings)
def delete_package_file(self, null_filename=True):
log.debug(_("Deleting package file for {pkg}").format(
pkg=self.safe_id))
if not self.filename:
log.debug(_("Package {pkg} has no filename, skipping "
"delete").format(pkg=self.safe_id))
return
storage = load_handler(self.upaas_config.storage.handler,
self.upaas_config.storage.settings)
if not storage:
log.error(_("Storage handler '{handler}' not found, cannot "
"package file").format(
handler=self.upaas_config.storage.handler))
return
log.debug(_("Checking if package file {path} is stored").format(
path=self.filename))
if storage.exists(self.filename):
log.info(_("Removing package {pkg} file from storage").format(
pkg=self.safe_id))
storage.delete(self.filename)
if null_filename:
log.info(_("Clearing filename for package {pkg}").format(
pkg=self.safe_id))
del self.filename
self.save()
def unpack(self):
# directory is encoded into string to prevent unicode errors
directory = tempfile.mkdtemp(dir=self.upaas_config.paths.workdir,
prefix="upaas_package_").encode("utf-8")
storage = load_handler(self.upaas_config.storage.handler,
self.upaas_config.storage.settings)
if not storage:
log.error("Storage handler '%s' not "
"found" % self.upaas_config.storage.handler)
workdir = os.path.join(directory, "system")
pkg_path = os.path.join(directory, self.filename)
if os.path.exists(self.package_path):
log.error(_("Package directory already exists: {path}").format(
path=self.package_path))
raise UnpackError(_("Package directory already exists"))
log.info("Fetching package '%s'" % self.filename)
try:
storage.get(self.filename, pkg_path)
except StorageError:
log.error(_("Storage error while fetching package {name}").format(
name=self.filename))
utils.rmdirs(directory)
raise UnpackError(_("Storage error while fetching package "
"{name}").format(name=self.filename))
log.info("Unpacking package")
os.mkdir(workdir, 0o755)
if not tar.unpack_tar(pkg_path, workdir):
log.error(_("Error while unpacking package to '{workdir}'").format(
workdir=workdir))
utils.rmdirs(directory)
raise UnpackError(_("Error during package unpack"))
with open(os.path.join(workdir, self.ack_filename), 'w') as ack:
ack.write(_('Unpacked: {now}').format(now=datetime.datetime.now()))
for feature in self.application.feature_helper.load_enabled_features():
feature.after_unpack(self.application, workdir)
log.info(_("Package unpacked, moving into '{path}'").format(
path=self.package_path))
try:
shutil.move(workdir, self.package_path)
except shutil.Error as e:
log.error(_("Error while moving unpacked package to final "
"destination: e").format(e=e))
utils.rmdirs(directory, self.package_path)
raise UnpackError(_("Can't move to final directory: "
"{path}").format(path=self.package_path))
log.info(_("Package moved"))
utils.rmdirs(directory)
def trim_package_files(self):
"""
Removes over limit package files from database. Number of packages per
app that are kept in database for rollback feature are set in user
limits as 'packages_per_app'.
"""
storage = load_handler(self.upaas_config.storage.handler,
self.upaas_config.storage.settings)
if not storage:
log.error("Storage handler '%s' not found, cannot trim "
"packages" % self.upaas_config.storage.handler)
return
removed = 0
for pkg in Package.objects(application=self, filename__exists=True)[
self.owner.limits['packages_per_app']:]:
if pkg.id == self.current_package.id:
continue
removed += 1
pkg.delete_package_file(null_filename=True)
if removed:
log.info("Removed %d package file(s) for app %s" % (removed,
self.name))
def build_package(self, system_filename=None, interpreter_version=None,
current_revision=None, env=None):
"""
Build a package
:param system_filename: Use given file as base system, if None empty
system image will be used.
:param interpreter_version: Use specific interpreter version, only used
for fresh packages.
:param current_revision: VCS revision id from current package.
"""
if interpreter_version:
self.interpreter_version = interpreter_version
log.info("Using forced interpreter version: "
"%s" % interpreter_version)
self.storage = load_handler(self.config.storage.handler,
self.config.storage.settings)
if system_filename and self.storage.exists(system_filename):
log.info("Starting package build using package "
"%s" % system_filename)
if current_revision:
log.info("VCS revision from current package: "
"%s" % current_revision)
self.current_revision = current_revision
else:
if system_filename:
log.warning("Requested base package file not found, using "
"empty system image")
self.envs['UPAAS_FRESH_PACKAGE'] = 'true'
system_filename = None
log.info("Starting package build using empty system image")
if not self.has_valid_os_image():
try:
self.bootstrap_os()
except exceptions.OSBootstrapError as e:
self.system_error("Error during os bootstrap: %s" % e)
except StorageError as e:
self.system_error("Error during uploading OS image: "
"%s" % e)
if not self.interpreter_version:
self.user_error("Unsupported interpreter version")
if env:
log.info("Additional ENV: %s" % ', '.join(env.keys()))
self.envs.update(env)
self.actions.update(self.parse_actions(self.metadata))
self.envs.update(self.parse_envs(self.metadata))
self.os_packages += self.parse_packages(self.metadata)
result = BuildResult()
result.parent = system_filename
result.interpreter_version = self.interpreter_version
# directory is encoded into string to prevent unicode errors
directory = tempfile.mkdtemp(dir=self.config.paths.workdir,
prefix="upaas_package_")
workdir = os.path.join(directory, "workdir")
chroot_homedir = self.config.apps.home
os.mkdir(workdir, 0o755)
log.info("Working directory created at '%s'" % workdir)
self.envs['HOME'] = chroot_homedir
if not self.unpack_os(directory, workdir,
system_filename=system_filename):
kill_and_remove_dir(directory)
self.system_error("Unpacking OS image failed")
log.info("OS image unpacked")
result.progress = 10
yield result
log.info("Using interpreter %s, version %s" % (
self.metadata.interpreter.type, self.interpreter_version))
if not self.run_actions(self.builder_action_names, workdir):
kill_and_remove_dir(directory)
self.system_error("System actions failed")
log.info("All builder actions executed")
result.progress = 20
yield result
if not self.install_packages(workdir, self.os_packages):
kill_and_remove_dir(directory)
self.user_error("Failed to install OS packages")
log.info("All packages installed")
result.progress = 35
yield result
if not self.run_actions(self.interpreter_action_names, workdir, '/'):
kill_and_remove_dir(directory)
self.system_error("Interpreter actions failed")
log.info("All interpreter actions executed")
result.progress = 40
yield result
# TODO if building fails up to this point, then we can try retry it
# on another builder (for a limited number of times)
if system_filename:
if not self.update(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Updating repository failed")
else:
if not self.clone(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Cloning repository failed")
log.info("Application repository ready")
result.progress = 45
yield result
result.vcs_revision = self.vcs_info(workdir, chroot_homedir)
result.progress = 46
yield result
if not self.write_files(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.user_error("Creating files from metadata failed")
log.info("Created all files from metadata")
result.progress = 49
yield result
if not self.run_actions(self.app_action_names, workdir,
chroot_homedir):
self.user_error("Application actions failed")
kill_and_remove_dir(directory)
log.info("All application actions executed")
result.progress = 85
yield result
if not self.run_actions(self.finalize_action_names, workdir, '/'):
kill_and_remove_dir(directory)
self.system_error("Finalize actions failed")
log.info("All final actions executed")
result.progress = 88
yield result
if not self.chown_app_dir(workdir, chroot_homedir):
kill_and_remove_dir(directory)
self.system_error("Setting file ownership failed")
log.info("Owner of application directory updated")
result.progress = 89
yield result
if not self.umount_filesystems(workdir):
kill_and_remove_dir(directory)
self.system_error("Failed to unmount filesystems")
result.progress = 90
yield result
package_path = os.path.join(directory, "package")
if not tar.pack_tar(workdir, package_path):
kill_and_remove_dir(directory)
self.system_error("Creating package file failed")
result.bytes = os.path.getsize(package_path)
log.info("Application package created, "
"%s" % utils.bytes_to_human(result.bytes))
result.progress = 93
yield result
checksum = calculate_file_sha256(package_path)
log.info("Package checksum: %s" % checksum)
result.progress = 96
yield result
try:
self.storage.put(package_path, checksum)
except StorageError as e:
kill_and_remove_dir(directory)
self.system_error("Package upload failed: %s" % e)
kill_and_remove_dir(directory)
result.progress = 100
result.filename = checksum
result.checksum = checksum
yield result
def test_find_storage():
assert load_handler(
'upaas.storage.mongodb.MongoDBStorage') is not None
def test_find_storage_invalid():
with pytest.raises(ConfigurationError):
load_handler("upaas.storage.local.LocalStorage")
def test_find_storage():
assert load_handler("upaas.storage.local.LocalStorage", settings={"dir": "/"}) is not None
def test_find_storage_invalid():
with pytest.raises(ConfigurationError):
load_handler('invalid.storage.module.Handler')
def load_feature(self, name, config, value):
try:
return load_handler(config.handler, name, config.settings, value)
except ConfigurationError as e:
log.error(_("Error while loading {name} feature: {msg}").format(
name=name, msg=e))
