def create_pkg_file(request): workdir = empty_dir(request) remote_path = "pkg.tar.gz" local_path = os.path.join(workdir, remote_path) d = tarfile.TarInfo('home') d.type = tarfile.DIRTYPE tar = tarfile.open(local_path, "w:gz") tar.addfile(d) tar.close() upaas_config = load_main_config() storage = load_handler(upaas_config.storage.handler, upaas_config.storage.settings) storage.put(local_path, remote_path) def cleanup(): try: storage.delete(remote_path) except FileNotFound: pass request.addfinalizer(cleanup) request.instance.storage = storage request.instance.pkg_file_path = remote_path
def __init__(self, builder_config): """ :param builder_config: Builder configuration. """ self.config = builder_config self.storage = load_handler(self.config.storage.handler, self.config.storage.settings)
def delete_package_file(self, null_filename=True): log.debug(_("Deleting package file for {pkg}").format( pkg=self.safe_id)) if not self.filename: log.debug(_("Package {pkg} has no filename, skipping " "delete").format(pkg=self.safe_id)) return storage = load_handler(self.upaas_config.storage.handler, self.upaas_config.storage.settings) if not storage: log.error(_("Storage handler '{handler}' not found, cannot " "package file").format( handler=self.upaas_config.storage.handler)) return log.debug(_("Checking if package file {path} is stored").format( path=self.filename)) if storage.exists(self.filename): log.info(_("Removing package {pkg} file from storage").format( pkg=self.safe_id)) storage.delete(self.filename) if null_filename: log.info(_("Clearing filename for package {pkg}").format( pkg=self.safe_id)) del self.filename self.save()
def unpack(self): # directory is encoded into string to prevent unicode errors directory = tempfile.mkdtemp(dir=self.upaas_config.paths.workdir, prefix="upaas_package_").encode("utf-8") storage = load_handler(self.upaas_config.storage.handler, self.upaas_config.storage.settings) if not storage: log.error("Storage handler '%s' not " "found" % self.upaas_config.storage.handler) workdir = os.path.join(directory, "system") pkg_path = os.path.join(directory, self.filename) if os.path.exists(self.package_path): log.error(_("Package directory already exists: {path}").format( path=self.package_path)) raise UnpackError(_("Package directory already exists")) log.info("Fetching package '%s'" % self.filename) try: storage.get(self.filename, pkg_path) except StorageError: log.error(_("Storage error while fetching package {name}").format( name=self.filename)) utils.rmdirs(directory) raise UnpackError(_("Storage error while fetching package " "{name}").format(name=self.filename)) log.info("Unpacking package") os.mkdir(workdir, 0o755) if not tar.unpack_tar(pkg_path, workdir): log.error(_("Error while unpacking package to '{workdir}'").format( workdir=workdir)) utils.rmdirs(directory) raise UnpackError(_("Error during package unpack")) with open(os.path.join(workdir, self.ack_filename), 'w') as ack: ack.write(_('Unpacked: {now}').format(now=datetime.datetime.now())) for feature in self.application.feature_helper.load_enabled_features(): feature.after_unpack(self.application, workdir) log.info(_("Package unpacked, moving into '{path}'").format( path=self.package_path)) try: shutil.move(workdir, self.package_path) except shutil.Error as e: log.error(_("Error while moving unpacked package to final " "destination: e").format(e=e)) utils.rmdirs(directory, self.package_path) raise UnpackError(_("Can't move to final directory: " "{path}").format(path=self.package_path)) log.info(_("Package moved")) utils.rmdirs(directory)
def trim_package_files(self): """ Removes over limit package files from database. Number of packages per app that are kept in database for rollback feature are set in user limits as 'packages_per_app'. """ storage = load_handler(self.upaas_config.storage.handler, self.upaas_config.storage.settings) if not storage: log.error("Storage handler '%s' not found, cannot trim " "packages" % self.upaas_config.storage.handler) return removed = 0 for pkg in Package.objects(application=self, filename__exists=True)[ self.owner.limits['packages_per_app']:]: if pkg.id == self.current_package.id: continue removed += 1 pkg.delete_package_file(null_filename=True) if removed: log.info("Removed %d package file(s) for app %s" % (removed, self.name))
def build_package(self, system_filename=None, interpreter_version=None, current_revision=None, env=None): """ Build a package :param system_filename: Use given file as base system, if None empty system image will be used. :param interpreter_version: Use specific interpreter version, only used for fresh packages. :param current_revision: VCS revision id from current package. """ if interpreter_version: self.interpreter_version = interpreter_version log.info("Using forced interpreter version: " "%s" % interpreter_version) self.storage = load_handler(self.config.storage.handler, self.config.storage.settings) if system_filename and self.storage.exists(system_filename): log.info("Starting package build using package " "%s" % system_filename) if current_revision: log.info("VCS revision from current package: " "%s" % current_revision) self.current_revision = current_revision else: if system_filename: log.warning("Requested base package file not found, using " "empty system image") self.envs['UPAAS_FRESH_PACKAGE'] = 'true' system_filename = None log.info("Starting package build using empty system image") if not self.has_valid_os_image(): try: self.bootstrap_os() except exceptions.OSBootstrapError as e: self.system_error("Error during os bootstrap: %s" % e) except StorageError as e: self.system_error("Error during uploading OS image: " "%s" % e) if not self.interpreter_version: self.user_error("Unsupported interpreter version") if env: log.info("Additional ENV: %s" % ', '.join(env.keys())) self.envs.update(env) self.actions.update(self.parse_actions(self.metadata)) self.envs.update(self.parse_envs(self.metadata)) self.os_packages += self.parse_packages(self.metadata) result = BuildResult() result.parent = system_filename result.interpreter_version = self.interpreter_version # directory is encoded into string to prevent unicode errors directory = tempfile.mkdtemp(dir=self.config.paths.workdir, prefix="upaas_package_") workdir = os.path.join(directory, "workdir") chroot_homedir = self.config.apps.home os.mkdir(workdir, 0o755) log.info("Working directory created at '%s'" % workdir) self.envs['HOME'] = chroot_homedir if not self.unpack_os(directory, workdir, system_filename=system_filename): kill_and_remove_dir(directory) self.system_error("Unpacking OS image failed") log.info("OS image unpacked") result.progress = 10 yield result log.info("Using interpreter %s, version %s" % ( self.metadata.interpreter.type, self.interpreter_version)) if not self.run_actions(self.builder_action_names, workdir): kill_and_remove_dir(directory) self.system_error("System actions failed") log.info("All builder actions executed") result.progress = 20 yield result if not self.install_packages(workdir, self.os_packages): kill_and_remove_dir(directory) self.user_error("Failed to install OS packages") log.info("All packages installed") result.progress = 35 yield result if not self.run_actions(self.interpreter_action_names, workdir, '/'): kill_and_remove_dir(directory) self.system_error("Interpreter actions failed") log.info("All interpreter actions executed") result.progress = 40 yield result # TODO if building fails up to this point, then we can try retry it # on another builder (for a limited number of times) if system_filename: if not self.update(workdir, chroot_homedir): kill_and_remove_dir(directory) self.user_error("Updating repository failed") else: if not self.clone(workdir, chroot_homedir): kill_and_remove_dir(directory) self.user_error("Cloning repository failed") log.info("Application repository ready") result.progress = 45 yield result result.vcs_revision = self.vcs_info(workdir, chroot_homedir) result.progress = 46 yield result if not self.write_files(workdir, chroot_homedir): kill_and_remove_dir(directory) self.user_error("Creating files from metadata failed") log.info("Created all files from metadata") result.progress = 49 yield result if not self.run_actions(self.app_action_names, workdir, chroot_homedir): self.user_error("Application actions failed") kill_and_remove_dir(directory) log.info("All application actions executed") result.progress = 85 yield result if not self.run_actions(self.finalize_action_names, workdir, '/'): kill_and_remove_dir(directory) self.system_error("Finalize actions failed") log.info("All final actions executed") result.progress = 88 yield result if not self.chown_app_dir(workdir, chroot_homedir): kill_and_remove_dir(directory) self.system_error("Setting file ownership failed") log.info("Owner of application directory updated") result.progress = 89 yield result if not self.umount_filesystems(workdir): kill_and_remove_dir(directory) self.system_error("Failed to unmount filesystems") result.progress = 90 yield result package_path = os.path.join(directory, "package") if not tar.pack_tar(workdir, package_path): kill_and_remove_dir(directory) self.system_error("Creating package file failed") result.bytes = os.path.getsize(package_path) log.info("Application package created, " "%s" % utils.bytes_to_human(result.bytes)) result.progress = 93 yield result checksum = calculate_file_sha256(package_path) log.info("Package checksum: %s" % checksum) result.progress = 96 yield result try: self.storage.put(package_path, checksum) except StorageError as e: kill_and_remove_dir(directory) self.system_error("Package upload failed: %s" % e) kill_and_remove_dir(directory) result.progress = 100 result.filename = checksum result.checksum = checksum yield result
def test_find_storage(): assert load_handler( 'upaas.storage.mongodb.MongoDBStorage') is not None
def test_find_storage_invalid(): with pytest.raises(ConfigurationError): load_handler("upaas.storage.local.LocalStorage")
def test_find_storage(): assert load_handler("upaas.storage.local.LocalStorage", settings={"dir": "/"}) is not None
def test_find_storage_invalid(): with pytest.raises(ConfigurationError): load_handler('invalid.storage.module.Handler')
def load_feature(self, name, config, value): try: return load_handler(config.handler, name, config.settings, value) except ConfigurationError as e: log.error(_("Error while loading {name} feature: {msg}").format( name=name, msg=e))