def install_frontend(ctx): """Install frontend into an environment""" next_environment = get_next_environment(ctx) set_instance(ctx.obj["instance"], next_environment) _install_frontend(ctx) finish(ctx)
def _install_backend(ctx): """Installs the backend into an environment""" instance_name = ctx.obj["instance"] env = get_next_environment(ctx) set_instance(instance_name, env) log("Installing backend on", env, lvl=debug) env_path = get_path("lib", "") user = ctx.obj["instance_configuration"]["user"] success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "setup.py", "develop" ], sudo=user, ) if not success: output = str(result) if "was unable to detect version" in output: log( "Installing from dirty repository. This might result in dependency " "version problems!", lvl=hilight, ) else: log( "Something unexpected happened during backend installation:\n", result, lvl=hilight, ) # TODO: Another fault might be an unclean package path. # But i forgot the log message to check for. # log('This might be a problem due to unclean installations of Python' # ' libraries. Please check your path.') log("Installing requirements") success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "pip3"), "install", "-r", "requirements.txt", ], sudo=user, ) if not success: log(format_result(result), lvl=error) return True
def install(self, event): """Installs a new Isomer package on the running instance""" self.log("Installing package:", event.data) instance_config = self.context.obj["instance_configuration"] # repository = get_path("lib", "repository") environments = instance_config["environments"] active = instance_config["environment"] next_environment = get_next_environment(self.context) self.fireEvent( notify_restart_required(reason="Module installed: " + event.data))
def cert(ctx, selfsigned): """instance a local SSL certificate""" instance_configuration = ctx.obj["instance_configuration"] instance_name = ctx.obj["instance"] next_environment = get_next_environment(ctx) set_instance(instance_name, next_environment) if selfsigned: _instance_selfsigned(instance_configuration) else: log("This is work in progress") abort(55555) _instance_letsencrypt(instance_configuration) finish(ctx)
def install_environment_modules(ctx, source, force, urls, store_url): """Add and install a module only to a single environment Note: This does not modify the instance configuration, so this will not be permanent during upgrades etc. """ instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instances"][instance_name] next_environment = get_next_environment(ctx) user = instance_configuration["user"] installed = instance_configuration["environments"][next_environment][ "installed"] if not installed: log("Please install the '%s' environment first." % next_environment, lvl=error) abort(50000) set_instance(instance_name, next_environment) for url in urls: result = _install_module(source, url, force=force, user=user, store_url=store_url) if result is False: log("Installation failed!", lvl=error) abort(50000) package_name, package_version = result descriptor = {"version": package_version, "source": source, "url": url} if store_url != DEFAULT_STORE_URL: descriptor["store_url"] = store_url instance_configuration["environments"][next_environment]["modules"][ package_name] = descriptor write_instance(instance_configuration) finish(ctx)
def set_parameter(ctx, parameter, value, force): """Set a configuration parameter of an environment""" # TODO: Generalize and improve this. # - To get less code redundancy (this is also in instance.py) # - To be able to set lists, dicts and other types correctly log("Setting %s to %s" % (parameter, value)) next_environment = get_next_environment(ctx) environment_configuration = ctx.obj["instance_configuration"][ 'environments'][next_environment] defaults = environment_template converted_value = None try: parameter_type = type(defaults[parameter]) log(parameter_type, pretty=True, lvl=debug) if parameter_type == tomlkit.items.Integer: converted_value = int(value) elif parameter_type == bool: converted_value = value.upper() == "TRUE" else: converted_value = value except KeyError: log("Available parameters:", sorted(list(defaults.keys()))) abort(EXIT_INVALID_PARAMETER) if converted_value is None: log("Converted value was None! Recheck the new config!", lvl=warn) environment_configuration[parameter] = converted_value log("New config:", environment_configuration, pretty=True, lvl=debug) ctx.obj["instances"][ctx.obj["instance"]]['environments'][ next_environment] = environment_configuration if valid_configuration(ctx) or force: write_instance(ctx.obj["instances"][ctx.obj["instance"]]) finish(ctx) else: log("New configuration would not be valid", lvl=critical) abort(EXIT_INVALID_CONFIGURATION)
def _install_provisions(ctx, import_file=None, skip_provisions=False): """Load provisions into database""" instance_configuration = ctx.obj["instance_configuration"] env = get_next_environment(ctx) env_path = get_path("lib", "") log("Installing provisioning data") if not skip_provisions: success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "./iso", "-nc", "--flog", "5", "--config-path", get_etc_path(), "-i", instance_configuration["name"], "-e", env, "install", "provisions", ], # Note: no sudo necessary as long as we do not enforce # authentication on databases ) if not success: log("Could not provision data:", lvl=error) log(format_result(result), lvl=error) return False if import_file is not None: log("Importing backup") log(ctx.obj, pretty=True) host, port = ctx.obj["dbhost"].split(":") load(host, int(port), ctx.obj["dbname"], import_file) return True
def install_instance_modules(ctx, source, urls, install_env, force, store_url): """Add (and optionally immediately install) modules for an instance. This will add them to the instance's configuration, so they will be upgraded as well as reinstalled on other environment changes. If you're installing from a store, you can specify a custom store URL with the --store-url argument. """ instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instances"][instance_name] for url in urls: descriptor = [source, url] if store_url != DEFAULT_STORE_URL: descriptor.append(store_url) if descriptor not in instance_configuration["modules"]: instance_configuration["modules"].append(descriptor) elif not force: log("Module %s is already installed. Use --force to install anyway." % url) abort(50000) write_instance(instance_configuration) if install_env is True: next_environment = get_next_environment(ctx) environments = instance_configuration['environments'] if environments[next_environment]["installed"] is False: log("Environment %s is not installed, cannot install modules." % next_environment, lvl=warn) abort(50600) return del ctx.params["install_env"] ctx.forward(install_environment_modules) finish(ctx)
def _install_frontend(ctx): """Install and build the frontend""" env = get_next_environment(ctx) env_path = get_path("lib", "") instance_configuration = ctx.obj["instance_configuration"] user = instance_configuration["user"] log("Building frontend") success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "./iso", "-nc", "--config-path", get_etc_path(), "--prefix-path", get_prefix_path(), "-i", instance_configuration["name"], "-e", env, "--clog", "10", "install", "frontend", "--rebuild", ], sudo=user, ) if not success: log(format_result(result), lvl=error) return False return True
def _turnover(ctx, force): """Internal turnover operation""" # if ctx.obj['acting_environment'] is not None: # next_environment = ctx.obj['acting_environment'] # else: next_environment = get_next_environment(ctx) log("Activating environment:", next_environment) env = ctx.obj["instance_configuration"]["environments"][next_environment] log("Inspecting new environment") if not force: if env.get("database", "") == "": log("Database has not been set up correctly.", lvl=critical) abort(EXIT_INSTALLATION_FAILED) if (not env.get("installed", False) or not env.get("tested", False) or not env.get("migrated", False)): log("Installation failed, cannot activate!", lvl=critical) abort(EXIT_INSTALLATION_FAILED) update_service(ctx, next_environment) ctx.obj["instance_configuration"]["environment"] = next_environment write_instance(ctx.obj["instance_configuration"]) # TODO: Effect reload of service # * Systemctl reload # * (Re)start service # * confirm correct operation # - if not, switch back to the other instance, maybe indicate a broken # state for next_environment # - if yes, Store instance configuration and terminate, we're done log("Turned instance over to", next_environment)
def _install_modules(ctx): """Internal function to install modules""" env = get_next_environment(ctx) log("Installing modules into", env, pretty=True) instance_configuration = ctx.obj["instance_configuration"] modules = instance_configuration["modules"] user = instance_configuration["user"] if len(modules) == 0: log("No modules defined for instance") return True for module in modules: log("Installing:", module, pretty=True) store_url = module[2] if module[0] == "store" else DEFAULT_STORE_URL result = _install_module(module[0], module[1], user=user, store_url=store_url) if result is False: log("Installation of module failed!", lvl=warn) else: module_name, module_version = result descriptor = { "name": module_name, "source": module[0], "url": module[1] } if store_url != DEFAULT_STORE_URL: descriptor["store_url"] = store_url instance_configuration["environments"][env]["modules"][ module_name] = descriptor write_instance(instance_configuration) return True
def upgrade(ctx, release, upgrade_modules, restart, handle_cache, source, url): """Upgrades an instance on its other environment and turns over on success. \b 1. Test if other environment is empty 1.1. No - archive and clear it 2. Copy current environment to other environment 3. Clear old bits (venv, frontend) 4. Fetch updates in other environment repository 5. Select a release 6. Checkout that release and its submodules 7. Install release 8. Copy database 9. Migrate data (WiP) 10. Turnover """ instance_config = ctx.obj["instance_configuration"] repository = get_path("lib", "repository") installation_source = source if source is not None else instance_config[ 'source'] installation_url = url if url is not None else instance_config['url'] environments = instance_config["environments"] active = instance_config["environment"] next_environment = get_next_environment(ctx) if environments[next_environment]["installed"] is True: _clear_environment(ctx, clear_env=next_environment) source_paths = [ get_path("lib", "", environment=active), get_path("local", "", environment=active) ] destination_paths = [ get_path("lib", "", environment=next_environment), get_path("local", "", environment=next_environment) ] log(source_paths, destination_paths, pretty=True) for source, destination in zip(source_paths, destination_paths): log("Copying to new environment:", source, destination) copy_directory_tree(source, destination) if handle_cache != "ignore": log("Handling cache") move = handle_cache == "move" copy_directory_tree(get_path("cache", "", environment=active), get_path("cache", "", environment=next_environment), move=move) rmtree(get_path("lib", "venv"), ignore_errors=True) # TODO: This potentially leaves frontend-dev: rmtree(get_path("lib", "frontend"), ignore_errors=True) releases = _get_versions(ctx, source=installation_source, url=installation_url, fetch=True) releases_keys = sorted_alphanumerical(releases.keys()) if release is None: release = releases_keys[-1] else: if release not in releases_keys: log("Unknown release. Maybe try a different release or source.") abort(50100) log("Choosing release", release) _install_environment(ctx, installation_source, installation_url, upgrade=True, release=release) new_database_name = instance_config["name"] + "_" + next_environment copy_database(ctx.obj["dbhost"], active['database'], new_database_name) apply_migrations(ctx) finish(ctx)
def _install_environment( ctx, source=None, url=None, import_file=None, no_sudo=False, force=False, release=None, upgrade=False, skip_modules=False, skip_data=False, skip_frontend=False, skip_test=False, skip_provisions=False, ): """Internal function to perform environment installation""" if url is None: url = source_url elif url[0] == '.': url = url.replace(".", os.getcwd(), 1) if url[0] == '/': url = os.path.abspath(url) instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instance_configuration"] next_environment = get_next_environment(ctx) set_instance(instance_name, next_environment) env = copy(instance_configuration["environments"][next_environment]) env["database"] = instance_name + "_" + next_environment env_path = get_path("lib", "") user = instance_configuration["user"] if no_sudo: user = None log("Installing new other environment for %s on %s from %s in %s" % (instance_name, next_environment, source, env_path)) try: result = get_isomer(source, url, env_path, upgrade=upgrade, sudo=user, release=release) if result is False: log("Getting Isomer failed", lvl=critical) abort(50011, ctx) except FileExistsError: if not force: log( "Isomer already present, please safely clear or " "inspect the environment before continuing! Use --force to ignore.", lvl=warn, ) abort(50012, ctx) else: log("Isomer already present, forcing through anyway.") try: repository = Repo(os.path.join(env_path, "repository")) log("Repo:", repository, lvl=debug) env["version"] = str(repository.git.describe()) except (exc.InvalidGitRepositoryError, exc.NoSuchPathError, exc.GitCommandError): env["version"] = version log( "Not running from a git repository; Using isomer.version:", version, lvl=warn, ) ctx.obj["instance_configuration"]["environments"][next_environment] = env # TODO: Does it make sense to early-write the configuration and then again later? write_instance(ctx.obj["instance_configuration"]) log("Creating virtual environment") success, result = run_process( env_path, [ "virtualenv", "-p", "/usr/bin/python3", "--system-site-packages", "venv" ], sudo=user, ) if not success: log(format_result(result), lvl=error) try: if _install_backend(ctx): log("Backend installed") env["installed"] = True if not skip_modules and _install_modules(ctx): log("Modules installed") # env['installed_modules'] = True if not skip_provisions and _install_provisions( ctx, import_file=import_file): log("Provisions installed") env["provisioned"] = True if not skip_data and _migrate(ctx): log("Data migrated") env["migrated"] = True if not skip_frontend and _install_frontend(ctx): log("Frontend installed") env["frontend"] = True if not skip_test and _check_environment(ctx): log("Environment tested") env["tested"] = True except Exception: log("Error during installation:", exc=True, lvl=critical) log("Environment status now:", env) ctx.obj["instance_configuration"]["environments"][next_environment] = env write_instance(ctx.obj["instance_configuration"])
def _archive(ctx, force=False, dynamic=False): instance_configuration = ctx.obj["instance_configuration"] next_environment = get_next_environment(ctx) env = instance_configuration["environments"][next_environment] log("Instance info:", instance_configuration, next_environment, pretty=True, lvl=debug) log("Installed:", env["installed"], "Tested:", env["tested"], lvl=debug) if (not env["installed"] or not env["tested"]) and not force: log("Environment has not been installed - not archiving.", lvl=warn) return False log("Archiving environment:", next_environment) set_instance(ctx.obj["instance"], next_environment) timestamp = std_now().replace(":", "-").replace(".", "-") temp_path = mkdtemp(prefix="isomer_backup") log("Archiving database") if not dump( instance_configuration["database_host"], instance_configuration["database_port"], env["database"], os.path.join(temp_path, "db_" + timestamp + ".json"), ): if not force: log("Could not archive database.") return False archive_filename = os.path.join( "/var/backups/isomer/", "%s_%s_%s.tgz" % (ctx.obj["instance"], next_environment, timestamp), ) try: shutil.copy( os.path.join(get_etc_instance_path(), ctx.obj["instance"] + ".conf"), temp_path, ) with tarfile.open(archive_filename, "w:gz") as f: if not dynamic: for item in locations: path = get_path(item, "") log("Archiving [%s]: %s" % (item, path)) f.add(path) f.add(temp_path, "db_etc") except (PermissionError, FileNotFoundError) as e: log("Could not archive environment:", e, lvl=error) if not force: return False finally: log("Clearing temporary backup target") shutil.rmtree(temp_path) ctx.obj["instance_configuration"]["environments"]["archive"][ timestamp] = env log(ctx.obj["instance_configuration"]) return archive_filename
def _clear_environment(ctx, force=False, clear_env=None, clear=False, no_archive=False): """Tests an environment for usage, then clears it :param ctx: Click Context :param force: Irrefutably destroy environment content :param clear_env: Environment to clear (Green/Blue) :param clear: Also destroy generated folders :param no_archive: Don't attempt to archive instance """ instance_name = ctx.obj["instance"] if clear_env is None: next_environment = get_next_environment(ctx) else: next_environment = clear_env log("Clearing environment:", next_environment) set_instance(instance_name, next_environment) # log('Testing', environment, 'for usage') env = ctx.obj["instance_configuration"]["environments"][next_environment] if not no_archive: if not (_archive(ctx, force) or force): log("Archival failed, stopping.") abort(5000) log("Clearing env:", env, lvl=debug) for item in locations: path = get_path(item, "") log("Clearing [%s]: %s" % (item, path), lvl=debug) try: shutil.rmtree(path) except FileNotFoundError: log("Path not found:", path, lvl=debug) except PermissionError: log("No permission to clear environment", lvl=error) return False if not clear: _create_folders(ctx) try: delete_database(ctx.obj["dbhost"], "%s_%s" % (instance_name, next_environment), force=True) except pymongo.errors.ServerSelectionTimeoutError: log("No database available") except Exception as e: log("Could not delete database:", e, lvl=warn, exc=True) ctx.obj["instance_configuration"]["environments"][ next_environment] = environment_template write_instance(ctx.obj["instance_configuration"]) return True
def _check_environment(ctx, env=None, dev=False): """General fitness tests of the built environment""" if env is None: env = get_next_environment(ctx) log("Health checking the environment '%s'" % env) # Frontend not_enough_files = False html_missing = False loader_missing = False size_too_small = False # Backend repository_missing = False modules_missing = False venv_missing = False local_missing = False cache_missing = False # Backend if not os.path.exists(os.path.join(get_path('lib', 'repository'))): log("Repository is missing", lvl=warn) repository_missing = True if not os.path.exists(os.path.join(get_path('lib', 'modules'))): log("Modules folder is missing", lvl=warn) modules_missing = True if not os.path.exists(os.path.join(get_path('lib', 'venv'))): log("Virtual environment is missing", lvl=warn) venv_missing = True if not os.path.exists(os.path.join(get_path('local', ''))): log("Local data folder is missing", lvl=warn) local_missing = True if not os.path.exists(os.path.join(get_path('cache', ''))): log("Cache folder is missing", lvl=warn) cache_missing = True # Frontend _, frontend_target = get_frontend_locations(dev) if not os.path.exists(os.path.join(frontend_target, 'index.html')): log("A compiled frontend html seems to be missing", lvl=warn) html_missing = True if not glob.glob(frontend_target + '/main.*.js'): log("A compiled frontend loader seems to be missing", lvl=warn) loader_missing = True size_sum = 0 amount_files = 0 for file in glob.glob(os.path.join(frontend_target, '*.gz')): size_sum += os.stat(file).st_size amount_files += 1 if amount_files < 4: log("The frontend probably did not compile completely", lvl=warn) not_enough_files = True if size_sum < 2 * 1024 * 1024: log("The compiled frontend seems exceptionally small", lvl=warn) size_too_small = True frontend = (repository_missing or modules_missing or venv_missing or local_missing or cache_missing) backend = (not_enough_files or loader_missing or size_too_small or html_missing) result = not (frontend or backend) if result is False: log("Health check failed", lvl=error) return result