def install(ctx, **kwargs): """Install a new environment of an instance""" log("Installing instance") env = ctx.obj["instance_configuration"]["environments"] green = env["green"]["installed"] blue = env["blue"]["installed"] if green or blue: log( "At least one environment is installed or in a non-clear state.\n" "Please use 'iso instance upgrade' to upgrade an instance.", lvl=warn, ) abort(50081) _clear_instance(ctx, force=kwargs["force"], clear=False, no_archive=True) _install_environment(ctx, **kwargs) ctx.obj["instance_configuration"]["source"] = kwargs["source"] ctx.obj["instance_configuration"]["url"] = kwargs["url"] write_instance(ctx.obj["instance_configuration"]) _turnover(ctx, force=kwargs["force"]) finish(ctx)
def create(ctx, instance_name): """Create a new instance""" if instance_name == "": instance_name = ctx.obj["instance"] if instance_name in ctx.obj["instances"]: abort(EXIT_INSTANCE_EXISTS) log("Creating instance:", instance_name) instance_configuration = instance_template instance_configuration["name"] = instance_name ctx.obj["instances"][instance_name] = instance_configuration write_instance(instance_configuration) finish(ctx)
def install_environment_modules(ctx, source, force, urls, store_url): """Add and install a module only to a single environment Note: This does not modify the instance configuration, so this will not be permanent during upgrades etc. """ instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instances"][instance_name] next_environment = get_next_environment(ctx) user = instance_configuration["user"] installed = instance_configuration["environments"][next_environment][ "installed"] if not installed: log("Please install the '%s' environment first." % next_environment, lvl=error) abort(50000) set_instance(instance_name, next_environment) for url in urls: result = _install_module(source, url, force=force, user=user, store_url=store_url) if result is False: log("Installation failed!", lvl=error) abort(50000) package_name, package_version = result descriptor = {"version": package_version, "source": source, "url": url} if store_url != DEFAULT_STORE_URL: descriptor["store_url"] = store_url instance_configuration["environments"][next_environment]["modules"][ package_name] = descriptor write_instance(instance_configuration) finish(ctx)
def set_parameter(ctx, parameter, value, force): """Set a configuration parameter of an environment""" # TODO: Generalize and improve this. # - To get less code redundancy (this is also in instance.py) # - To be able to set lists, dicts and other types correctly log("Setting %s to %s" % (parameter, value)) next_environment = get_next_environment(ctx) environment_configuration = ctx.obj["instance_configuration"][ 'environments'][next_environment] defaults = environment_template converted_value = None try: parameter_type = type(defaults[parameter]) log(parameter_type, pretty=True, lvl=debug) if parameter_type == tomlkit.items.Integer: converted_value = int(value) elif parameter_type == bool: converted_value = value.upper() == "TRUE" else: converted_value = value except KeyError: log("Available parameters:", sorted(list(defaults.keys()))) abort(EXIT_INVALID_PARAMETER) if converted_value is None: log("Converted value was None! Recheck the new config!", lvl=warn) environment_configuration[parameter] = converted_value log("New config:", environment_configuration, pretty=True, lvl=debug) ctx.obj["instances"][ctx.obj["instance"]]['environments'][ next_environment] = environment_configuration if valid_configuration(ctx) or force: write_instance(ctx.obj["instances"][ctx.obj["instance"]]) finish(ctx) else: log("New configuration would not be valid", lvl=critical) abort(EXIT_INVALID_CONFIGURATION)
def install_instance_modules(ctx, source, urls, install_env, force, store_url): """Add (and optionally immediately install) modules for an instance. This will add them to the instance's configuration, so they will be upgraded as well as reinstalled on other environment changes. If you're installing from a store, you can specify a custom store URL with the --store-url argument. """ instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instances"][instance_name] for url in urls: descriptor = [source, url] if store_url != DEFAULT_STORE_URL: descriptor.append(store_url) if descriptor not in instance_configuration["modules"]: instance_configuration["modules"].append(descriptor) elif not force: log("Module %s is already installed. Use --force to install anyway." % url) abort(50000) write_instance(instance_configuration) if install_env is True: next_environment = get_next_environment(ctx) environments = instance_configuration['environments'] if environments[next_environment]["installed"] is False: log("Environment %s is not installed, cannot install modules." % next_environment, lvl=warn) abort(50600) return del ctx.params["install_env"] ctx.forward(install_environment_modules) finish(ctx)
def set_parameter(ctx, parameter, value): """Set a configuration parameter of an instance""" log("Setting %s to %s" % (parameter, value)) instance_configuration = ctx.obj["instance_configuration"] defaults = instance_template converted_value = None path = parameter.split(".") try: default = nested_map_find(defaults, path) parameter_type = type(default) log(parameter_type, pretty=True, lvl=debug) if parameter_type == tomlkit.items.Integer: converted_value = int(value) elif parameter_type == bool: converted_value = value.upper() == "TRUE" else: converted_value = value except KeyError: log("Available parameters:", sorted(list(defaults.keys()))) abort(EXIT_INVALID_PARAMETER) if converted_value is None: log("Converted value was None! Recheck the new config!", lvl=warn) nested_map_update(instance_configuration, converted_value, path) #instance_configuration[parameter] = converted_value log("New config:", instance_configuration, pretty=True, lvl=debug) ctx.obj["instances"][ctx.obj["instance"]] = instance_configuration if valid_configuration(ctx): write_instance(instance_configuration) finish(ctx) else: log("New configuration would not be valid", lvl=critical) abort(EXIT_INVALID_CONFIGURATION)
def _turnover(ctx, force): """Internal turnover operation""" # if ctx.obj['acting_environment'] is not None: # next_environment = ctx.obj['acting_environment'] # else: next_environment = get_next_environment(ctx) log("Activating environment:", next_environment) env = ctx.obj["instance_configuration"]["environments"][next_environment] log("Inspecting new environment") if not force: if env.get("database", "") == "": log("Database has not been set up correctly.", lvl=critical) abort(EXIT_INSTALLATION_FAILED) if (not env.get("installed", False) or not env.get("tested", False) or not env.get("migrated", False)): log("Installation failed, cannot activate!", lvl=critical) abort(EXIT_INSTALLATION_FAILED) update_service(ctx, next_environment) ctx.obj["instance_configuration"]["environment"] = next_environment write_instance(ctx.obj["instance_configuration"]) # TODO: Effect reload of service # * Systemctl reload # * (Re)start service # * confirm correct operation # - if not, switch back to the other instance, maybe indicate a broken # state for next_environment # - if yes, Store instance configuration and terminate, we're done log("Turned instance over to", next_environment)
def _install_modules(ctx): """Internal function to install modules""" env = get_next_environment(ctx) log("Installing modules into", env, pretty=True) instance_configuration = ctx.obj["instance_configuration"] modules = instance_configuration["modules"] user = instance_configuration["user"] if len(modules) == 0: log("No modules defined for instance") return True for module in modules: log("Installing:", module, pretty=True) store_url = module[2] if module[0] == "store" else DEFAULT_STORE_URL result = _install_module(module[0], module[1], user=user, store_url=store_url) if result is False: log("Installation of module failed!", lvl=warn) else: module_name, module_version = result descriptor = { "name": module_name, "source": module[0], "url": module[1] } if store_url != DEFAULT_STORE_URL: descriptor["store_url"] = store_url instance_configuration["environments"][env]["modules"][ module_name] = descriptor write_instance(instance_configuration) return True
def _instance_selfsigned(instance_configuration): """Generates a snakeoil certificate that has only been self signed""" log("Generating self signed certificate/key combination") try: os.mkdir("/etc/ssl/certs/isomer") except FileExistsError: pass except PermissionError: log("Need root (e.g. via sudo) to generate ssl certificate") abort(1) try: from OpenSSL import crypto except ImportError: log("Need python3-openssl to do this.") abort(1) return def create_self_signed_cert(target): """Create a simple self signed SSL certificate""" key_file = os.path.join(target, "selfsigned.key") cert_file = os.path.join(target, "selfsigned.crt") combined_file = os.path.join(target, "selfsigned.pem") web_section = instance_configuration['web'] cert_conf = { k: v for k, v in web_section.items() if k.startswith("certificate") } log("Certificate data:", cert_conf, pretty=True) hostname = instance_configuration.get("web", {}).get("hostnames", gethostname()) if isinstance(hostname, list): hostname = hostname[0] # create a key pair k = crypto.PKey() k.generate_key(crypto.TYPE_RSA, 2048) if os.path.exists(cert_file): try: certificate = open(cert_file, "rb").read() old_cert = crypto.load_certificate(crypto.FILETYPE_PEM, certificate) serial = old_cert.get_serial_number() + 1 except (crypto.Error, OSError) as e: log( "Could not read old certificate to increment serial:", type(e), e, exc=True, lvl=warn, ) serial = 1 else: serial = 1 # create a self-signed certificate certificate = crypto.X509() certificate.get_subject().C = cert_conf.get("certificate_country", "EU") certificate.get_subject().ST = cert_conf.get("certificate_state", "Sol") certificate.get_subject().L = cert_conf.get("certificate_location", "Earth") # noinspection PyPep8 certificate.get_subject().O = cert_conf.get("certificate_issuer", "Unknown") certificate.get_subject().OU = cert_conf.get("certificate_unit", "Unknown") certificate.get_subject().CN = hostname certificate.set_serial_number(serial) certificate.gmtime_adj_notBefore(0) certificate.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) certificate.set_issuer(certificate.get_subject()) certificate.set_pubkey(k) certificate.sign(k, "sha512") open(key_file, "wt").write( str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII")) open(cert_file, "wt").write( str( crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), encoding="ASCII", )) open(combined_file, "wt").write( str( crypto.dump_certificate(crypto.FILETYPE_PEM, certificate), encoding="ASCII", ) + str(crypto.dump_privatekey(crypto.FILETYPE_PEM, k), encoding="ASCII")) location = os.path.join(get_etc_instance_path(), instance_configuration.get("name")) create_self_signed_cert(location) instance_configuration["web"]["key"] = os.path.join( location, "selfsigned.key") instance_configuration["web"]["certificate"] = os.path.join( location, "selfsigned.crt") write_instance(instance_configuration)
def _install_environment( ctx, source=None, url=None, import_file=None, no_sudo=False, force=False, release=None, upgrade=False, skip_modules=False, skip_data=False, skip_frontend=False, skip_test=False, skip_provisions=False, ): """Internal function to perform environment installation""" if url is None: url = source_url elif url[0] == '.': url = url.replace(".", os.getcwd(), 1) if url[0] == '/': url = os.path.abspath(url) instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instance_configuration"] next_environment = get_next_environment(ctx) set_instance(instance_name, next_environment) env = copy(instance_configuration["environments"][next_environment]) env["database"] = instance_name + "_" + next_environment env_path = get_path("lib", "") user = instance_configuration["user"] if no_sudo: user = None log("Installing new other environment for %s on %s from %s in %s" % (instance_name, next_environment, source, env_path)) try: result = get_isomer(source, url, env_path, upgrade=upgrade, sudo=user, release=release) if result is False: log("Getting Isomer failed", lvl=critical) abort(50011, ctx) except FileExistsError: if not force: log( "Isomer already present, please safely clear or " "inspect the environment before continuing! Use --force to ignore.", lvl=warn, ) abort(50012, ctx) else: log("Isomer already present, forcing through anyway.") try: repository = Repo(os.path.join(env_path, "repository")) log("Repo:", repository, lvl=debug) env["version"] = str(repository.git.describe()) except (exc.InvalidGitRepositoryError, exc.NoSuchPathError, exc.GitCommandError): env["version"] = version log( "Not running from a git repository; Using isomer.version:", version, lvl=warn, ) ctx.obj["instance_configuration"]["environments"][next_environment] = env # TODO: Does it make sense to early-write the configuration and then again later? write_instance(ctx.obj["instance_configuration"]) log("Creating virtual environment") success, result = run_process( env_path, [ "virtualenv", "-p", "/usr/bin/python3", "--system-site-packages", "venv" ], sudo=user, ) if not success: log(format_result(result), lvl=error) try: if _install_backend(ctx): log("Backend installed") env["installed"] = True if not skip_modules and _install_modules(ctx): log("Modules installed") # env['installed_modules'] = True if not skip_provisions and _install_provisions( ctx, import_file=import_file): log("Provisions installed") env["provisioned"] = True if not skip_data and _migrate(ctx): log("Data migrated") env["migrated"] = True if not skip_frontend and _install_frontend(ctx): log("Frontend installed") env["frontend"] = True if not skip_test and _check_environment(ctx): log("Environment tested") env["tested"] = True except Exception: log("Error during installation:", exc=True, lvl=critical) log("Environment status now:", env) ctx.obj["instance_configuration"]["environments"][next_environment] = env write_instance(ctx.obj["instance_configuration"])
def _clear_environment(ctx, force=False, clear_env=None, clear=False, no_archive=False): """Tests an environment for usage, then clears it :param ctx: Click Context :param force: Irrefutably destroy environment content :param clear_env: Environment to clear (Green/Blue) :param clear: Also destroy generated folders :param no_archive: Don't attempt to archive instance """ instance_name = ctx.obj["instance"] if clear_env is None: next_environment = get_next_environment(ctx) else: next_environment = clear_env log("Clearing environment:", next_environment) set_instance(instance_name, next_environment) # log('Testing', environment, 'for usage') env = ctx.obj["instance_configuration"]["environments"][next_environment] if not no_archive: if not (_archive(ctx, force) or force): log("Archival failed, stopping.") abort(5000) log("Clearing env:", env, lvl=debug) for item in locations: path = get_path(item, "") log("Clearing [%s]: %s" % (item, path), lvl=debug) try: shutil.rmtree(path) except FileNotFoundError: log("Path not found:", path, lvl=debug) except PermissionError: log("No permission to clear environment", lvl=error) return False if not clear: _create_folders(ctx) try: delete_database(ctx.obj["dbhost"], "%s_%s" % (instance_name, next_environment), force=True) except pymongo.errors.ServerSelectionTimeoutError: log("No database available") except Exception as e: log("Could not delete database:", e, lvl=warn, exc=True) ctx.obj["instance_configuration"]["environments"][ next_environment] = environment_template write_instance(ctx.obj["instance_configuration"]) return True