def _launch_service(ctx): """Actually enable and launch newly set up environment""" instance_name = ctx.obj["instance"] service_name = "isomer-" + instance_name + ".service" success, result = run_process("/", ["systemctl", "enable", service_name], sudo="root") if not success: log("Error activating service:", format_result(result), pretty=True, lvl=error) abort(5000) log("Launching service") success, result = run_process("/", ["systemctl", "start", service_name], sudo="root") if not success: log("Error activating service:", format_result(result), pretty=True, lvl=error) abort(5000) return True
def _add_system_user(use_sudo=False): """instance Isomer system user (isomer.isomer)""" command = [ "/usr/sbin/adduser", "--system", "--quiet", "--home", "/var/run/isomer", "--group", "--disabled-password", "--disabled-login", "isomer", ] success, output = run_process("/", command, sudo=use_sudo) if success is False: log("Error adding system user:"******"/usr/sbin/adduser", "isomer", "dialout"] success, output = run_process("/", command, sudo=use_sudo) if success is False: log("Error adding system user to dialout group:", lvl=error) log(output, lvl=error) time.sleep(2)
def _install_backend(ctx): """Installs the backend into an environment""" instance_name = ctx.obj["instance"] env = get_next_environment(ctx) set_instance(instance_name, env) log("Installing backend on", env, lvl=debug) env_path = get_path("lib", "") user = ctx.obj["instance_configuration"]["user"] success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "setup.py", "develop" ], sudo=user, ) if not success: output = str(result) if "was unable to detect version" in output: log( "Installing from dirty repository. This might result in dependency " "version problems!", lvl=hilight, ) else: log( "Something unexpected happened during backend installation:\n", result, lvl=hilight, ) # TODO: Another fault might be an unclean package path. # But i forgot the log message to check for. # log('This might be a problem due to unclean installations of Python' # ' libraries. Please check your path.') log("Installing requirements") success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "pip3"), "install", "-r", "requirements.txt", ], sudo=user, ) if not success: log(format_result(result), lvl=error) return True
def cmdmap(xdot): """Generates a command map""" # TODO: Integrate the output into documentation from copy import copy def generate_command_graph(command, map_output, groups=None, depth=0): """Generate a strict digraph (as indented representation) of all known subgroups and commands""" if groups is None: groups = [] if "commands" in command.__dict__: if len(groups) > 0: if xdot: line = ' "%s" -> "%s" [weight=1.0];\n' % ( groups[-1], command.name, ) else: line = " " * (depth - 1) + "%s %s\n" % (groups[-1], command.name) map_output.append(line) for item in command.commands.values(): subgroups = copy(groups) subgroups.append(command.name) generate_command_graph(item, map_output, subgroups, depth + 1) else: if xdot: line = ' "%s" -> "%s" [weight=%1.1f];\n' % ( groups[-1], command.name, len(groups), ) else: line = " " * (len(groups) - 3 + depth) + "%s %s\n" % ( groups[-1], command.name, ) map_output.append(line) output = [] generate_command_graph(cli, output) output = [line.replace("cli", "isomer") for line in output] if xdot: with open("iso-tool.dot", "w") as f: f.write("strict digraph {\n") f.writelines(sorted(output)) f.write("}") run_process(".", ["xdot", "iso-tool.dot"]) else: print("".join(output))
def remote(ctx, name, install, platform, source, url, existing): """Remote instance control (Work in Progress!)""" ctx.obj["remote"] = name ctx.obj["platform"] = platform ctx.obj["source"] = source ctx.obj["url"] = url ctx.obj["existing"] = existing if ctx.invoked_subcommand == "add": return remotes = ctx.obj["remotes"] = load_remotes() if ctx.invoked_subcommand == "list": return # log('Remote configurations:', remotes, pretty=True) host_config = remotes.get(name, None) if host_config is None: log("Cannot proceed, remote unknown", lvl=error) abort(5000) ctx.obj["host_config"] = host_config if platform is None: platform = ctx.obj["host_config"].get("platform", "debian") ctx.obj["platform"] = platform spur_config = dict(host_config["login"]) if spur_config["private_key_file"] == "": spur_config.pop("private_key_file") if spur_config["port"] != 22: log( "Warning! Using any port other than 22 is not supported right now.", lvl=warn, ) spur_config.pop("port") shell = spur.SshShell(**spur_config) if install: success, result = run_process("/", ["iso", "info"], shell) if success: log("Isomer version on remote:", format_result(result)) else: log('Use "remote install" for now') # if existing is None: # get_isomer(source, url, '/root', shell=shell) # destination = '/' + host_config['login']['username'] + '/repository' # else: # destination = existing # install_isomer(platform, host_config.get('use_sudo', True), shell, cwd=destination) ctx.obj["shell"] = shell
def get_git_releases(repository_path, fetch=False): """Get release data from a git repository. Optionally, fetch from upstream first""" log("Getting git tags from", repository_path) releases = {} repo = git.Repo(repository_path) if fetch is True: log("Fetching origin tags") success, result = run_process( repository_path, ["git", "fetch", "--tags", "-v", "origin"], ) if not success: log(result, lvl=error) #origin = repo.remotes["origin"] #log("Origin:", list(origin.urls), pretty=True) #origin.fetch(tags=True) tags = repo.tags log("Raw tags:", tags, pretty=True, lvl=debug) for item in tags: releases[str(item)] = item return releases
def test(ctx): """Run and return info command on a remote""" shell = ctx.obj["shell"] username = ctx.obj["host_config"]["login"]["username"] success, result = run_process(get_remote_home(username), ["iso", "-nc", "version"], shell=shell) log(success, "\n", format_result(result), pretty=True)
def get_module_info(directory): log("Getting name") success, result = run_process(directory, ["python3", "setup.py", "--name"], sudo=user) if not success: log(format_result(result), pretty=True, lvl=error) return False package_name = str(result.output, encoding="utf8").rstrip("\n") log("Getting version") success, result = run_process(directory, ["python3", "setup.py", "--version"], sudo=user) if not success: log(format_result(result), pretty=True, lvl=error) return False package_version = str(result.output, encoding="utf8").rstrip("\n") log("Package name:", package_name, "version:", package_version) return package_name, package_version
def _create_system_folders(use_sudo=False): target_paths = [ "/var/www/challenges", # For LetsEncrypt acme certificate challenges "/var/backups/isomer", "/var/log/isomer", "/var/run/isomer", ] for item in locations: target_paths.append(get_path(item, "")) target_paths.append(get_log_path()) for item in target_paths: run_process("/", ["sudo", "mkdir", "-p", item], sudo=use_sudo) run_process("/", ["sudo", "chown", "isomer", item], sudo=use_sudo) # TODO: The group/ownership should be assigned per instance.user/group run_process("/", ["sudo", "chgrp", "isomer", "/var/log/isomer"], sudo=use_sudo) run_process("/", ["sudo", "chmod", "g+w", "/var/log/isomer"], sudo=use_sudo)
def _instance_letsencrypt(instance_configuration): hostnames = instance_configuration.get("web", {}).get("hostnames", False) hostnames = hostnames.replace(" ", "") if not hostnames or hostnames == "localhost": log( "Please configure the public fully qualified domain names of this instance.\n" "Use 'iso instance set web_hostnames your.hostname.tld' to do that.\n" "You can add multiple names by separating them with commas.", lvl=error, ) abort(50031) contact = instance_configuration.get("contact", False) if not contact: log( "You need to specify a contact mail address for this instance to generate certificates.\n" "Use 'iso instance set contact [email protected]' to do that.", lvl=error, ) abort(50032) success, result = run_process( "/", [ "certbot", "--nginx", "certonly", "-m", contact, "-d", hostnames, "--agree-tos", "-n", ], ) if not success: log( "Error getting certificate:", format_result(result), pretty=True, lvl=error, ) abort(50033)
def backup(ctx, backup_instance, fetch, target): """Backup a remote""" log("Backing up %s on remote %s" % (backup_instance, ctx.obj["remote"])) shell = ctx.obj["shell"] args = [ "iso", "-nc", "--clog", "10", "-i", backup_instance, "-e", "current", "environment", "archive", ] log(args) success, result = run_process(get_remote_home( ctx.obj["host_config"]["login"]["username"]), args, shell=shell) if not success or b"Archived to" not in result.output: log("Execution error:", format_result(result), pretty=True, lvl=error) else: log("Local backup created") if fetch: full_path = result.split("'")[1] filename = os.path.basename(full_path) with shell.open(full_path, "r") as input_file: with open(os.path.join(target, filename), "w") as output_file: output = input_file.read() output_file.write(output) log("Backup downloaded. Size:", len(output))
def _install_provisions(ctx, import_file=None, skip_provisions=False): """Load provisions into database""" instance_configuration = ctx.obj["instance_configuration"] env = get_next_environment(ctx) env_path = get_path("lib", "") log("Installing provisioning data") if not skip_provisions: success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "./iso", "-nc", "--flog", "5", "--config-path", get_etc_path(), "-i", instance_configuration["name"], "-e", env, "install", "provisions", ], # Note: no sudo necessary as long as we do not enforce # authentication on databases ) if not success: log("Could not provision data:", lvl=error) log(format_result(result), lvl=error) return False if import_file is not None: log("Importing backup") log(ctx.obj, pretty=True) host, port = ctx.obj["dbhost"].split(":") load(host, int(port), ctx.obj["dbname"], import_file) return True
def command(ctx, commands): """Execute a remote command""" log("Executing commands %s on remote %s" % (commands, ctx.obj["remote"])) shell = ctx.obj["shell"] args = ["iso"] + list(commands) log(args) success, result = run_process(get_remote_home( ctx.obj["host_config"]["login"]["username"]), args, shell=shell) if not success: log("Execution error:", format_result(result), pretty=True, lvl=error) else: log("Success:") log(format_result(result))
def _install_frontend(ctx): """Install and build the frontend""" env = get_next_environment(ctx) env_path = get_path("lib", "") instance_configuration = ctx.obj["instance_configuration"] user = instance_configuration["user"] log("Building frontend") success, result = run_process( os.path.join(env_path, "repository"), [ os.path.join(env_path, "venv", "bin", "python3"), "./iso", "-nc", "--config-path", get_etc_path(), "--prefix-path", get_prefix_path(), "-i", instance_configuration["name"], "-e", env, "--clog", "10", "install", "frontend", "--rebuild", ], sudo=user, ) if not success: log(format_result(result), lvl=error) return False return True
def _install_environment( ctx, source=None, url=None, import_file=None, no_sudo=False, force=False, release=None, upgrade=False, skip_modules=False, skip_data=False, skip_frontend=False, skip_test=False, skip_provisions=False, ): """Internal function to perform environment installation""" if url is None: url = source_url elif url[0] == '.': url = url.replace(".", os.getcwd(), 1) if url[0] == '/': url = os.path.abspath(url) instance_name = ctx.obj["instance"] instance_configuration = ctx.obj["instance_configuration"] next_environment = get_next_environment(ctx) set_instance(instance_name, next_environment) env = copy(instance_configuration["environments"][next_environment]) env["database"] = instance_name + "_" + next_environment env_path = get_path("lib", "") user = instance_configuration["user"] if no_sudo: user = None log("Installing new other environment for %s on %s from %s in %s" % (instance_name, next_environment, source, env_path)) try: result = get_isomer(source, url, env_path, upgrade=upgrade, sudo=user, release=release) if result is False: log("Getting Isomer failed", lvl=critical) abort(50011, ctx) except FileExistsError: if not force: log( "Isomer already present, please safely clear or " "inspect the environment before continuing! Use --force to ignore.", lvl=warn, ) abort(50012, ctx) else: log("Isomer already present, forcing through anyway.") try: repository = Repo(os.path.join(env_path, "repository")) log("Repo:", repository, lvl=debug) env["version"] = str(repository.git.describe()) except (exc.InvalidGitRepositoryError, exc.NoSuchPathError, exc.GitCommandError): env["version"] = version log( "Not running from a git repository; Using isomer.version:", version, lvl=warn, ) ctx.obj["instance_configuration"]["environments"][next_environment] = env # TODO: Does it make sense to early-write the configuration and then again later? write_instance(ctx.obj["instance_configuration"]) log("Creating virtual environment") success, result = run_process( env_path, [ "virtualenv", "-p", "/usr/bin/python3", "--system-site-packages", "venv" ], sudo=user, ) if not success: log(format_result(result), lvl=error) try: if _install_backend(ctx): log("Backend installed") env["installed"] = True if not skip_modules and _install_modules(ctx): log("Modules installed") # env['installed_modules'] = True if not skip_provisions and _install_provisions( ctx, import_file=import_file): log("Provisions installed") env["provisioned"] = True if not skip_data and _migrate(ctx): log("Data migrated") env["migrated"] = True if not skip_frontend and _install_frontend(ctx): log("Frontend installed") env["frontend"] = True if not skip_test and _check_environment(ctx): log("Environment tested") env["tested"] = True except Exception: log("Error during installation:", exc=True, lvl=critical) log("Environment status now:", env) ctx.obj["instance_configuration"]["environments"][next_environment] = env write_instance(ctx.obj["instance_configuration"])
def install_remote(ctx, archive, setup): """Installs Isomer (Management) on a remote host""" shell = ctx.obj["shell"] platform = ctx.obj["platform"] host_config = ctx.obj["host_config"] use_sudo = host_config["use_sudo"] username = host_config["login"]["username"] existing = ctx.obj["existing"] remote_home = get_remote_home(username) target = os.path.join(remote_home, "isomer") log(remote_home) if shell is None: log("Remote was not configured properly.", lvl=warn) abort(5000) if archive: log("Renaming remote isomer copy") success, result = run_process( remote_home, ["mv", target, os.path.join(remote_home, "isomer_" + std_now())], shell=shell, ) if not success: log("Could not rename remote copy:", result, pretty=True, lvl=error) abort(5000) if existing is None: url = ctx.obj["url"] if url is None: url = host_config.get("url", None) source = ctx.obj["source"] if source is None: source = host_config.get("source", None) if url is None or source is None: log('Need a source and url to install. Try "iso remote --help".') abort(5000) get_isomer(source, url, target, upgrade=ctx.obj["upgrade"], shell=shell) destination = os.path.join(remote_home, "isomer") else: destination = existing install_isomer(platform, use_sudo, shell=shell, cwd=destination) if setup: log("Setting up system user and paths") success, result = run_process(remote_home, ["iso", "system", "all"]) if not success: log( "Setting up system failed:", format_result(result), pretty=True, lvl=error, )
def upload_key(ctx, accept): """Upload a remote key to a user account on a remote machine""" login_config = dict(ctx.obj["host_config"]["login"]) if login_config["password"] == "": login_config["password"] = getpass.getpass() with open(login_config["private_key_file"] + ".pub") as f: key = f.read() username = login_config["username"] if accept: host_key_flag = spur.ssh.MissingHostKey.warn else: host_key_flag = spur.ssh.MissingHostKey.raise_error shell = spur.SshShell( hostname=login_config["hostname"], username=login_config["username"], password=login_config["password"], missing_host_key=host_key_flag, ) try: with shell.open("/home/" + username + "/.ssh/authorized_keys", "r") as f: result = f.read() except spur.ssh.ConnectionError as e: log("SSH Connection error:\n", e, lvl=error) log("Host not in known hosts or other problem. Use --accept to add to known_hosts." ) abort(50071) except FileNotFoundError as e: log("No authorized key file yet, creating") success, result = run_process( "/home/" + username, ["/bin/mkdir", "/home/" + username + "/.ssh"], shell=shell, ) if not success: log( "Error creating .ssh directory:", e, format_result(result), pretty=True, lvl=error, ) success, result = run_process( "/home/" + login_config["username"], ["/usr/bin/touch", "/home/" + username + "/.ssh/authorized_keys"], shell=shell, ) if not success: log( "Error creating authorized hosts file:", e, format_result(result).output, lvl=error, ) result = "" if key not in result: log("Key not yet authorized - adding") with shell.open("/home/" + username + "/.ssh/authorized_keys", "a") as f: f.write(key) else: log("Key is already authorized.", lvl=warn) log("Uploaded key")
def _create_nginx_config(ctx): """instance nginx configuration""" # TODO: Specify template url very precisely. Currently one needs to be in # the repository root instance_name = ctx.obj["instance"] config = ctx.obj["instance_configuration"] current_env = config["environment"] env = config["environments"][current_env] dbhost = config["database"]["host"] dbname = env["database"] hostnames = ctx.obj.get("web", {}).get("hostnames", None) if hostnames is None: hostnames = config.get("web", {}).get("hostnames", None) if hostnames is None: try: configuration = _get_system_configuration(dbhost, dbname) hostnames = configuration.hostname except Exception as e: log("Exception:", e, type(e), exc=True, lvl=error) log( """Could not determine public fully qualified hostname! Check systemconfig (see db view and db modify commands) or specify manually with --hostname host.domain.tld Using 'localhost' for now""", lvl=warn, ) hostnames = "localhost" port = config["web"]["port"] address = config["web"]["address"] log("Creating nginx configuration for %s:%i using %s@%s" % (hostnames, port, dbname, dbhost)) definitions = { "server_public_name": hostnames.replace(",", " "), "ssl_certificate": config["web"]["certificate"], "ssl_key": config["web"]["key"], "host_url": "http://%s:%i/" % (address, port), "instance": instance_name, "environment": current_env, } if distribution == "DEBIAN": configuration_file = "/etc/nginx/sites-available/isomer.%s.conf" % instance_name configuration_link = "/etc/nginx/sites-enabled/isomer.%s.conf" % instance_name elif distribution == "ARCH": configuration_file = "/etc/nginx/nginx.conf" configuration_link = None else: log( "Unsure how to proceed, you may need to specify your " "distribution", lvl=error, ) return log("Writing nginx Isomer site definition") write_template(nginx_template, configuration_file, definitions) if configuration_link is not None: log("Enabling nginx Isomer site (symlink)") if not os.path.exists(configuration_link): os.symlink(configuration_file, configuration_link) if os.path.exists("/bin/systemctl"): log("Restarting nginx service") run_process("/", ["systemctl", "restart", "nginx.service"], sudo="root") else: log("No systemctl found, not restarting nginx")
def _install_module(source, url, store_url=DEFAULT_STORE_URL, auth=None, force=False, user=None): """Actually installs a module into an environment""" package_name = package_version = success = output = "" def get_module_info(directory): log("Getting name") success, result = run_process(directory, ["python3", "setup.py", "--name"], sudo=user) if not success: log(format_result(result), pretty=True, lvl=error) return False package_name = str(result.output, encoding="utf8").rstrip("\n") log("Getting version") success, result = run_process(directory, ["python3", "setup.py", "--version"], sudo=user) if not success: log(format_result(result), pretty=True, lvl=error) return False package_version = str(result.output, encoding="utf8").rstrip("\n") log("Package name:", package_name, "version:", package_version) return package_name, package_version if source == "develop": log("Installing module for development") success, output = run_process( url, [ os.path.join(get_path("lib", "venv"), "bin", "python3"), "setup.py", "develop", ], sudo=user, ) if not success: log(output, lvl=verbose) return False else: return get_module_info(url) module_path = get_path("lib", "modules", ensure=True) module_info = False if source not in ("git", "link", "copy", "store"): abort(EXIT_INVALID_SOURCE) uuid = std_uuid() temporary_path = os.path.join(module_path, "%s" % uuid) log("Installing module: %s [%s]" % (url, source)) if source in ("link", "copy") and url.startswith("/"): absolute_path = url else: absolute_path = os.path.abspath(url) if source == "git": log("Cloning repository from", url) success, output = run_process(module_path, ["git", "clone", url, temporary_path], sudo=user) if not success: log("Error:", output, lvl=error) elif source == "link": log("Linking repository from", absolute_path) success, output = run_process( module_path, ["ln", "-s", absolute_path, temporary_path], sudo=user) if not success: log("Error:", output, lvl=error) elif source == "copy": log("Copying repository from", absolute_path) success, output = run_process( module_path, ["cp", "-a", absolute_path, temporary_path], sudo=user) if not success: log("Error:", output, lvl=error) elif source == "store": log("Installing wheel from store", absolute_path) log(store_url, auth) store = get_store(store_url, auth) if url not in store["packages"]: abort(EXIT_STORE_PACKAGE_NOT_FOUND) meta = store["packages"][url] package_name = meta['name'] package_version = meta['version'] venv_path = os.path.join(get_path("lib", "venv"), "bin") success, output = run_process( venv_path, ["pip3", "install", "--extra-index-url", store_url, package_name]) if source != "store": module_info = get_module_info(temporary_path) if module_info is False: log("Could not get name and version information from module.", lvl=error) return False package_name, package_version = module_info final_path = os.path.join(module_path, package_name) if os.path.exists(final_path): log("Module exists.", lvl=warn) if force: log("Removing previous version.") success, result = run_process(module_path, ["rm", "-rf", final_path], sudo=user) if not success: log("Could not remove previous version!", lvl=error) abort(50000) else: log("Not overwriting previous version without --force", lvl=error) abort(50000) log("Renaming to", final_path) os.rename(temporary_path, final_path) log("Installing module") success, output = run_process( final_path, [ os.path.join(get_path("lib", "venv"), "bin", "python3"), "setup.py", "develop", ], sudo=user, ) if not success: log(output, lvl=verbose) return False else: return package_name, package_version
def export_schemata(ctx, output_path, output_format, no_entity_mode, include_meta, schemata): """Utility function for exporting known schemata to various formats Exporting to typescript requires the "json-schema-to-typescript" tool. You can install it via: npm -g install json-schema-to-typescript """ # TODO: This one is rather ugly to edit, should the need arise.. banner = ( "/* tslint:disable */\n/**\n* This file was automatically generated " "by Isomer's command line tool using:\n" " * 'iso dev export-schemata -f typescript' " "- using json-schema-to-typescript.\n" " * DO NOT MODIFY IT BY HAND. Instead, modify the source isomer object " "file,\n* and run the iso tool schemata exporter again, to regenerate this file.\n*/" ) # TODO: This should be employable to automatically generate # typescript definitions inside a modules frontend part as part # of the development cycle. from isomer import database, schemastore database.initialize(ctx.obj["dbhost"], ctx.obj["dbname"], ignore_fail=True) if len(schemata) == 0: schemata = database.objectmodels.keys() if output_path is not None: stdout = False if not os.path.exists(output_path): abort("Output Path doesn't exist.") else: stdout = True for item in schemata: if item not in schemastore.schemastore: log("Schema not registered:", item, lvl=warn) continue schema = schemastore.schemastore[item]["schema"] if not include_meta: if "perms" in schema["properties"]: del schema["properties"]["perms"] if "roles_create" in schema: del schema["roles_create"] if "required" in schema: del schema["required"] if output_format == "jsonschema": log("Generating json schema of", item) if stdout: print(json.dumps(schema, indent=4)) else: with open(os.path.join(output_path, item + ".json"), "w") as f: json.dump(schema, f, indent=4) elif output_format == "typescript": log("Generating typescript annotations of", item) # TODO: Fix typing issues here and esp. in run_process success, result = run_process( output_path, [ "json2ts", "--bannerComment", banner, ], stdin=json.dumps(schema).encode("utf-8"), ) typescript = result.output.decode("utf-8") if no_entity_mode is False: typescript = ( "import { Entity, Key } from '@briebug/ngrx-auto-entity';\n" + typescript) typescript = typescript.replace("uuid", "@Key uuid") typescript = typescript.replace( "export interface", "@Entity({modelName: '%s'})\n" "export class" % item, ) if stdout: print(typescript) else: with open(os.path.join(output_path, item + ".ts"), "w") as f: f.write(typescript) finish(ctx)