def install_frontend(ctx):
"""Install frontend into an environment"""
next_environment = get_next_environment(ctx)
set_instance(ctx.obj["instance"], next_environment)
_install_frontend(ctx)
finish(ctx)
def _install_backend(ctx):
"""Installs the backend into an environment"""
instance_name = ctx.obj["instance"]
env = get_next_environment(ctx)
set_instance(instance_name, env)
log("Installing backend on", env, lvl=debug)
env_path = get_path("lib", "")
user = ctx.obj["instance_configuration"]["user"]
success, result = run_process(
os.path.join(env_path, "repository"),
[
os.path.join(env_path, "venv", "bin", "python3"), "setup.py",
"develop"
],
sudo=user,
)
if not success:
output = str(result)
if "was unable to detect version" in output:
log(
"Installing from dirty repository. This might result in dependency "
"version problems!",
lvl=hilight,
)
else:
log(
"Something unexpected happened during backend installation:\n",
result,
lvl=hilight,
)
# TODO: Another fault might be an unclean package path.
# But i forgot the log message to check for.
# log('This might be a problem due to unclean installations of Python'
# ' libraries. Please check your path.')
log("Installing requirements")
success, result = run_process(
os.path.join(env_path, "repository"),
[
os.path.join(env_path, "venv", "bin", "pip3"),
"install",
"-r",
"requirements.txt",
],
sudo=user,
)
if not success:
log(format_result(result), lvl=error)
return True
def install(self, event):
"""Installs a new Isomer package on the running instance"""
self.log("Installing package:", event.data)
instance_config = self.context.obj["instance_configuration"]
# repository = get_path("lib", "repository")
environments = instance_config["environments"]
active = instance_config["environment"]
next_environment = get_next_environment(self.context)
self.fireEvent(
notify_restart_required(reason="Module installed: " + event.data))
def cert(ctx, selfsigned):
"""instance a local SSL certificate"""
instance_configuration = ctx.obj["instance_configuration"]
instance_name = ctx.obj["instance"]
next_environment = get_next_environment(ctx)
set_instance(instance_name, next_environment)
if selfsigned:
_instance_selfsigned(instance_configuration)
else:
log("This is work in progress")
abort(55555)
_instance_letsencrypt(instance_configuration)
finish(ctx)
def install_environment_modules(ctx, source, force, urls, store_url):
"""Add and install a module only to a single environment
Note: This does not modify the instance configuration, so this will not
be permanent during upgrades etc.
"""
instance_name = ctx.obj["instance"]
instance_configuration = ctx.obj["instances"][instance_name]
next_environment = get_next_environment(ctx)
user = instance_configuration["user"]
installed = instance_configuration["environments"][next_environment][
"installed"]
if not installed:
log("Please install the '%s' environment first." % next_environment,
lvl=error)
abort(50000)
set_instance(instance_name, next_environment)
for url in urls:
result = _install_module(source,
url,
force=force,
user=user,
store_url=store_url)
if result is False:
log("Installation failed!", lvl=error)
abort(50000)
package_name, package_version = result
descriptor = {"version": package_version, "source": source, "url": url}
if store_url != DEFAULT_STORE_URL:
descriptor["store_url"] = store_url
instance_configuration["environments"][next_environment]["modules"][
package_name] = descriptor
write_instance(instance_configuration)
finish(ctx)
def set_parameter(ctx, parameter, value, force):
"""Set a configuration parameter of an environment"""
# TODO: Generalize and improve this.
# - To get less code redundancy (this is also in instance.py)
# - To be able to set lists, dicts and other types correctly
log("Setting %s to %s" % (parameter, value))
next_environment = get_next_environment(ctx)
environment_configuration = ctx.obj["instance_configuration"][
'environments'][next_environment]
defaults = environment_template
converted_value = None
try:
parameter_type = type(defaults[parameter])
log(parameter_type, pretty=True, lvl=debug)
if parameter_type == tomlkit.items.Integer:
converted_value = int(value)
elif parameter_type == bool:
converted_value = value.upper() == "TRUE"
else:
converted_value = value
except KeyError:
log("Available parameters:", sorted(list(defaults.keys())))
abort(EXIT_INVALID_PARAMETER)
if converted_value is None:
log("Converted value was None! Recheck the new config!", lvl=warn)
environment_configuration[parameter] = converted_value
log("New config:", environment_configuration, pretty=True, lvl=debug)
ctx.obj["instances"][ctx.obj["instance"]]['environments'][
next_environment] = environment_configuration
if valid_configuration(ctx) or force:
write_instance(ctx.obj["instances"][ctx.obj["instance"]])
finish(ctx)
else:
log("New configuration would not be valid", lvl=critical)
abort(EXIT_INVALID_CONFIGURATION)
def _install_provisions(ctx, import_file=None, skip_provisions=False):
"""Load provisions into database"""
instance_configuration = ctx.obj["instance_configuration"]
env = get_next_environment(ctx)
env_path = get_path("lib", "")
log("Installing provisioning data")
if not skip_provisions:
success, result = run_process(
os.path.join(env_path, "repository"),
[
os.path.join(env_path, "venv", "bin", "python3"),
"./iso",
"-nc",
"--flog",
"5",
"--config-path",
get_etc_path(),
"-i",
instance_configuration["name"],
"-e",
env,
"install",
"provisions",
],
# Note: no sudo necessary as long as we do not enforce
# authentication on databases
)
if not success:
log("Could not provision data:", lvl=error)
log(format_result(result), lvl=error)
return False
if import_file is not None:
log("Importing backup")
log(ctx.obj, pretty=True)
host, port = ctx.obj["dbhost"].split(":")
load(host, int(port), ctx.obj["dbname"], import_file)
return True
def install_instance_modules(ctx, source, urls, install_env, force, store_url):
"""Add (and optionally immediately install) modules for an instance.
This will add them to the instance's configuration, so they will be upgraded as well
as reinstalled on other environment changes.
If you're installing from a store, you can specify a custom store URL with the
--store-url argument.
"""
instance_name = ctx.obj["instance"]
instance_configuration = ctx.obj["instances"][instance_name]
for url in urls:
descriptor = [source, url]
if store_url != DEFAULT_STORE_URL:
descriptor.append(store_url)
if descriptor not in instance_configuration["modules"]:
instance_configuration["modules"].append(descriptor)
elif not force:
log("Module %s is already installed. Use --force to install anyway."
% url)
abort(50000)
write_instance(instance_configuration)
if install_env is True:
next_environment = get_next_environment(ctx)
environments = instance_configuration['environments']
if environments[next_environment]["installed"] is False:
log("Environment %s is not installed, cannot install modules." %
next_environment,
lvl=warn)
abort(50600)
return
del ctx.params["install_env"]
ctx.forward(install_environment_modules)
finish(ctx)
def _install_frontend(ctx):
"""Install and build the frontend"""
env = get_next_environment(ctx)
env_path = get_path("lib", "")
instance_configuration = ctx.obj["instance_configuration"]
user = instance_configuration["user"]
log("Building frontend")
success, result = run_process(
os.path.join(env_path, "repository"),
[
os.path.join(env_path, "venv", "bin", "python3"),
"./iso",
"-nc",
"--config-path",
get_etc_path(),
"--prefix-path",
get_prefix_path(),
"-i",
instance_configuration["name"],
"-e",
env,
"--clog",
"10",
"install",
"frontend",
"--rebuild",
],
sudo=user,
)
if not success:
log(format_result(result), lvl=error)
return False
return True
def _turnover(ctx, force):
"""Internal turnover operation"""
# if ctx.obj['acting_environment'] is not None:
# next_environment = ctx.obj['acting_environment']
# else:
next_environment = get_next_environment(ctx)
log("Activating environment:", next_environment)
env = ctx.obj["instance_configuration"]["environments"][next_environment]
log("Inspecting new environment")
if not force:
if env.get("database", "") == "":
log("Database has not been set up correctly.", lvl=critical)
abort(EXIT_INSTALLATION_FAILED)
if (not env.get("installed", False) or not env.get("tested", False)
or not env.get("migrated", False)):
log("Installation failed, cannot activate!", lvl=critical)
abort(EXIT_INSTALLATION_FAILED)
update_service(ctx, next_environment)
ctx.obj["instance_configuration"]["environment"] = next_environment
write_instance(ctx.obj["instance_configuration"])
# TODO: Effect reload of service
# * Systemctl reload
# * (Re)start service
# * confirm correct operation
# - if not, switch back to the other instance, maybe indicate a broken
# state for next_environment
# - if yes, Store instance configuration and terminate, we're done
log("Turned instance over to", next_environment)
def _install_modules(ctx):
"""Internal function to install modules"""
env = get_next_environment(ctx)
log("Installing modules into", env, pretty=True)
instance_configuration = ctx.obj["instance_configuration"]
modules = instance_configuration["modules"]
user = instance_configuration["user"]
if len(modules) == 0:
log("No modules defined for instance")
return True
for module in modules:
log("Installing:", module, pretty=True)
store_url = module[2] if module[0] == "store" else DEFAULT_STORE_URL
result = _install_module(module[0],
module[1],
user=user,
store_url=store_url)
if result is False:
log("Installation of module failed!", lvl=warn)
else:
module_name, module_version = result
descriptor = {
"name": module_name,
"source": module[0],
"url": module[1]
}
if store_url != DEFAULT_STORE_URL:
descriptor["store_url"] = store_url
instance_configuration["environments"][env]["modules"][
module_name] = descriptor
write_instance(instance_configuration)
return True
def upgrade(ctx, release, upgrade_modules, restart, handle_cache, source, url):
"""Upgrades an instance on its other environment and turns over on success.
\b
1. Test if other environment is empty
1.1. No - archive and clear it
2. Copy current environment to other environment
3. Clear old bits (venv, frontend)
4. Fetch updates in other environment repository
5. Select a release
6. Checkout that release and its submodules
7. Install release
8. Copy database
9. Migrate data (WiP)
10. Turnover
"""
instance_config = ctx.obj["instance_configuration"]
repository = get_path("lib", "repository")
installation_source = source if source is not None else instance_config[
'source']
installation_url = url if url is not None else instance_config['url']
environments = instance_config["environments"]
active = instance_config["environment"]
next_environment = get_next_environment(ctx)
if environments[next_environment]["installed"] is True:
_clear_environment(ctx, clear_env=next_environment)
source_paths = [
get_path("lib", "", environment=active),
get_path("local", "", environment=active)
]
destination_paths = [
get_path("lib", "", environment=next_environment),
get_path("local", "", environment=next_environment)
]
log(source_paths, destination_paths, pretty=True)
for source, destination in zip(source_paths, destination_paths):
log("Copying to new environment:", source, destination)
copy_directory_tree(source, destination)
if handle_cache != "ignore":
log("Handling cache")
move = handle_cache == "move"
copy_directory_tree(get_path("cache", "", environment=active),
get_path("cache", "",
environment=next_environment),
move=move)
rmtree(get_path("lib", "venv"), ignore_errors=True)
# TODO: This potentially leaves frontend-dev:
rmtree(get_path("lib", "frontend"), ignore_errors=True)
releases = _get_versions(ctx,
source=installation_source,
url=installation_url,
fetch=True)
releases_keys = sorted_alphanumerical(releases.keys())
if release is None:
release = releases_keys[-1]
else:
if release not in releases_keys:
log("Unknown release. Maybe try a different release or source.")
abort(50100)
log("Choosing release", release)
_install_environment(ctx,
installation_source,
installation_url,
upgrade=True,
release=release)
new_database_name = instance_config["name"] + "_" + next_environment
copy_database(ctx.obj["dbhost"], active['database'], new_database_name)
apply_migrations(ctx)
finish(ctx)
def _install_environment(
ctx,
source=None,
url=None,
import_file=None,
no_sudo=False,
force=False,
release=None,
upgrade=False,
skip_modules=False,
skip_data=False,
skip_frontend=False,
skip_test=False,
skip_provisions=False,
):
"""Internal function to perform environment installation"""
if url is None:
url = source_url
elif url[0] == '.':
url = url.replace(".", os.getcwd(), 1)
if url[0] == '/':
url = os.path.abspath(url)
instance_name = ctx.obj["instance"]
instance_configuration = ctx.obj["instance_configuration"]
next_environment = get_next_environment(ctx)
set_instance(instance_name, next_environment)
env = copy(instance_configuration["environments"][next_environment])
env["database"] = instance_name + "_" + next_environment
env_path = get_path("lib", "")
user = instance_configuration["user"]
if no_sudo:
user = None
log("Installing new other environment for %s on %s from %s in %s" %
(instance_name, next_environment, source, env_path))
try:
result = get_isomer(source,
url,
env_path,
upgrade=upgrade,
sudo=user,
release=release)
if result is False:
log("Getting Isomer failed", lvl=critical)
abort(50011, ctx)
except FileExistsError:
if not force:
log(
"Isomer already present, please safely clear or "
"inspect the environment before continuing! Use --force to ignore.",
lvl=warn,
)
abort(50012, ctx)
else:
log("Isomer already present, forcing through anyway.")
try:
repository = Repo(os.path.join(env_path, "repository"))
log("Repo:", repository, lvl=debug)
env["version"] = str(repository.git.describe())
except (exc.InvalidGitRepositoryError, exc.NoSuchPathError,
exc.GitCommandError):
env["version"] = version
log(
"Not running from a git repository; Using isomer.version:",
version,
lvl=warn,
)
ctx.obj["instance_configuration"]["environments"][next_environment] = env
# TODO: Does it make sense to early-write the configuration and then again later?
write_instance(ctx.obj["instance_configuration"])
log("Creating virtual environment")
success, result = run_process(
env_path,
[
"virtualenv", "-p", "/usr/bin/python3", "--system-site-packages",
"venv"
],
sudo=user,
)
if not success:
log(format_result(result), lvl=error)
try:
if _install_backend(ctx):
log("Backend installed")
env["installed"] = True
if not skip_modules and _install_modules(ctx):
log("Modules installed")
# env['installed_modules'] = True
if not skip_provisions and _install_provisions(
ctx, import_file=import_file):
log("Provisions installed")
env["provisioned"] = True
if not skip_data and _migrate(ctx):
log("Data migrated")
env["migrated"] = True
if not skip_frontend and _install_frontend(ctx):
log("Frontend installed")
env["frontend"] = True
if not skip_test and _check_environment(ctx):
log("Environment tested")
env["tested"] = True
except Exception:
log("Error during installation:", exc=True, lvl=critical)
log("Environment status now:", env)
ctx.obj["instance_configuration"]["environments"][next_environment] = env
write_instance(ctx.obj["instance_configuration"])
def _archive(ctx, force=False, dynamic=False):
instance_configuration = ctx.obj["instance_configuration"]
next_environment = get_next_environment(ctx)
env = instance_configuration["environments"][next_environment]
log("Instance info:",
instance_configuration,
next_environment,
pretty=True,
lvl=debug)
log("Installed:", env["installed"], "Tested:", env["tested"], lvl=debug)
if (not env["installed"] or not env["tested"]) and not force:
log("Environment has not been installed - not archiving.", lvl=warn)
return False
log("Archiving environment:", next_environment)
set_instance(ctx.obj["instance"], next_environment)
timestamp = std_now().replace(":", "-").replace(".", "-")
temp_path = mkdtemp(prefix="isomer_backup")
log("Archiving database")
if not dump(
instance_configuration["database_host"],
instance_configuration["database_port"],
env["database"],
os.path.join(temp_path, "db_" + timestamp + ".json"),
):
if not force:
log("Could not archive database.")
return False
archive_filename = os.path.join(
"/var/backups/isomer/",
"%s_%s_%s.tgz" % (ctx.obj["instance"], next_environment, timestamp),
)
try:
shutil.copy(
os.path.join(get_etc_instance_path(),
ctx.obj["instance"] + ".conf"),
temp_path,
)
with tarfile.open(archive_filename, "w:gz") as f:
if not dynamic:
for item in locations:
path = get_path(item, "")
log("Archiving [%s]: %s" % (item, path))
f.add(path)
f.add(temp_path, "db_etc")
except (PermissionError, FileNotFoundError) as e:
log("Could not archive environment:", e, lvl=error)
if not force:
return False
finally:
log("Clearing temporary backup target")
shutil.rmtree(temp_path)
ctx.obj["instance_configuration"]["environments"]["archive"][
timestamp] = env
log(ctx.obj["instance_configuration"])
return archive_filename
def _clear_environment(ctx,
force=False,
clear_env=None,
clear=False,
no_archive=False):
"""Tests an environment for usage, then clears it
:param ctx: Click Context
:param force: Irrefutably destroy environment content
:param clear_env: Environment to clear (Green/Blue)
:param clear: Also destroy generated folders
:param no_archive: Don't attempt to archive instance
"""
instance_name = ctx.obj["instance"]
if clear_env is None:
next_environment = get_next_environment(ctx)
else:
next_environment = clear_env
log("Clearing environment:", next_environment)
set_instance(instance_name, next_environment)
# log('Testing', environment, 'for usage')
env = ctx.obj["instance_configuration"]["environments"][next_environment]
if not no_archive:
if not (_archive(ctx, force) or force):
log("Archival failed, stopping.")
abort(5000)
log("Clearing env:", env, lvl=debug)
for item in locations:
path = get_path(item, "")
log("Clearing [%s]: %s" % (item, path), lvl=debug)
try:
shutil.rmtree(path)
except FileNotFoundError:
log("Path not found:", path, lvl=debug)
except PermissionError:
log("No permission to clear environment", lvl=error)
return False
if not clear:
_create_folders(ctx)
try:
delete_database(ctx.obj["dbhost"],
"%s_%s" % (instance_name, next_environment),
force=True)
except pymongo.errors.ServerSelectionTimeoutError:
log("No database available")
except Exception as e:
log("Could not delete database:", e, lvl=warn, exc=True)
ctx.obj["instance_configuration"]["environments"][
next_environment] = environment_template
write_instance(ctx.obj["instance_configuration"])
return True
def _check_environment(ctx, env=None, dev=False):
"""General fitness tests of the built environment"""
if env is None:
env = get_next_environment(ctx)
log("Health checking the environment '%s'" % env)
# Frontend
not_enough_files = False
html_missing = False
loader_missing = False
size_too_small = False
# Backend
repository_missing = False
modules_missing = False
venv_missing = False
local_missing = False
cache_missing = False
# Backend
if not os.path.exists(os.path.join(get_path('lib', 'repository'))):
log("Repository is missing", lvl=warn)
repository_missing = True
if not os.path.exists(os.path.join(get_path('lib', 'modules'))):
log("Modules folder is missing", lvl=warn)
modules_missing = True
if not os.path.exists(os.path.join(get_path('lib', 'venv'))):
log("Virtual environment is missing", lvl=warn)
venv_missing = True
if not os.path.exists(os.path.join(get_path('local', ''))):
log("Local data folder is missing", lvl=warn)
local_missing = True
if not os.path.exists(os.path.join(get_path('cache', ''))):
log("Cache folder is missing", lvl=warn)
cache_missing = True
# Frontend
_, frontend_target = get_frontend_locations(dev)
if not os.path.exists(os.path.join(frontend_target, 'index.html')):
log("A compiled frontend html seems to be missing", lvl=warn)
html_missing = True
if not glob.glob(frontend_target + '/main.*.js'):
log("A compiled frontend loader seems to be missing", lvl=warn)
loader_missing = True
size_sum = 0
amount_files = 0
for file in glob.glob(os.path.join(frontend_target, '*.gz')):
size_sum += os.stat(file).st_size
amount_files += 1
if amount_files < 4:
log("The frontend probably did not compile completely", lvl=warn)
not_enough_files = True
if size_sum < 2 * 1024 * 1024:
log("The compiled frontend seems exceptionally small", lvl=warn)
size_too_small = True
frontend = (repository_missing or modules_missing or venv_missing
or local_missing or cache_missing)
backend = (not_enough_files or loader_missing or size_too_small
or html_missing)
result = not (frontend or backend)
if result is False:
log("Health check failed", lvl=error)
return result
