def ls(obj, all_versions, long_format, plugins):
"""Print information about plugins in local store. By default, only includes latest versions."""
plugins_local_dir = obj["plugins_local_dir"]
plugin_infos = PluginInfos.make_from_local_store(plugins_local_dir)
if not all_versions:
plugin_infos = plugin_infos.filter_to_latest()
if plugins:
plugin_infos = plugin_infos.filter_to_specs(plugins)
def _read_description(pi):
filename = plugins_local_dir / pi.get_filename()
try:
with lib.temp_directory() as tmp_dir:
lib.extract_targz(filename, tmp_dir)
manifests = list(tmp_dir.glob("**/plugin.toml"))
return lib.read_toml(manifests[0])["description"]
except Exception:
lib.log_error(f"Malformed? Unable to read: {filename}")
if long_format:
info = ([pi.name,
pi.formatted_version(),
_read_description(pi)]
for pi in plugin_infos.as_sorted_list())
lib.log(tabulate(info, headers=["name", "version", "description"]))
else:
info = ([pi.name, pi.formatted_version()]
for pi in plugin_infos.as_sorted_list())
lib.log(tabulate(info, headers=["name", "version"]))
_log_message_explaining_semver()
def status(obj, long_format, plugins):
"""Print information about (successfully) installed plugins."""
host = obj["host"]
plugins_local_dir = obj["plugins_local_dir"]
local_versions = PluginInfos.make_from_local_store(
plugins_local_dir).filter_to_latest()
plugin_infos = PluginInfos.make_from_encapsia(host)
if plugins:
specs = PluginSpecs.make_from_spec_strings(plugins)
plugin_infos = specs.filter(plugin_infos)
headers = ["name*", "version**", "available**", "installed"]
if long_format:
headers.extend(["description", "plugin-tags"])
info = []
for pi in plugin_infos:
pi_info = [
pi.name_and_variant(),
pi.formatted_version(),
_get_available_from_local_store(local_versions, pi),
pi.extras["installed"],
]
if long_format:
pi_info.extend(
[pi.extras["description"], pi.extras["plugin-tags"]])
info.append(pi_info)
lib.log(tabulate(info, headers=headers))
_log_message_explaining_headers()
def dev_update(obj, directory):
"""Update plugin parts which have changed since previous update.
Optionally pass in the DIRECTORY of the plugin (defaults to cwd).
"""
directory = Path(directory)
plugin_toml_path = directory / "plugin.toml"
if not plugin_toml_path.exists():
lib.log_error("Not in a plugin directory.", abort=True)
with _get_modified_plugin_directories(
directory,
reset=obj["plugins_force"]) as modified_plugin_directories:
if modified_plugin_directories:
with lib.temp_directory() as temp_directory:
shutil.copy(plugin_toml_path, temp_directory)
for modified_directory in modified_plugin_directories:
lib.log(f"Including: {modified_directory}")
shutil.copytree(
directory / modified_directory,
temp_directory / modified_directory,
)
api = lib.get_api(**obj)
result = lib.run_plugins_task(
api,
"dev_update_plugin",
dict(),
"Uploading to server",
data=lib.create_targz_as_bytes(temp_directory),
)
if not result:
raise _PluginsTaskError
else:
lib.log("Nothing to do.")
def build_from_src(obj, sources):
"""Build plugins from given source directories."""
plugins_cache_dir = obj["plugins_cache_dir"]
force = obj["force"]
for source_directory in sources:
source_directory = Path(source_directory)
manifest = lib.read_toml(source_directory / "plugin.toml")
name = manifest["name"]
version = manifest["version"]
output_filename = plugins_cache_dir / f"plugin-{name}-{version}.tar.gz"
if not force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
with lib.temp_directory() as temp_directory:
base_dir = temp_directory / f"plugin-{name}-{version}"
base_dir.mkdir()
for t in (
"webfiles",
"views",
"tasks",
"wheels",
"schedules",
"plugin.toml",
):
source_t = source_directory / t
if source_t.exists():
if source_t.is_file():
shutil.copy(source_t, base_dir / t)
else:
shutil.copytree(source_t, base_dir / t)
lib.create_targz(base_dir, output_filename)
lib.log(f"Created: {output_filename}")
def _install_plugin(api, filename):
"""Use the API to install plugin directly from a file."""
if not filename.exists():
lib.log_error(f"Cannot find plugin: {filename}", abort=True)
blob_id = api.upload_file_as_blob(filename.as_posix())
lib.log(f"Uploaded {filename} to blob: {blob_id}")
lib.run_plugins_task(api, "install_plugin", dict(blob_id=blob_id),
"Installing")
def status(obj, plugins):
"""Print information about (successfully) installed plugins."""
local_versions = PluginInfos.make_from_local_store(
obj["plugins_local_dir"]).filter_to_latest()
api = lib.get_api(**obj)
raw_info = api.run_view("pluginsmanager", "installed_plugins_with_tags")
plugin_infos = []
for i in raw_info:
pi = PluginInfo.make_from_name_version(i["name"], i["version"])
available_pi = local_versions.latest_version_matching_spec(i["name"])
if available_pi:
temp = available_pi.formatted_version()
available = "<same>" if temp == pi.formatted_version() else temp
else:
available = ""
pi.extras = {
"description":
i["description"],
"plugin-tags": (", ".join(sorted(i["plugin_tags"])) if isinstance(
i["plugin_tags"], list) else ""),
"installed":
_format_datetime(i["when"]),
"available":
available,
}
plugin_infos.append(pi)
if plugins:
plugin_infos = PluginInfos(plugin_infos).filter_to_specs(plugins).pis
for i in raw_info:
i["version"] = PluginInfo.make_from_name_version(
i["name"], i["version"]).formatted_version()
headers = [
"name",
"version",
"available",
"description",
"installed",
"plugin-tags",
]
info = ([
pi.name,
pi.formatted_version(),
pi.extras["available"],
pi.extras["description"],
pi.extras["installed"],
pi.extras["plugin-tags"],
] for pi in plugin_infos)
lib.log(tabulate(info, headers=headers))
_log_message_explaining_semver()
def use_fixture(obj, name, yes):
"""Switch to fixture with given name."""
if not yes:
click.confirm(
f'Are you sure you want to change the database to fixture "{name}"?',
abort=True,
)
api = lib.get_api(**obj)
poll, NoTaskResultYet = api.dbctl_action("use_fixture", dict(name=name))
lib.log(f"Requested change to fixture {name}.")
lib.log("Please verify by other means (e.g. look at the logs).")
def extend(obj, lifespan):
"""Extend the lifespan of token and update encapsia credentials (if used)."""
api = lib.get_api(**obj)
new_token = api.login_again(lifespan=lifespan)
host = get_host(obj)
if host:
store = CredentialsStore()
url, old_token = store.get(host)
store.set(host, url, new_token)
lib.log("Encapsia credentials file updated.")
else:
lib.log(new_token)
def info(obj, logs):
"""Provide information about installed plugins."""
api = lib.get_api(**obj)
raw_info = api.run_view("pluginsmanager", "installed_plugins_with_tags")
headers = ["name", "version", "description", "when", "plugin_tags"]
info = ([p[h] for h in headers] for p in raw_info)
lib.log(tabulate(info, headers=headers))
if logs:
for i in raw_info:
name, output = i["name"], i["output"]
lib.log(f"\n[Install log for {name}]")
lib.log_output(output.strip())
def expire(obj):
"""Expire token from server, and update encapsia credentials if used."""
api = lib.get_api(**obj)
try:
api.delete("logout")
lib.log("Expired token on server.")
except EncapsiaApiError as e:
lib.log_error("Failed to expire given token!")
lib.log_error(str(e), abort=True)
host = get_host(obj)
if host:
CredentialsStore().remove(host)
lib.log("Removed entry from encapsia credentials file.")
def _log_result(result):
"""Pretty-print log the result from running a task, job, or view."""
if isinstance(result, FileDownloadResponse):
lib.log(
f"Response saved to: {result.filename} (mime_type={result.mime_type})"
)
else:
try:
# Try to pretty print if it converts to JSON.
lib.pretty_print(result, "json")
except json.decoder.JSONDecodeError:
# Otherwise print normally.
lib.log_output(str(result))
def _add_to_local_store_from_s3(pi, plugins_local_dir, force=False):
filename = plugins_local_dir / pi.get_filename()
if not force and filename.exists():
lib.log(f"Found: {filename} (Skipping)")
else:
s3 = boto3.client("s3")
try:
s3.download_file(pi.get_s3_bucket(), pi.get_s3_name(),
str(filename))
except botocore.exceptions.ClientError:
lib.log_error(f"Unable to download: {pi.get_s3_name()}")
else:
lib.log(f"Added to local store: {filename}")
def backup(obj, filename):
"""Backup database to given filename. (or create a temp one if not given)."""
if filename:
filename = Path(filename)
api = lib.get_api(**obj)
handle = lib.dbctl_action(api, "backup_database", dict(), "Backing up database")
temp_filename = Path(
api.dbctl_download_data(handle)
) # TODO remove Path() once encapsia_api switched over to pathlib
if filename is None:
filename = temp_filename
else:
temp_filename.rename(filename)
lib.log(f"Downloaded {filename.stat().st_size} bytes to {filename}")
def build_from_legacy_s3(obj, versions, email, s3_directory):
"""Build plugins from legacy webapps hosted on AWS S3."""
plugins_cache_dir = obj["plugins_cache_dir"]
force = obj["force"]
versions = Path(versions)
for name, version in lib.read_toml(versions).items():
output_filename = Path(plugins_cache_dir,
f"plugin-{name}-{version}.tar.gz")
if not force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
_download_and_build_plugin_from_s3(s3_directory, name, version,
email, output_filename)
lib.log(f"Created: {output_filename}")
def logs(obj, plugins):
"""Print the latest install logs for given plugins."""
api = lib.get_api(**obj)
# Despite the name, this only fetches the latest log for each plugin, not all!
raw_info = lib.resilient_call(
api.run_view,
"pluginsmanager",
"all_plugin_logs",
description="api.run_view('pluginsmanager', 'all_plugin_logs')",
idempotent=True,
)
if plugins:
# Filter to specified plugins.
raw_info = [i for i in raw_info if i["name"] in plugins]
for i in raw_info:
for f in (
"name",
"version",
"description",
"action",
"server",
"success",
"when",
):
lib.log(f"{f.capitalize()}: {i[f]}")
lib.log("Logs:")
if i["success"]:
lib.log_output(i["output"].strip())
else:
lib.log_error(i["output"].strip())
lib.log("")
if len(raw_info) == 0 and len(plugins) > 0:
lib.log(
"No logs found. Note that any plugins must be exact name matches without version info."
)
def _add_to_local_store_from_uri(plugins_local_dir, uri, force=False):
full_name = uri.rsplit("/", 1)[-1]
try:
PluginInfo.make_from_filename(
full_name) # Will raise if name is invalid.
except ValueError:
lib.log_error("That doesn't look like a plugin. Aborting!", abort=True)
store_filename = plugins_local_dir / full_name
if not force and store_filename.exists():
lib.log(f"Found: {store_filename} (Skipping)")
else:
filename, headers = urllib.request.urlretrieve(uri,
tempfile.mkstemp()[1])
shutil.move(filename, store_filename)
lib.log(f"Added to local store: {store_filename}")
def _fetch_plugin_and_store_in_cache(url, plugins_cache_dir, force=False):
full_name = url.rsplit("/", 1)[-1]
m = re.match(r"plugin-([^-]*)-([^-]*).tar.gz", full_name)
if m:
output_filename = plugins_cache_dir / full_name
if not force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
filename, headers = urllib.request.urlretrieve(
url,
tempfile.mkstemp()[1])
shutil.move(filename, output_filename)
lib.log(f"Created: {output_filename}")
else:
lib.log_error("That doesn't look like a plugin. Aborting!", abort=True)
def upstream(obj, plugins, all_versions):
"""Print information about plugins on S3. By default, only includes latest versions."""
plugins_s3_buckets = obj["plugins_s3_buckets"]
lib.log(
f"Searching for plugins in S3 bucket(s): {', '.join(plugins_s3_buckets)}"
)
plugin_infos = PluginInfos.make_from_s3_buckets(plugins_s3_buckets)
if plugins:
plugin_infos = plugin_infos.filter_to_specs(plugins)
if not all_versions:
plugin_infos = plugin_infos.filter_to_latest()
info = ([r.bucket, r.name, r.formatted_version()]
for r in plugin_infos.as_sorted_list())
lib.log(tabulate(info, headers=["bucket", "name", "version"]))
_log_message_explaining_semver()
def _add_to_local_store_from_s3(pi: PluginInfo,
plugins_local_dir: Path,
force: bool = False):
target = plugins_local_dir / pi.get_filename()
if not force and target.exists():
lib.log(f"Found: {target} (Skipping)")
else:
try:
s3.download_file(pi.get_s3_bucket(), pi.get_s3_name(),
target.as_posix())
except s3.S3Error as e:
lib.log_error(str(e))
else:
lib.log(
f"Downloaded {pi.get_s3_bucket()}/{pi.get_s3_name()} and saved to {target}"
)
def restore(obj, filename, yes):
"""Restore database from given backup file."""
filename = Path(filename)
if not yes:
click.confirm(
f'Are you sure you want to restore the database from "{filename}"?',
abort=True,
)
api = lib.get_api(**obj)
handle = api.dbctl_upload_data(filename)
# On a restore, the server is temporarily stopped.
# This means that attempts to use it will generate a 500 error when
# Nginx tries to check the permission.
# Further, the current token may no longer work.
poll, NoResultYet = api.dbctl_action("restore_database", dict(data_handle=handle))
lib.log("Database restore requested.")
lib.log("Please verify by other means (e.g. look at the logs).")
def uninstall(obj, show_logs, namespaces):
"""Uninstall named plugin(s)."""
if namespaces and not obj["plugins_force"]:
lib.log("Preparing to uninstall: " + ", ".join(namespaces))
click.confirm(
"Are you sure?",
abort=True,
)
api = lib.get_api(**obj)
for namespace in namespaces:
lib.run_plugins_task(
api,
"uninstall_plugin",
dict(namespace=namespace),
f"Uninstalling {namespace}",
print_output=show_logs,
)
def add(obj, versions, latest_existing, plugins):
"""Add plugin(s) to local store from file, URL, or S3."""
plugins_local_dir = obj["plugins_local_dir"]
plugins_s3_buckets = obj["plugins_s3_buckets"]
plugins_force = obj["plugins_force"]
host = obj["host"]
specs_to_search_in_s3 = []
to_download_from_s3 = []
s3_versions = None # For performance, only fetch if/when first needed.
added_from_file_or_uri = False
for plugin in plugins:
if Path(plugin).is_file():
_add_to_local_store_from_uri(
plugins_local_dir,
Path(plugin).resolve().as_uri(),
plugins_force,
)
added_from_file_or_uri = True
elif urllib.parse.urlparse(plugin).scheme != "":
_add_to_local_store_from_uri(plugins_local_dir, plugin,
plugins_force)
added_from_file_or_uri = True
else:
specs_to_search_in_s3.append(PluginSpec.make_from_string(plugin))
if versions:
specs_to_search_in_s3.extend(
PluginSpecs.make_from_version_dict(
lib.read_toml(Path(versions).expanduser())))
if latest_existing:
specs_to_search_in_s3.extend(
PluginSpecs.make_from_plugininfos(
PluginInfos.make_from_encapsia(host)))
if specs_to_search_in_s3:
s3_versions = PluginInfos.make_from_s3_buckets(plugins_s3_buckets)
to_download_from_s3.extend(pi for spec in specs_to_search_in_s3 if (
pi := s3_versions.latest_version_matching_spec(spec)) is not None)
if to_download_from_s3:
for pi in to_download_from_s3:
_add_to_local_store_from_s3(pi,
plugins_local_dir,
force=plugins_force)
else:
if not added_from_file_or_uri:
lib.log("Nothing to do!")
def _install_plugin(api, filename: Path, print_output: bool = False):
"""Use the API to install plugin directly from a file."""
if not filename.is_file():
lib.log_error(f"Cannot find plugin: {filename}", abort=True)
blob_id = lib.resilient_call(
api.upload_file_as_blob,
filename.as_posix(),
description=f"api.upload_file_as_blob({filename})",
idempotent=True, # re-uploading a blob is safe
)
lib.log(f"Uploaded {filename} to blob: {blob_id}")
return lib.run_plugins_task(
api,
"install_plugin",
dict(blob_id=blob_id),
"Installing",
print_output=print_output,
idempotent=True, # re-installing a plugin should be safe
)
def dev_build(obj, sources):
"""Build plugins from given source directories."""
plugins_local_dir = obj["plugins_local_dir"]
plugins_force = obj["plugins_force"]
for source_directory in sources:
source_directory = Path(source_directory)
manifest = lib.read_toml(source_directory / "plugin.toml")
name = manifest["name"]
version = manifest["version"]
tags = manifest.get("tags", [])
try:
variant = get_variant_from_tags(tags)
except TooManyVariantTagsError as e:
lib.log_error(str(e), abort=True)
if variant:
output_filename = (
plugins_local_dir /
f"plugin-{name}-variant-{variant}-{version}.tar.gz")
else:
output_filename = plugins_local_dir / f"plugin-{name}-{version}.tar.gz"
if not plugins_force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
with lib.temp_directory() as temp_directory:
base_dir = temp_directory / f"plugin-{name}-{version}"
base_dir.mkdir()
for t in (
"webfiles",
"views",
"tasks",
"wheels",
"schedules",
"plugin.toml",
):
source_t = source_directory / t
if source_t.exists():
if source_t.is_file():
shutil.copy(source_t, base_dir / t)
else:
shutil.copytree(source_t, base_dir / t)
lib.create_targz(base_dir, output_filename)
lib.log(f"Added to local store: {output_filename}")
def main(ctx, command):
"""Print longer help information about the CLI."""
root_command = ctx.parent.command
if command:
lib.log(root_command.get_command(ctx, command).get_help(ctx))
else:
lib.log(root_command.get_help(ctx))
lib.log()
lib.log("Subcommands:")
long_list = []
for name in root_command.list_commands(ctx):
command = root_command.get_command(ctx, name)
if isinstance(command, click.Group):
for subname in command.list_commands(ctx):
subcommand = command.get_command(ctx, subname)
help_str = subcommand.get_short_help_str()
long_list.append((name, subname, help_str))
width = max(
len(name) + len(subname) for (name, subname, _) in long_list)
for name, subname, help_str in long_list:
left = f"{name} {subname}"
left = left + " " * (width + 2 - len(left))
lib.log(f" {left} {help_str}")
def upstream(obj, plugins, all_versions):
"""Print information about plugins on S3.
By default, only includes latest versions.
"""
plugins_s3_buckets = obj["plugins_s3_buckets"]
lib.log(
f"Searching for plugins in S3 bucket(s): {', '.join(plugins_s3_buckets)}"
)
plugin_infos = PluginInfos.make_from_s3_buckets(plugins_s3_buckets)
if plugins:
plugin_infos = PluginSpecs.make_from_spec_strings(plugins).filter(
plugin_infos)
if not all_versions:
plugin_infos = plugin_infos.filter_to_latest()
info = ([
r.name_and_variant(),
r.formatted_version(),
r.get_s3_bucket(),
r.get_s3_path(),
] for r in sorted(plugin_infos))
lib.log(tabulate(info, headers=["name*", "version**", "bucket", "path"]))
_log_message_explaining_headers()
def main():
"""Print version information and exits."""
lib.log(f"Encapsia CLI version: {encapsia_cli.__version__}")
lib.log(f"Encapsia API version: {encapsia_api.__version__}")
def _log_message_explaining_headers():
lib.log(
"\n(*) Plugin variant shown in square brackets when defined."
"\n(**) Equivalent semver versions are shown in brackets when non-semver version is used."
)
def install(obj, versions, show_logs, latest_existing, plugins):
"""Install/upgrade plugins by name, from files, or from a versions.toml file.
Plugins provided as files are put in the local store before being installed.
When described by name alone, the latest plugin of that name in the local store will be used.
Plugins specified in the versions.toml file will be taken from the local store.
"""
plugins_local_dir = obj["plugins_local_dir"]
plugins_force = obj["plugins_force"]
host = obj["host"]
# Create a list of installation candidates.
to_install_candidates = []
for plugin in plugins:
plugin_filename = Path(plugin).resolve()
if plugin_filename.is_file():
# If it looks like a file then just add it.
_add_to_local_store_from_uri(plugins_local_dir,
plugin_filename.as_uri(),
force=True)
to_install_candidates.append(
PluginInfo.make_from_filename(plugin_filename))
else:
# Else assume it is a spec for a plugin already in the local store.
to_install_candidates.append(PluginSpec.make_from_string(plugin))
if versions:
# Assume plugins already present in local store.
to_install_candidates.extend(
PluginSpecs.make_from_version_dict(
lib.read_toml(Path(versions).expanduser())))
if latest_existing:
to_install_candidates.extend(
PluginSpec(pi.name, pi.variant)
for pi in PluginInfos.make_from_encapsia(host))
# Work out and list installation plan.
# to_install_candidates = sorted(PluginInfos(to_install_candidates))
installed = PluginInfos.make_from_encapsia(host)
local_store = PluginInfos.make_from_local_store(plugins_local_dir)
plan = _create_install_plan(to_install_candidates,
installed,
local_store,
force_install=plugins_force)
to_install = [i[0] for i in plan if i[4] != "skip"]
headers = ["name*", "existing version**", "new version**", "action"]
lib.log(tabulate([i[1:] for i in plan], headers=headers))
_log_message_explaining_headers()
# Seek confirmation unless force.
if to_install and not plugins_force:
click.confirm(
"Do you wish to proceed with the above plan?",
abort=True,
)
# Install them.
lib.log("")
if to_install:
api = lib.get_api(**obj)
for pi in to_install:
success = _install_plugin(api,
plugins_local_dir / pi.get_filename(),
print_output=show_logs)
if not success:
lib.log_error("Some plugins failed to install.", abort=True)
else:
lib.log("Nothing to do!")
def main(ctx, colour):
"""Print version information about the CLI."""
ctx.color = {"always": True, "never": False, "auto": None}[colour]
lib.log(f"Encapsia CLI version: {encapsia_cli.__version__}")
lib.log(f"Encapsia API version: {encapsia_api.__version__}")
