def make_from_encapsia(host: str) -> PluginInfos:
# TODO: use pluginsmanager.plugins() if it exists
api = lib.get_api(host=host)
raw_info = lib.resilient_call(
api.run_view,
"pluginsmanager",
"installed_plugins_with_tags",
description=
"api.run_view('pluginsmanager', 'installed_plugins_with_tags')",
idempotent=True,
)
pis = []
for i in raw_info:
tags = i.get("plugin_tags")
if not isinstance(tags, list):
tags = []
try:
variant = get_variant_from_tags(tags)
except TooManyVariantTagsError as e:
lib.log_error(f"Error in {i['name']} tag list: {e}")
pi = PluginInfo.make_from_name_variant_version(
i["name"], variant, i["version"])
pi.extras.update({
"description": i["description"],
"installed": _format_datetime(i["when"]),
"plugin-tags": ", ".join(sorted(tags)),
})
pis.append(pi)
return PluginInfos(pis)
def get_command(self, ctx, name):
try:
return COMMANDS[name]
except KeyError:
lib.log_error(ctx.get_help())
lib.log_error()
raise click.UsageError(f"Unknown command {name}")
def logs(obj, plugins):
"""Print the latest install logs for given plugins."""
api = lib.get_api(**obj)
# Despite the name, this only fetches the latest log for each plugin, not all!
raw_info = lib.resilient_call(
api.run_view,
"pluginsmanager",
"all_plugin_logs",
description="api.run_view('pluginsmanager', 'all_plugin_logs')",
idempotent=True,
)
if plugins:
# Filter to specified plugins.
raw_info = [i for i in raw_info if i["name"] in plugins]
for i in raw_info:
for f in (
"name",
"version",
"description",
"action",
"server",
"success",
"when",
):
lib.log(f"{f.capitalize()}: {i[f]}")
lib.log("Logs:")
if i["success"]:
lib.log_output(i["output"].strip())
else:
lib.log_error(i["output"].strip())
lib.log("")
if len(raw_info) == 0 and len(plugins) > 0:
lib.log(
"No logs found. Note that any plugins must be exact name matches without version info."
)
def _parse_version(cls, version):
# Consider a 4th digit to be a SemVer pre-release.
# E.g. 1.2.3.4 is 1.2.3-4
m = cls.FOUR_DIGIT_VERSION_REGEX.match(version)
if m:
major, minor, patch, prerelease = m.groups()
return semver.VersionInfo(major=major,
minor=minor,
patch=patch,
prerelease=prerelease)
# Consider a "dev" build to be a SemVer pre-release.
# E.g. 0.0.209dev12 is 0.0.209-12
m = cls.DEV_VERSION_REGEX.match(version)
if m:
major, minor, patch, prerelease = m.groups()
return semver.VersionInfo(major=major,
minor=minor,
patch=patch,
prerelease=prerelease)
# Otherwise hope that the semver package can deal with it.
try:
return semver.VersionInfo.parse(version)
except ValueError as e:
lib.log_error(str(e))
# At least return something comparable.
return semver.VersionInfo(major=0)
def dev_update(obj, directory):
"""Update plugin parts which have changed since previous update.
Optionally pass in the DIRECTORY of the plugin (defaults to cwd).
"""
directory = Path(directory)
plugin_toml_path = directory / "plugin.toml"
if not plugin_toml_path.exists():
lib.log_error("Not in a plugin directory.", abort=True)
with _get_modified_plugin_directories(
directory,
reset=obj["plugins_force"]) as modified_plugin_directories:
if modified_plugin_directories:
with lib.temp_directory() as temp_directory:
shutil.copy(plugin_toml_path, temp_directory)
for modified_directory in modified_plugin_directories:
lib.log(f"Including: {modified_directory}")
shutil.copytree(
directory / modified_directory,
temp_directory / modified_directory,
)
api = lib.get_api(**obj)
result = lib.run_plugins_task(
api,
"dev_update_plugin",
dict(),
"Uploading to server",
data=lib.create_targz_as_bytes(temp_directory),
)
if not result:
raise _PluginsTaskError
else:
lib.log("Nothing to do.")
def _create_install_plan(candidates, installed, local_store, force_install):
plan = []
for spec in candidates:
candidate = local_store.latest_version_matching_spec(spec)
if not candidate:
lib.log_error(
f"Could not find plugin matching {spec} in local store!",
abort=True)
current = installed.latest_version_matching_spec(
PluginSpec(spec.name, spec.variant))
if current:
current_version = current.formatted_version()
if current.semver < candidate.semver:
action = "upgrade"
elif current.semver > candidate.semver:
action = "downgrade"
else:
action = "reinstall" if force_install else "skip"
else:
current_version = ""
action = "install"
plan.append([
candidate, # keep first for sorting
candidate.name_and_variant(),
current_version,
candidate.formatted_version(),
action,
])
return sorted(plan)
def _install_plugin(api, filename):
"""Use the API to install plugin directly from a file."""
if not filename.exists():
lib.log_error(f"Cannot find plugin: {filename}", abort=True)
blob_id = api.upload_file_as_blob(filename.as_posix())
lib.log(f"Uploaded {filename} to blob: {blob_id}")
lib.run_plugins_task(api, "install_plugin", dict(blob_id=blob_id),
"Installing")
def make_from_local_store(plugins_local_dir):
result = plugins_local_dir.glob("plugin-*-*.tar.gz")
pis = []
for p in result:
try:
pis.append(PluginInfo.make_from_filename(p))
except ValueError as e:
lib.log_error(str(e))
return PluginInfos(pis)
def _read_description(pi):
filename = plugins_local_dir / pi.get_filename()
try:
with lib.temp_directory() as tmp_dir:
lib.extract_targz(filename, tmp_dir)
manifests = list(tmp_dir.glob("**/plugin.toml"))
return lib.read_toml(manifests[0])["description"]
except Exception:
lib.log_error(f"Malformed? Unable to read: {filename}")
def make_from_s3_buckets(plugins_s3_buckets):
try:
return PluginInfos([
PluginInfo.make_from_s3(bucket, x["Key"])
for bucket, x in s3.list_buckets(plugins_s3_buckets)
if x["Key"].endswith(".tar.gz")
])
except s3.S3Error as e:
lib.log_error(str(e), abort=True)
return None # Never reached, but keep linters happy
def _add_to_local_store_from_s3(pi, plugins_local_dir, force=False):
filename = plugins_local_dir / pi.get_filename()
if not force and filename.exists():
lib.log(f"Found: {filename} (Skipping)")
else:
s3 = boto3.client("s3")
try:
s3.download_file(pi.get_s3_bucket(), pi.get_s3_name(),
str(filename))
except botocore.exceptions.ClientError:
lib.log_error(f"Unable to download: {pi.get_s3_name()}")
else:
lib.log(f"Added to local store: {filename}")
def expire(obj):
"""Expire token from server, and update encapsia credentials if used."""
api = lib.get_api(**obj)
try:
api.delete("logout")
lib.log("Expired token on server.")
except EncapsiaApiError as e:
lib.log_error("Failed to expire given token!")
lib.log_error(str(e), abort=True)
host = get_host(obj)
if host:
CredentialsStore().remove(host)
lib.log("Removed entry from encapsia credentials file.")
def shell(obj):
"""Launch an httpie interactive shell with passed-in credentials."""
try:
importlib.import_module("http_prompt")
except ModuleNotFoundError:
lib.log_error("Please install http-prompt first.", abort=True)
api = lib.get_api(**obj)
argv = [
"http-prompt",
api.url,
f"Authorization: Bearer {api.token}",
"Accept: application/json",
]
subprocess.run(argv)
def make_from_s3_buckets(plugins_s3_buckets):
s3 = boto3.client("s3")
plugin_infos = []
for bucket in plugins_s3_buckets:
try:
paginator = s3.get_paginator("list_objects_v2")
response = paginator.paginate(Bucket=bucket)
plugin_infos.extend(
PluginInfo.make_from_s3_path(bucket, x["Key"])
for r in response for x in r.get("Contents", [])
if x["Key"].endswith(".tar.gz"))
except botocore.exceptions.ClientError as e:
lib.log_error(f"Unable to search bucket: {bucket}")
lib.log_error(str(e), abort=True)
return PluginInfos(plugin_infos)
def _add_to_local_store_from_uri(plugins_local_dir, uri, force=False):
full_name = uri.rsplit("/", 1)[-1]
try:
PluginInfo.make_from_filename(
full_name) # Will raise if name is invalid.
except ValueError:
lib.log_error("That doesn't look like a plugin. Aborting!", abort=True)
store_filename = plugins_local_dir / full_name
if not force and store_filename.exists():
lib.log(f"Found: {store_filename} (Skipping)")
else:
filename, headers = urllib.request.urlretrieve(uri,
tempfile.mkstemp()[1])
shutil.move(filename, store_filename)
lib.log(f"Added to local store: {store_filename}")
def _fetch_plugin_and_store_in_cache(url, plugins_cache_dir, force=False):
full_name = url.rsplit("/", 1)[-1]
m = re.match(r"plugin-([^-]*)-([^-]*).tar.gz", full_name)
if m:
output_filename = plugins_cache_dir / full_name
if not force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
filename, headers = urllib.request.urlretrieve(
url,
tempfile.mkstemp()[1])
shutil.move(filename, output_filename)
lib.log(f"Created: {output_filename}")
else:
lib.log_error("That doesn't look like a plugin. Aborting!", abort=True)
def _add_to_local_store_from_s3(pi: PluginInfo,
plugins_local_dir: Path,
force: bool = False):
target = plugins_local_dir / pi.get_filename()
if not force and target.exists():
lib.log(f"Found: {target} (Skipping)")
else:
try:
s3.download_file(pi.get_s3_bucket(), pi.get_s3_name(),
target.as_posix())
except s3.S3Error as e:
lib.log_error(str(e))
else:
lib.log(
f"Downloaded {pi.get_s3_bucket()}/{pi.get_s3_name()} and saved to {target}"
)
def _install_plugin(api, filename: Path, print_output: bool = False):
"""Use the API to install plugin directly from a file."""
if not filename.is_file():
lib.log_error(f"Cannot find plugin: {filename}", abort=True)
blob_id = lib.resilient_call(
api.upload_file_as_blob,
filename.as_posix(),
description=f"api.upload_file_as_blob({filename})",
idempotent=True, # re-uploading a blob is safe
)
lib.log(f"Uploaded {filename} to blob: {blob_id}")
return lib.run_plugins_task(
api,
"install_plugin",
dict(blob_id=blob_id),
"Installing",
print_output=print_output,
idempotent=True, # re-installing a plugin should be safe
)
def dev_build(obj, sources):
"""Build plugins from given source directories."""
plugins_local_dir = obj["plugins_local_dir"]
plugins_force = obj["plugins_force"]
for source_directory in sources:
source_directory = Path(source_directory)
manifest = lib.read_toml(source_directory / "plugin.toml")
name = manifest["name"]
version = manifest["version"]
tags = manifest.get("tags", [])
try:
variant = get_variant_from_tags(tags)
except TooManyVariantTagsError as e:
lib.log_error(str(e), abort=True)
if variant:
output_filename = (
plugins_local_dir /
f"plugin-{name}-variant-{variant}-{version}.tar.gz")
else:
output_filename = plugins_local_dir / f"plugin-{name}-{version}.tar.gz"
if not plugins_force and output_filename.exists():
lib.log(f"Found: {output_filename} (Skipping)")
else:
with lib.temp_directory() as temp_directory:
base_dir = temp_directory / f"plugin-{name}-{version}"
base_dir.mkdir()
for t in (
"webfiles",
"views",
"tasks",
"wheels",
"schedules",
"plugin.toml",
):
source_t = source_directory / t
if source_t.exists():
if source_t.is_file():
shutil.copy(source_t, base_dir / t)
else:
shutil.copytree(source_t, base_dir / t)
lib.create_targz(base_dir, output_filename)
lib.log(f"Added to local store: {output_filename}")
def add(obj, versions, latest_existing, plugins):
"""Add plugin(s) to local store from file, URL, or S3."""
plugins_local_dir = obj["plugins_local_dir"]
plugins_s3_buckets = obj["plugins_s3_buckets"]
plugins_force = obj["plugins_force"]
to_download_from_s3 = []
s3_versions = None # For performance, only fetch if/when first needed.
for plugin in plugins:
if Path(plugin).is_file():
_add_to_local_store_from_uri(
plugins_local_dir,
Path(plugin).resolve().as_uri(),
plugins_force,
)
elif urllib.parse.urlparse(plugin).scheme != "":
_add_to_local_store_from_uri(plugins_local_dir, plugin,
plugins_force)
else:
if s3_versions is None:
s3_versions = PluginInfos.make_from_s3_buckets(
plugins_s3_buckets)
pi = s3_versions.latest_version_matching_spec(plugin)
if pi is None:
lib.log_error(f"Cannot find plugin: {plugin}", abort=True)
else:
to_download_from_s3.append(pi)
if versions:
if s3_versions is None:
s3_versions = PluginInfos.make_from_s3_buckets(plugins_s3_buckets)
to_download_from_s3.extend(
s3_versions.latest_version_matching_spec(f"{name}-{version}")
for name, version in _read_versions_toml(versions))
if latest_existing:
to_download_from_s3.extend(
PluginInfos.make_from_encapsia(obj["host"]).as_sorted_list())
for pi in to_download_from_s3:
_add_to_local_store_from_s3(pi, plugins_local_dir, force=plugins_force)
def install(obj, versions, show_logs, latest_existing, plugins):
"""Install/upgrade plugins by name, from files, or from a versions.toml file.
Plugins provided as files are put in the local store before being installed.
When described by name alone, the latest plugin of that name in the local store will be used.
Plugins specified in the versions.toml file will be taken from the local store.
"""
plugins_local_dir = obj["plugins_local_dir"]
plugins_force = obj["plugins_force"]
host = obj["host"]
# Create a list of installation candidates.
to_install_candidates = []
for plugin in plugins:
plugin_filename = Path(plugin).resolve()
if plugin_filename.is_file():
# If it looks like a file then just add it.
_add_to_local_store_from_uri(plugins_local_dir,
plugin_filename.as_uri(),
force=True)
to_install_candidates.append(
PluginInfo.make_from_filename(plugin_filename))
else:
# Else assume it is a spec for a plugin already in the local store.
to_install_candidates.append(PluginSpec.make_from_string(plugin))
if versions:
# Assume plugins already present in local store.
to_install_candidates.extend(
PluginSpecs.make_from_version_dict(
lib.read_toml(Path(versions).expanduser())))
if latest_existing:
to_install_candidates.extend(
PluginSpec(pi.name, pi.variant)
for pi in PluginInfos.make_from_encapsia(host))
# Work out and list installation plan.
# to_install_candidates = sorted(PluginInfos(to_install_candidates))
installed = PluginInfos.make_from_encapsia(host)
local_store = PluginInfos.make_from_local_store(plugins_local_dir)
plan = _create_install_plan(to_install_candidates,
installed,
local_store,
force_install=plugins_force)
to_install = [i[0] for i in plan if i[4] != "skip"]
headers = ["name*", "existing version**", "new version**", "action"]
lib.log(tabulate([i[1:] for i in plan], headers=headers))
_log_message_explaining_headers()
# Seek confirmation unless force.
if to_install and not plugins_force:
click.confirm(
"Do you wish to proceed with the above plan?",
abort=True,
)
# Install them.
lib.log("")
if to_install:
api = lib.get_api(**obj)
for pi in to_install:
success = _install_plugin(api,
plugins_local_dir / pi.get_filename(),
print_output=show_logs)
if not success:
lib.log_error("Some plugins failed to install.", abort=True)
else:
lib.log("Nothing to do!")
def install(obj, versions, show_logs, latest_existing, plugins):
"""Install/upgrade plugins by name, from files, or from a versions.toml file.
Plugins provided as files are put in the local store before being installed.
When described by name alone, the latest plugin of that name in the local store will be used.
Plugins specified in the versions.toml file will be taken from the local store.
"""
plugins_local_dir = obj["plugins_local_dir"]
plugins_force = obj["plugins_force"]
# Create a list of installation candidates.
to_install_candidates = []
for plugin in plugins:
plugin_filename = Path(plugin).resolve()
if plugin_filename.is_file():
# If it looks like a file then just add it.
_add_to_local_store_from_uri(plugins_local_dir,
plugin_filename.as_uri(),
force=True)
to_install_candidates.append(
PluginInfo.make_from_filename(plugin_filename))
else:
# Else assume it is a spec for a plugin already in the local store.
pi = PluginInfos.make_from_local_store(
plugins_local_dir).latest_version_matching_spec(plugin)
if pi:
to_install_candidates.append(pi)
else:
lib.log_error(f"Cannot find plugin: {plugin}", abort=True)
if versions:
for name, version in _read_versions_toml(versions):
to_install_candidates.append(
PluginInfo.make_from_name_version(name, version))
if latest_existing:
available = PluginInfos.make_from_local_store(
plugins_local_dir).filter_to_latest()
for pi in PluginInfos.make_from_encapsia(obj["host"]).as_sorted_list():
a = available.latest_version_matching_spec(pi.name)
if a:
to_install_candidates.append(a)
# Work out and list installation plan.
to_install_candidates = PluginInfos(to_install_candidates).as_sorted_list()
to_install = []
installed = PluginInfos.make_from_encapsia(obj["host"])
headers = ["name", "existing version", "new version", "action"]
info = []
for pi in to_install_candidates:
current = installed.latest_version_matching_spec(pi.name)
if current:
current_version = current.formatted_version()
if current.semver < pi.semver:
action = "upgrade"
elif current.semver > pi.semver:
action = "downgrade"
else:
action = "reinstall" if plugins_force else "skip"
else:
current_version = ""
action = "install"
info.append([pi.name, current_version, pi.formatted_version(), action])
if action != "skip":
to_install.append(pi)
lib.log(tabulate(info, headers=headers))
_log_message_explaining_semver()
# Seek confirmation unless force.
if to_install and not plugins_force:
click.confirm(
"Do you wish to proceed with the above plan?",
abort=True,
)
# Install them.
lib.log("")
if to_install:
api = lib.get_api(**obj)
for pi in to_install:
_install_plugin(api,
plugins_local_dir / pi.get_filename(),
print_output=show_logs)
else:
lib.log("Nothing to do!")
