def installed():
"""Returns JSON of all installed plugins
Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name
"""
project = Project.find()
config = ConfigService(project)
discovery = PluginDiscoveryService(project)
installed_plugins = {}
# merge definitions
for plugin in sorted(config.plugins(), key=lambda x: x.name):
try:
definition = discovery.find_plugin(plugin.type, plugin.name)
merged_plugin_definition = {
**definition.canonical(),
**plugin.canonical()
}
except PluginNotFoundError:
merged_plugin_definition = {**plugin.canonical()}
merged_plugin_definition.pop("settings", None)
merged_plugin_definition.pop("select", None)
if not plugin.type in installed_plugins:
installed_plugins[plugin.type] = []
installed_plugins[plugin.type].append(merged_plugin_definition)
return jsonify({
**project.meltano.canonical(), "plugins": installed_plugins
})
def add_related_plugins(project,
plugins,
add_service: ProjectAddService,
plugin_types=list(PluginType)):
discovery_service = PluginDiscoveryService(project)
added_plugins = []
for plugin_install in plugins:
try:
plugin_def = discovery_service.find_plugin(plugin_install.type,
plugin_install.name)
except PluginNotFoundError:
continue
related_plugins = add_service.add_related(plugin_def,
plugin_types=plugin_types)
for related_plugin in related_plugins:
if related_plugin.should_add_to_file(project):
click.secho(
f"Added related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project",
fg="green",
)
else:
click.secho(
f"Adding related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project...",
fg="green",
)
added_plugins.extend(related_plugins)
return added_plugins
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration profiles
"""
project = Project.find()
settings = PluginSettingsService(project, show_hidden=False)
plugin = ConfigService(project).get_plugin(plugin_ref)
discovery_service = PluginDiscoveryService(project)
try:
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
settings_group_validation = plugin_def.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify({
"profiles":
profiles,
"settings":
Canonical.as_canonical(settings.definitions(plugin)),
"settings_group_validation":
settings_group_validation,
})
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
# We use the DiscoveryService rather than the ConfigService because the
# plugin may not actually be installed yet at this point.
discovery = PluginDiscoveryService(project)
plugin = discovery.find_plugin(plugin_type, plugin_name)
add_service = ProjectAddService(project)
related_plugins = add_service.add_related(plugin)
install_service = PluginInstallService(project)
install_service.install_plugins(related_plugins)
return jsonify([plugin.canonical() for plugin in related_plugins])
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def add_plugin(
add_service,
project: Project,
plugin_type: PluginType,
plugin_name: str,
include_related=False,
):
try:
plugin = add_service.add(plugin_type, plugin_name)
click.secho(f"Added '{plugin_name}' to your Meltano project.",
fg="green")
except PluginAlreadyAddedException as err:
click.secho(
f"'{plugin_name}' was found in your Meltano project. Use `meltano install` to install it.",
fg="yellow",
err=True,
)
plugin = err.plugin
except (PluginNotSupportedException, PluginNotFoundError):
click.secho(f"Error: {plugin_type} '{plugin_name}' is not supported.",
fg="red")
raise click.Abort()
install_service = PluginInstallService(project)
try:
click.secho(f"Installing '{plugin_name}'...")
run = install_service.install_plugin(plugin)
if run:
click.secho(run.stdout)
click.secho(f"Installed '{plugin_name}'.", fg="green")
click.secho(f"Added and installed {plugin_type} '{plugin_name}'.",
fg="green")
except PluginNotInstallable as install_err:
logging.info(f"{plugin_type} is not installable, skipping install.")
except SubprocessError as proc_err:
click.secho(str(proc_err), fg="red")
click.secho(proc_err.process.stderr, err=True)
raise click.Abort()
docs_link = plugin._extras.get("docs")
if docs_link:
click.secho(
f"Visit {docs_link} for more details about '{plugin.name}'.")
if include_related:
discovery_service = PluginDiscoveryService(project)
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
related_plugins = add_service.add_related(plugin_def)
if len(related_plugins) == 0:
click.secho(
"No related plugins found that are not already installed.")
else:
for plugin in related_plugins:
click.secho(
f"Added related plugin '{plugin.name}' to your Meltano project.",
fg="green",
)
click.secho(
f"Installing {len(related_plugins)} related plugins...")
install_status = install_service.install_plugins(related_plugins)
num_installed = len(install_status["installed"])
num_failed = len(install_status["errors"])
fg = "green"
if num_failed >= 0 and num_installed == 0:
fg = "red"
elif num_failed > 0 and num_installed > 0:
fg = "yellow"
click.secho(
f"Installed {num_installed}/{num_installed+num_failed} related plugins.",
fg=fg,
)
def install_missing_plugins(project: Project, extractor: str, loader: str,
transform: str):
add_service = ProjectAddService(project)
config_service = ConfigService(project)
plugins = []
if transform != "only":
try:
config_service.find_plugin(extractor,
plugin_type=PluginType.EXTRACTORS)
except PluginMissingError:
click.secho(
f"Extractor '{extractor}' is missing, trying to install it...",
fg="yellow",
)
plugin = add_plugin(project,
PluginType.EXTRACTORS,
extractor,
add_service=add_service)
plugins.append(plugin)
try:
config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
except PluginMissingError:
click.secho(
f"Loader '{loader}' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.LOADERS,
loader,
add_service=add_service)
plugins.append(plugin)
if transform != "skip":
try:
config_service.find_plugin("dbt",
plugin_type=PluginType.TRANSFORMERS)
except PluginMissingError as e:
click.secho(
f"Transformer 'dbt' is missing, trying to install it...",
fg="yellow")
plugin = add_plugin(project,
PluginType.TRANSFORMERS,
"dbt",
add_service=add_service)
plugins.append(plugin)
discovery_service = PluginDiscoveryService(project)
extractor_plugin_def = discovery_service.find_plugin(
PluginType.EXTRACTORS, extractor)
try:
transform_plugin = config_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
# Update dbt_project.yml in case the vars values have changed in meltano.yml
transform_add_service = TransformAddService(project)
transform_add_service.update_dbt_project(transform_plugin)
except PluginMissingError:
try:
# Check if there is a default transform for this extractor
transform_plugin_def = discovery_service.find_plugin_by_namespace(
extractor_plugin_def.namespace, PluginType.TRANSFORMS)
click.secho(
f"Transform '{transform_plugin_def.name}' is missing, trying to install it...",
fg="yellow",
)
add_plugin(
project,
PluginType.TRANSFORMS,
transform_plugin_def.name,
add_service=add_service,
)
plugins.append(plugin)
except PluginNotFoundError:
# There is no default transform for this extractor..
# Don't panic, everything is cool - just run custom transforms
pass
return install_plugins(project, plugins)