def test_discovery_yaml(self, subject):
# test for all
discovery = subject.discover()
# raw yaml load
for plugin_type, plugin_defs in subject._discovery:
if not PluginType.value_exists(plugin_type):
continue
plugin_type = PluginType(plugin_type)
plugin_names = [plugin["name"] for plugin in plugin_defs]
assert plugin_type in discovery
assert sorted(discovery[plugin_type]) == sorted(plugin_names)
def test_discovery_yaml(self, subject):
plugins_by_type = subject.plugins_by_type()
# raw yaml load
for plugin_type, raw_plugin_defs in subject._discovery:
if not PluginType.value_exists(plugin_type):
continue
plugin_type = PluginType(plugin_type)
plugin_names = [
plugin.name for plugin in plugins_by_type[plugin_type]
]
for raw_plugin_def in raw_plugin_defs:
assert raw_plugin_def["name"] in plugin_names
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
plugins_service = ProjectPluginsService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = plugins_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(plugins_service.plugins())
if include_related:
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise CliError("Failed to install plugin(s)")
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_service.add_related(plugin)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
related_plugins.reverse()
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_status = install_service.install_plugins(
related_plugins, reason=PluginInstallReason.ADD)
for error in install_status["errors"]:
raise PluginInstallError(error["message"])
return jsonify([plugin.canonical() for plugin in related_plugins])
def invoke(project, plugin_type, plugin_name, plugin_args):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
_, Session = project_engine(project)
session = Session()
try:
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name,
plugin_type=plugin_type,
invokable=True)
service = invoker_factory(project,
plugin,
prepare_with_session=session)
handle = service.invoke(*plugin_args)
exit_code = handle.wait()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_invoke(plugin_name=plugin_name,
plugin_args=" ".join(plugin_args))
sys.exit(exit_code)
except Exception as err:
logging.exception(err)
click.secho(f"An error occured: {err}.", fg="red")
raise click.Abort() from err
finally:
session.close()
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
config_service = ConfigService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = config_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
plugins = list(config_service.plugins())
if include_related:
add_service = ProjectAddService(project, config_service=config_service)
related_plugins = add_related_plugins(project, plugins, add_service=add_service)
plugins.extend(related_plugins)
click.echo(f"Installing {len(plugins)} plugins...")
success = install_plugins(project, plugins)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_install()
if not success:
raise click.Abort()
def invoke(project, plugin_type, dump, plugin_name, plugin_args):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
_, Session = project_engine(project)
session = Session()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
invokable=True)
try:
invoker = invoker_factory(project,
plugin,
plugins_service=plugins_service)
with invoker.prepared(session):
if dump:
dump_file(invoker, dump)
exit_code = 0
else:
handle = invoker.invoke(*plugin_args)
exit_code = handle.wait()
except SubprocessError as err:
logger.error(err.stderr)
raise
finally:
session.close()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_invoke(plugin_name=plugin_name,
plugin_args=" ".join(plugin_args))
sys.exit(exit_code)
def config(ctx, project, plugin_type, plugin_name, format):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
config = ConfigService(project)
plugin = config.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
_, Session = project_engine(project)
session = Session()
try:
settings = PluginSettingsService(project).build(plugin)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
config = settings.as_config(session=session)
print(json.dumps(config))
elif format == "env":
for env, value in settings.as_env(session=session).items():
print(f"{env}={value}")
finally:
session.close()
def add():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
add_service = ProjectAddService(project)
plugin = add_service.add(plugin_type, plugin_name)
return jsonify(plugin.canonical())
def config(ctx, project, plugin_type, plugin_name, format, extras):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
plugins_service = ProjectPluginsService(project)
try:
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
except PluginNotFoundError:
if plugin_name == "meltano":
plugin = None
else:
raise
_, Session = project_engine(project)
session = Session()
try:
if plugin:
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service)
else:
settings = ProjectSettingsService(
project, config_service=plugins_service.config_service)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
process = extras is not True
config = settings.as_dict(extras=extras,
process=process,
session=session)
print(json.dumps(config, indent=2))
elif format == "env":
env = settings.as_env(extras=extras, session=session)
with tempfile.NamedTemporaryFile() as temp_dotenv:
path = temp_dotenv.name
for key, value in env.items():
dotenv.set_key(path, key, value)
dotenv_content = Path(temp_dotenv.name).read_text()
print(dotenv_content, end="")
finally:
session.close()
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type)
install_service = PluginInstallService(project)
install_service.install_plugin(plugin)
return jsonify(plugin.canonical())
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_service.install_plugin(plugin, reason=PluginInstallReason.ADD)
return jsonify(plugin.canonical())
def load_plugins(self, plugins) -> Canonical:
"""Parse the meltano.yml file and return it as `ProjectPlugin` instances."""
plugin_type_plugins = Canonical()
for plugin_type in PluginType:
plugin_type_plugins[plugin_type] = []
# this will parse the meltano.yml file and create an instance of the
# corresponding `plugin_class` for all the plugins.
for plugin_type, raw_plugins in plugins.items():
for raw_plugin in raw_plugins:
plugin = ProjectPlugin(PluginType(plugin_type), **raw_plugin)
plugin_type_plugins[plugin.type].append(plugin)
return plugin_type_plugins
def add(ctx, project, plugin_type, plugin_name, **flags):
plugin_type = PluginType.from_cli_argument(plugin_type)
plugin_names = plugin_name # nargs=-1
if flags["custom"]:
if plugin_type in (
PluginType.TRANSFORMERS,
PluginType.TRANSFORMS,
PluginType.ORCHESTRATORS,
):
click.secho(
f"--custom is not supported for {ctx.invoked_subcommand}")
raise click.Abort()
add_service = ProjectAddCustomService(project)
else:
add_service = ProjectAddService(project)
plugins = [
add_plugin(project, plugin_type, plugin_name, add_service=add_service)
for plugin_name in plugin_names
]
related_plugin_types = [PluginType.FILES]
if flags["include_related"]:
related_plugin_types = list(PluginType)
related_plugins = add_related_plugins(project,
plugins,
add_service=add_service,
plugin_types=related_plugin_types)
plugins.extend(related_plugins)
success = install_plugins(project, plugins, reason=PluginInstallReason.ADD)
for plugin in plugins: # TODO: Only works on Plugin from discovery...
docs_link = plugin._extras.get("docs")
if docs_link:
click.echo(
f"For more details about {plugin.type.descriptor} '{plugin.name}', visit {docs_link}"
)
if not success:
raise click.Abort()
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
# We use the DiscoveryService rather than the ConfigService because the
# plugin may not actually be installed yet at this point.
discovery = PluginDiscoveryService(project)
plugin = discovery.find_plugin(plugin_type, plugin_name)
add_service = ProjectAddService(project)
related_plugins = add_service.add_related(plugin)
install_service = PluginInstallService(project)
install_service.install_plugins(related_plugins)
return jsonify([plugin.canonical() for plugin in related_plugins])
def add(ctx, project, plugin_type, plugin_name, **flags):
if flags["custom"]:
if plugin_type in ("transformer", "transform", "orchestrator"):
click.secho(
f"--custom is not supported for {ctx.invoked_subcommand}")
raise click.Abort()
add_service = ProjectAddCustomService(project)
else:
add_service = ProjectAddService(project)
add_plugin(
add_service,
project,
PluginType(f"{plugin_type}s"),
plugin_name,
include_related=flags["include_related"],
)
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
compiler = ProjectCompiler(project)
install_service = PluginInstallService(project)
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type)
run_venv = install_service.create_venv(plugin)
run_install_plugin = install_service.install_plugin(plugin)
if plugin_type is PluginType.MODELS:
try:
compiler.compile()
except Exception as e:
pass
return jsonify(plugin.canonical())
def discover(project, plugin_type):
discover_service = PluginDiscoveryService(project)
if plugin_type == "all":
plugin_types = list(PluginType)
else:
plugin_types = [PluginType.from_cli_argument(plugin_type)]
for i, plugin_type in enumerate(plugin_types):
if i > 0:
click.echo()
click.secho(f"{str(plugin_type).capitalize()}", fg="green")
for plugin_def in discover_service.get_plugins_of_type(plugin_type):
click.echo(plugin_def.name, nl=False)
if len(plugin_def.variants) > 1:
click.echo(f", variants: {plugin_def.variant_labels}")
else:
click.echo()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_discover(plugin_type=plugin_type)
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
from .params import project
from meltano.core.db import project_engine
from meltano.core.project import Project
from meltano.core.plugin import PluginType
from meltano.core.config_service import ConfigService
from meltano.core.plugin.settings_service import (
PluginSettingsService,
SettingValueSource,
SettingValueStore,
)
@cli.group(invoke_without_command=True)
@click.option("--plugin-type",
type=click.Choice(PluginType.cli_arguments()),
default=None)
@click.argument("plugin_name")
@click.option("--format", type=click.Choice(["json", "env"]), default="json")
@project(migrate=True)
@click.pass_context
def config(ctx, project, plugin_type, plugin_name, format):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
config = ConfigService(project)
plugin = config.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
_, Session = project_engine(project)
from urllib.parse import urlparse
import click
import yaml
from meltano.core.plugin import PluginType
from meltano.core.plugin_install_service import PluginInstallReason
from meltano.core.project_add_service import ProjectAddService
from meltano.core.project_plugins_service import ProjectPluginsService
from . import cli
from .params import pass_project
from .utils import CliError, add_plugin, add_related_plugins, install_plugins
@cli.command()
@click.argument("plugin_type", type=click.Choice(PluginType.cli_arguments()))
@click.argument("plugin_name", nargs=-1, required=True)
@click.option("--inherit-from")
@click.option("--variant")
@click.option("--as", "as_name")
@click.option("--custom", is_flag=True)
@click.option("--include-related", is_flag=True)
@pass_project()
@click.pass_context
def add(
ctx,
project,
plugin_type,
plugin_name,
inherit_from=None,
variant=None,
def add(
ctx,
project,
plugin_type,
plugin_name,
inherit_from=None,
variant=None,
as_name=None,
**flags,
):
"""Add a plugin to your project."""
plugin_type = PluginType.from_cli_argument(plugin_type)
plugin_names = plugin_name # nargs=-1
if as_name:
# `add <type> <inherit-from> --as <name>``
# is equivalent to:
# `add <type> <name> --inherit-from <inherit-from>``
inherit_from = plugin_names[0]
plugin_names = [as_name]
plugins_service = ProjectPluginsService(project)
if flags["custom"]:
if plugin_type in (
PluginType.TRANSFORMERS,
PluginType.TRANSFORMS,
PluginType.ORCHESTRATORS,
):
raise CliError(f"--custom is not supported for {plugin_type}")
add_service = ProjectAddService(project, plugins_service=plugins_service)
plugins = [
add_plugin(
project,
plugin_type,
plugin_name,
inherit_from=inherit_from,
variant=variant,
custom=flags["custom"],
add_service=add_service,
) for plugin_name in plugin_names
]
related_plugin_types = [PluginType.FILES]
if flags["include_related"]:
related_plugin_types = list(PluginType)
related_plugins = add_related_plugins(project,
plugins,
add_service=add_service,
plugin_types=related_plugin_types)
plugins.extend(related_plugins)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
plugins.reverse()
success = install_plugins(project, plugins, reason=PluginInstallReason.ADD)
if not success:
raise CliError("Failed to install plugin(s)")
_print_plugins(plugins)
import click
from meltano.core.plugin import PluginType
from meltano.core.project_add_service import ProjectAddService
from meltano.core.project_plugins_service import ProjectPluginsService
from meltano.core.tracking import GoogleAnalyticsTracker
from . import cli
from .params import pass_project
from .utils import CliError, add_related_plugins, install_plugins
@cli.command()
@click.argument(
"plugin_type", type=click.Choice(PluginType.cli_arguments()), required=False
)
@click.argument("plugin_name", nargs=-1, required=False)
@click.option("--include-related", is_flag=True)
@pass_project(migrate=True)
def install(project, plugin_type, plugin_name, include_related):
"""
Installs all the dependencies of your project based on the meltano.yml file.
Read more at https://www.meltano.com/docs/command-line-interface.html.
"""
plugins_service = ProjectPluginsService(project)
if plugin_type:
plugin_type = PluginType.from_cli_argument(plugin_type)
plugins = plugins_service.get_plugins_of_type(plugin_type)
if plugin_name:
plugins = [p for p in plugins if p.name in plugin_name]
else:
