def test_discovery_yaml(self, subject): # test for all discovery = subject.discover() # raw yaml load for plugin_type, plugin_defs in subject._discovery: if not PluginType.value_exists(plugin_type): continue plugin_type = PluginType(plugin_type) plugin_names = [plugin["name"] for plugin in plugin_defs] assert plugin_type in discovery assert sorted(discovery[plugin_type]) == sorted(plugin_names)
def test_discovery_yaml(self, subject): plugins_by_type = subject.plugins_by_type() # raw yaml load for plugin_type, raw_plugin_defs in subject._discovery: if not PluginType.value_exists(plugin_type): continue plugin_type = PluginType(plugin_type) plugin_names = [ plugin.name for plugin in plugins_by_type[plugin_type] ] for raw_plugin_def in raw_plugin_defs: assert raw_plugin_def["name"] in plugin_names
def install(project, plugin_type, plugin_name, include_related): """ Installs all the dependencies of your project based on the meltano.yml file. Read more at https://www.meltano.com/docs/command-line-interface.html. """ plugins_service = ProjectPluginsService(project) if plugin_type: plugin_type = PluginType.from_cli_argument(plugin_type) plugins = plugins_service.get_plugins_of_type(plugin_type) if plugin_name: plugins = [p for p in plugins if p.name in plugin_name] else: plugins = list(plugins_service.plugins()) if include_related: add_service = ProjectAddService(project, plugins_service=plugins_service) related_plugins = add_related_plugins(project, plugins, add_service=add_service) plugins.extend(related_plugins) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. plugins.reverse() click.echo(f"Installing {len(plugins)} plugins...") success = install_plugins(project, plugins) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_install() if not success: raise CliError("Failed to install plugin(s)")
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() plugins_service = ProjectPluginsService(project) plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type) add_service = ProjectAddService(project, plugins_service=plugins_service) related_plugins = add_service.add_related(plugin) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. related_plugins.reverse() install_service = PluginInstallService(project, plugins_service=plugins_service) install_status = install_service.install_plugins( related_plugins, reason=PluginInstallReason.ADD) for error in install_status["errors"]: raise PluginInstallError(error["message"]) return jsonify([plugin.canonical() for plugin in related_plugins])
def invoke(project, plugin_type, plugin_name, plugin_args): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None _, Session = project_engine(project) session = Session() try: config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type, invokable=True) service = invoker_factory(project, plugin, prepare_with_session=session) handle = service.invoke(*plugin_args) exit_code = handle.wait() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_invoke(plugin_name=plugin_name, plugin_args=" ".join(plugin_args)) sys.exit(exit_code) except Exception as err: logging.exception(err) click.secho(f"An error occured: {err}.", fg="red") raise click.Abort() from err finally: session.close()
def install(project, plugin_type, plugin_name, include_related): """ Installs all the dependencies of your project based on the meltano.yml file. Read more at https://www.meltano.com/docs/command-line-interface.html. """ config_service = ConfigService(project) if plugin_type: plugin_type = PluginType.from_cli_argument(plugin_type) plugins = config_service.get_plugins_of_type(plugin_type) if plugin_name: plugins = [p for p in plugins if p.name in plugin_name] else: plugins = list(config_service.plugins()) if include_related: add_service = ProjectAddService(project, config_service=config_service) related_plugins = add_related_plugins(project, plugins, add_service=add_service) plugins.extend(related_plugins) click.echo(f"Installing {len(plugins)} plugins...") success = install_plugins(project, plugins) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_install() if not success: raise click.Abort()
def invoke(project, plugin_type, dump, plugin_name, plugin_args): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None _, Session = project_engine(project) session = Session() plugins_service = ProjectPluginsService(project) plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type, invokable=True) try: invoker = invoker_factory(project, plugin, plugins_service=plugins_service) with invoker.prepared(session): if dump: dump_file(invoker, dump) exit_code = 0 else: handle = invoker.invoke(*plugin_args) exit_code = handle.wait() except SubprocessError as err: logger.error(err.stderr) raise finally: session.close() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_invoke(plugin_name=plugin_name, plugin_args=" ".join(plugin_args)) sys.exit(exit_code)
def config(ctx, project, plugin_type, plugin_name, format): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None config = ConfigService(project) plugin = config.find_plugin(plugin_name, plugin_type=plugin_type, configurable=True) _, Session = project_engine(project) session = Session() try: settings = PluginSettingsService(project).build(plugin) ctx.obj["settings"] = settings ctx.obj["session"] = session if ctx.invoked_subcommand is None: if format == "json": config = settings.as_config(session=session) print(json.dumps(config)) elif format == "env": for env, value in settings.as_env(session=session).items(): print(f"{env}={value}") finally: session.close()
def add(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() add_service = ProjectAddService(project) plugin = add_service.add(plugin_type, plugin_name) return jsonify(plugin.canonical())
def config(ctx, project, plugin_type, plugin_name, format, extras): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None plugins_service = ProjectPluginsService(project) try: plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type, configurable=True) except PluginNotFoundError: if plugin_name == "meltano": plugin = None else: raise _, Session = project_engine(project) session = Session() try: if plugin: settings = PluginSettingsService(project, plugin, plugins_service=plugins_service) else: settings = ProjectSettingsService( project, config_service=plugins_service.config_service) ctx.obj["settings"] = settings ctx.obj["session"] = session if ctx.invoked_subcommand is None: if format == "json": process = extras is not True config = settings.as_dict(extras=extras, process=process, session=session) print(json.dumps(config, indent=2)) elif format == "env": env = settings.as_env(extras=extras, session=session) with tempfile.NamedTemporaryFile() as temp_dotenv: path = temp_dotenv.name for key, value in env.items(): dotenv.set_key(path, key, value) dotenv_content = Path(temp_dotenv.name).read_text() print(dotenv_content, end="") finally: session.close()
def install(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type) install_service = PluginInstallService(project) install_service.install_plugin(plugin) return jsonify(plugin.canonical())
def install(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() plugins_service = ProjectPluginsService(project) plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type) install_service = PluginInstallService(project, plugins_service=plugins_service) install_service.install_plugin(plugin, reason=PluginInstallReason.ADD) return jsonify(plugin.canonical())
def load_plugins(self, plugins) -> Canonical: """Parse the meltano.yml file and return it as `ProjectPlugin` instances.""" plugin_type_plugins = Canonical() for plugin_type in PluginType: plugin_type_plugins[plugin_type] = [] # this will parse the meltano.yml file and create an instance of the # corresponding `plugin_class` for all the plugins. for plugin_type, raw_plugins in plugins.items(): for raw_plugin in raw_plugins: plugin = ProjectPlugin(PluginType(plugin_type), **raw_plugin) plugin_type_plugins[plugin.type].append(plugin) return plugin_type_plugins
def add(ctx, project, plugin_type, plugin_name, **flags): plugin_type = PluginType.from_cli_argument(plugin_type) plugin_names = plugin_name # nargs=-1 if flags["custom"]: if plugin_type in ( PluginType.TRANSFORMERS, PluginType.TRANSFORMS, PluginType.ORCHESTRATORS, ): click.secho( f"--custom is not supported for {ctx.invoked_subcommand}") raise click.Abort() add_service = ProjectAddCustomService(project) else: add_service = ProjectAddService(project) plugins = [ add_plugin(project, plugin_type, plugin_name, add_service=add_service) for plugin_name in plugin_names ] related_plugin_types = [PluginType.FILES] if flags["include_related"]: related_plugin_types = list(PluginType) related_plugins = add_related_plugins(project, plugins, add_service=add_service, plugin_types=related_plugin_types) plugins.extend(related_plugins) success = install_plugins(project, plugins, reason=PluginInstallReason.ADD) for plugin in plugins: # TODO: Only works on Plugin from discovery... docs_link = plugin._extras.get("docs") if docs_link: click.echo( f"For more details about {plugin.type.descriptor} '{plugin.name}', visit {docs_link}" ) if not success: raise click.Abort()
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() # We use the DiscoveryService rather than the ConfigService because the # plugin may not actually be installed yet at this point. discovery = PluginDiscoveryService(project) plugin = discovery.find_plugin(plugin_type, plugin_name) add_service = ProjectAddService(project) related_plugins = add_service.add_related(plugin) install_service = PluginInstallService(project) install_service.install_plugins(related_plugins) return jsonify([plugin.canonical() for plugin in related_plugins])
def add(ctx, project, plugin_type, plugin_name, **flags): if flags["custom"]: if plugin_type in ("transformer", "transform", "orchestrator"): click.secho( f"--custom is not supported for {ctx.invoked_subcommand}") raise click.Abort() add_service = ProjectAddCustomService(project) else: add_service = ProjectAddService(project) add_plugin( add_service, project, PluginType(f"{plugin_type}s"), plugin_name, include_related=flags["include_related"], ) tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_add(plugin_type=plugin_type, plugin_name=plugin_name)
def install(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() compiler = ProjectCompiler(project) install_service = PluginInstallService(project) config_service = ConfigService(project) plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type) run_venv = install_service.create_venv(plugin) run_install_plugin = install_service.install_plugin(plugin) if plugin_type is PluginType.MODELS: try: compiler.compile() except Exception as e: pass return jsonify(plugin.canonical())
def discover(project, plugin_type): discover_service = PluginDiscoveryService(project) if plugin_type == "all": plugin_types = list(PluginType) else: plugin_types = [PluginType.from_cli_argument(plugin_type)] for i, plugin_type in enumerate(plugin_types): if i > 0: click.echo() click.secho(f"{str(plugin_type).capitalize()}", fg="green") for plugin_def in discover_service.get_plugins_of_type(plugin_type): click.echo(plugin_def.name, nl=False) if len(plugin_def.variants) > 1: click.echo(f", variants: {plugin_def.variant_labels}") else: click.echo() tracker = GoogleAnalyticsTracker(project) tracker.track_meltano_discover(plugin_type=plugin_type)
def install_batch(): payload = request.get_json() plugin_type = PluginType(payload["plugin_type"]) plugin_name = payload["name"] project = Project.find() discovery = PluginDiscoveryService(project) target_plugin = discovery.find_plugin(plugin_type, plugin_name) config_service = ConfigService(project) add_service = ProjectAddService(project) install_service = PluginInstallService(project) ignored_types = [target_plugin.type, PluginType.TRANSFORMS] has_model = False batched = [] for plugin in discovery.plugins(): if plugin.namespace == target_plugin.namespace: if plugin.type not in ignored_types: add_service.add(plugin.type, plugin.name) plugin_install = config_service.find_plugin( plugin.name, plugin_type=plugin.type) batched.append(plugin_install.canonical()) run_venv = install_service.create_venv(plugin_install) run_install_plugin = install_service.install_plugin( plugin_install) if plugin.type is PluginType.MODELS: has_model = True if has_model: compiler = ProjectCompiler(project) try: compiler.compile() except Exception as e: pass return jsonify(batched)
from .params import project from meltano.core.db import project_engine from meltano.core.project import Project from meltano.core.plugin import PluginType from meltano.core.config_service import ConfigService from meltano.core.plugin.settings_service import ( PluginSettingsService, SettingValueSource, SettingValueStore, ) @cli.group(invoke_without_command=True) @click.option("--plugin-type", type=click.Choice(PluginType.cli_arguments()), default=None) @click.argument("plugin_name") @click.option("--format", type=click.Choice(["json", "env"]), default="json") @project(migrate=True) @click.pass_context def config(ctx, project, plugin_type, plugin_name, format): plugin_type = PluginType.from_cli_argument( plugin_type) if plugin_type else None config = ConfigService(project) plugin = config.find_plugin(plugin_name, plugin_type=plugin_type, configurable=True) _, Session = project_engine(project)
from urllib.parse import urlparse import click import yaml from meltano.core.plugin import PluginType from meltano.core.plugin_install_service import PluginInstallReason from meltano.core.project_add_service import ProjectAddService from meltano.core.project_plugins_service import ProjectPluginsService from . import cli from .params import pass_project from .utils import CliError, add_plugin, add_related_plugins, install_plugins @cli.command() @click.argument("plugin_type", type=click.Choice(PluginType.cli_arguments())) @click.argument("plugin_name", nargs=-1, required=True) @click.option("--inherit-from") @click.option("--variant") @click.option("--as", "as_name") @click.option("--custom", is_flag=True) @click.option("--include-related", is_flag=True) @pass_project() @click.pass_context def add( ctx, project, plugin_type, plugin_name, inherit_from=None, variant=None,
def add( ctx, project, plugin_type, plugin_name, inherit_from=None, variant=None, as_name=None, **flags, ): """Add a plugin to your project.""" plugin_type = PluginType.from_cli_argument(plugin_type) plugin_names = plugin_name # nargs=-1 if as_name: # `add <type> <inherit-from> --as <name>`` # is equivalent to: # `add <type> <name> --inherit-from <inherit-from>`` inherit_from = plugin_names[0] plugin_names = [as_name] plugins_service = ProjectPluginsService(project) if flags["custom"]: if plugin_type in ( PluginType.TRANSFORMERS, PluginType.TRANSFORMS, PluginType.ORCHESTRATORS, ): raise CliError(f"--custom is not supported for {plugin_type}") add_service = ProjectAddService(project, plugins_service=plugins_service) plugins = [ add_plugin( project, plugin_type, plugin_name, inherit_from=inherit_from, variant=variant, custom=flags["custom"], add_service=add_service, ) for plugin_name in plugin_names ] related_plugin_types = [PluginType.FILES] if flags["include_related"]: related_plugin_types = list(PluginType) related_plugins = add_related_plugins(project, plugins, add_service=add_service, plugin_types=related_plugin_types) plugins.extend(related_plugins) # We will install the plugins in reverse order, since dependencies # are listed after their dependents in `related_plugins`, but should # be installed first. plugins.reverse() success = install_plugins(project, plugins, reason=PluginInstallReason.ADD) if not success: raise CliError("Failed to install plugin(s)") _print_plugins(plugins)
import click from meltano.core.plugin import PluginType from meltano.core.project_add_service import ProjectAddService from meltano.core.project_plugins_service import ProjectPluginsService from meltano.core.tracking import GoogleAnalyticsTracker from . import cli from .params import pass_project from .utils import CliError, add_related_plugins, install_plugins @cli.command() @click.argument( "plugin_type", type=click.Choice(PluginType.cli_arguments()), required=False ) @click.argument("plugin_name", nargs=-1, required=False) @click.option("--include-related", is_flag=True) @pass_project(migrate=True) def install(project, plugin_type, plugin_name, include_related): """ Installs all the dependencies of your project based on the meltano.yml file. Read more at https://www.meltano.com/docs/command-line-interface.html. """ plugins_service = ProjectPluginsService(project) if plugin_type: plugin_type = PluginType.from_cli_argument(plugin_type) plugins = plugins_service.get_plugins_of_type(plugin_type) if plugin_name: plugins = [p for p in plugins if p.name in plugin_name] else: