Exemple #1
0
    def parse_packages(self):
        discovery = PluginDiscoveryService(self.project)
        model_plugins = discovery.get_plugins_of_type(PluginType.MODELS)

        for package in self.packages():
            if not package.topics and package.tables:
                raise MeltanoAnalysisMissingTopicFilesError(
                    f"Missing topic file(s) for package {package}")

            for table in package.tables:
                conf = self.parse_m5o_file(table)
                parsed_table = self.table(conf, table.name)
                self.tables.append(parsed_table)

            for topic in package.topics:
                conf = self.parse_m5o_file(topic)
                parsed_topic = self.topic(conf, topic.name)
                model = next(
                    (plugin for plugin in model_plugins
                     if plugin.name == package.name),
                    None,
                )
                parsed_topic[
                    "plugin_namespace"] = model.namespace if model else None
                parsed_topic["namespace"] = package.name
                self.packaged_topics.append(parsed_topic)

            # Reset the tables list so that tables with the same name from
            #  different packages are not interfearing with correctly parsing
            #  the packages that follow
            self.tables = []

        return self.packaged_topics
Exemple #2
0
def add_related_plugins(project,
                        plugins,
                        add_service: ProjectAddService,
                        plugin_types=list(PluginType)):
    discovery_service = PluginDiscoveryService(project)

    added_plugins = []
    for plugin_install in plugins:
        try:
            plugin_def = discovery_service.find_plugin(plugin_install.type,
                                                       plugin_install.name)
        except PluginNotFoundError:
            continue

        related_plugins = add_service.add_related(plugin_def,
                                                  plugin_types=plugin_types)
        for related_plugin in related_plugins:
            if related_plugin.should_add_to_file(project):
                click.secho(
                    f"Added related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project",
                    fg="green",
                )
            else:
                click.secho(
                    f"Adding related {related_plugin.type.descriptor} '{related_plugin.name}' to your Meltano project...",
                    fg="green",
                )

        added_plugins.extend(related_plugins)

    return added_plugins
Exemple #3
0
def installed():
    """Returns JSON of all installed plugins

    Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name
    """

    project = Project.find()
    config = ConfigService(project)
    discovery = PluginDiscoveryService(project)
    installed_plugins = {}

    # merge definitions
    for plugin in sorted(config.plugins(), key=lambda x: x.name):
        try:
            definition = discovery.find_plugin(plugin.type, plugin.name)
            merged_plugin_definition = {
                **definition.canonical(),
                **plugin.canonical()
            }
        except PluginNotFoundError:
            merged_plugin_definition = {**plugin.canonical()}

        merged_plugin_definition.pop("settings", None)
        merged_plugin_definition.pop("select", None)

        if not plugin.type in installed_plugins:
            installed_plugins[plugin.type] = []

        installed_plugins[plugin.type].append(merged_plugin_definition)

    return jsonify({
        **project.meltano.canonical(), "plugins": installed_plugins
    })
Exemple #4
0
def get_plugin_configuration(plugin_ref) -> Response:
    """
    Endpoint for getting a plugin's configuration profiles
    """

    project = Project.find()
    settings = PluginSettingsService(project, show_hidden=False)
    plugin = ConfigService(project).get_plugin(plugin_ref)

    discovery_service = PluginDiscoveryService(project)
    try:
        plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
        settings_group_validation = plugin_def.settings_group_validation
    except PluginNotFoundError:
        settings_group_validation = []

    profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
    for profile in profiles:
        freeze_profile_config_keys(profile)

    return jsonify({
        "profiles":
        profiles,
        "settings":
        Canonical.as_canonical(settings.definitions(plugin)),
        "settings_group_validation":
        settings_group_validation,
    })
Exemple #5
0
def all():
    project = Project.find()
    discovery = PluginDiscoveryService(project)
    ordered_plugins = {}

    for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type):
        ordered_plugins[type] = [plugin.canonical() for plugin in plugins]

    return jsonify(ordered_plugins)
Exemple #6
0
def all():
    project = Project.find()
    discovery = PluginDiscoveryService(project)

    all_plugins = {
        plugin_type:
        [plugin_def_json(plugin_def) for plugin_def in plugin_defs]
        for plugin_type, plugin_defs in discovery.plugins_by_type().items()
    }

    return jsonify(all_plugins)
Exemple #7
0
def discover(project, plugin_type):
    discover_service = PluginDiscoveryService(project)
    try:
        discovery_dict = discover_service.discover(plugin_type)

        for plugin_type, plugins in discovery_dict.items():
            click.secho(plugin_type, fg="green")
            for plugin in plugins:
                click.echo(plugin)

        tracker = GoogleAnalyticsTracker(project)
        tracker.track_meltano_discover(plugin_type=plugin_type)
    except Exception as e:
        click.secho("Cannot list available plugins.", fg="red")
        raise click.ClickException(str(e))
Exemple #8
0
def install_missing_plugins(project: Project, extractor: str, loader: str,
                            transform: str):
    add_service = ProjectAddService(project)
    config_service = ConfigService(project)

    if transform != "only":
        try:
            config_service.find_plugin(extractor,
                                       plugin_type=PluginType.EXTRACTORS)
        except PluginMissingError:
            click.secho(
                f"Extractor '{extractor}' is missing, trying to install it...",
                fg="yellow",
            )
            add_plugin(add_service, project, PluginType.EXTRACTORS, extractor)

        try:
            config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
        except PluginMissingError:
            click.secho(
                f"Loader '{loader}' is missing, trying to install it...",
                fg="yellow")
            add_plugin(add_service, project, PluginType.LOADERS, loader)

    if transform != "skip":
        try:
            config_service.find_plugin("dbt",
                                       plugin_type=PluginType.TRANSFORMERS)
        except PluginMissingError as e:
            click.secho(
                f"Transformer 'dbt' is missing, trying to install it...",
                fg="yellow")
            add_plugin(add_service, project, PluginType.TRANSFORMERS, "dbt")

        transform_add_service = TransformAddService(project)
        try:
            # the extractor name should match the transform name
            plugin = config_service.find_plugin(
                extractor, plugin_type=PluginType.TRANSFORMS)

            # Update dbt_project.yml in case the vars values have changed in meltano.yml
            transform_add_service.update_dbt_project(plugin)
        except PluginMissingError:
            try:
                # Check if there is a default transform for this extractor
                PluginDiscoveryService(project).find_plugin(
                    PluginType.TRANSFORMS, extractor)

                click.secho(
                    f"Transform '{extractor}' is missing, trying to install it...",
                    fg="yellow",
                )
                add_transform(project, extractor)
            except PluginNotFoundError:
                # There is no default transform for this extractor..
                # Don't panic, everything is cool - just run custom transforms
                pass
Exemple #9
0
 def __init__(
     self,
     project,
     config_service: ConfigService = None,
     plugin_discovery_service: PluginDiscoveryService = None,
 ):
     self.project = project
     self.config_service = config_service or ConfigService(project)
     self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
         project)
Exemple #10
0
def install_batch():
    payload = request.get_json()
    plugin_type = PluginType(payload["plugin_type"])
    plugin_name = payload["name"]

    project = Project.find()

    # We use the DiscoveryService rather than the ConfigService because the
    # plugin may not actually be installed yet at this point.
    discovery = PluginDiscoveryService(project)
    plugin = discovery.find_plugin(plugin_type, plugin_name)

    add_service = ProjectAddService(project)
    related_plugins = add_service.add_related(plugin)

    install_service = PluginInstallService(project)
    install_service.install_plugins(related_plugins)

    return jsonify([plugin.canonical() for plugin in related_plugins])
 def __init__(
     self,
     project: Project,
     mail_service: MailService = None,
     plugin_discovery_service: PluginDiscoveryService = None,
 ):
     self.project = project
     self.mail_service = mail_service or MailService(project)
     self.plugin_discovery_service = (plugin_discovery_service
                                      or PluginDiscoveryService(project))
Exemple #12
0
def all():
    project = Project.find()
    discovery = PluginDiscoveryService(project)
    ordered_plugins = {}

    for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type):
        canonical_plugins = []
        for plugin in plugins:
            canonical_plugin = plugin.canonical()

            # let's remove all the settings related data
            canonical_plugin.pop("settings", None)
            canonical_plugin.pop("settings_group_validation", None)

            canonical_plugins.append(canonical_plugin)

        ordered_plugins[type] = canonical_plugins

    return jsonify(ordered_plugins)
Exemple #13
0
def discover(project, plugin_type):
    discover_service = PluginDiscoveryService(project)
    if plugin_type == "all":
        plugin_types = list(PluginType)
    else:
        plugin_types = [PluginType.from_cli_argument(plugin_type)]

    for i, plugin_type in enumerate(plugin_types):
        if i > 0:
            click.echo()

        click.secho(f"{str(plugin_type).capitalize()}", fg="green")

        for plugin_def in discover_service.get_plugins_of_type(plugin_type):
            click.echo(plugin_def.name, nl=False)

            if len(plugin_def.variants) > 1:
                click.echo(f", variants: {plugin_def.variant_labels}")
            else:
                click.echo()

    tracker = GoogleAnalyticsTracker(project)
    tracker.track_meltano_discover(plugin_type=plugin_type)
Exemple #14
0
def install_batch():
    payload = request.get_json()
    plugin_type = PluginType(payload["plugin_type"])
    plugin_name = payload["name"]

    project = Project.find()
    discovery = PluginDiscoveryService(project)
    target_plugin = discovery.find_plugin(plugin_type, plugin_name)

    config_service = ConfigService(project)
    add_service = ProjectAddService(project)
    install_service = PluginInstallService(project)
    ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
    has_model = False
    batched = []
    for plugin in discovery.plugins():
        if plugin.namespace == target_plugin.namespace:
            if plugin.type not in ignored_types:
                add_service.add(plugin.type, plugin.name)
                plugin_install = config_service.find_plugin(
                    plugin.name, plugin_type=plugin.type)
                batched.append(plugin_install.canonical())
                run_venv = install_service.create_venv(plugin_install)
                run_install_plugin = install_service.install_plugin(
                    plugin_install)
                if plugin.type is PluginType.MODELS:
                    has_model = True

    if has_model:
        compiler = ProjectCompiler(project)
        try:
            compiler.compile()
        except Exception as e:
            pass

    return jsonify(batched)
Exemple #15
0
 def __init__(
     self,
     project,
     loader: PluginContext,
     job: Optional[Job] = None,
     extractor: Optional[PluginContext] = None,
     plugin_settings_service: PluginSettingsService = None,
     plugin_discovery_service: PluginDiscoveryService = None,
 ):
     self.project = project
     self.job = job
     self.loader = loader
     self.extractor = extractor
     self.plugin_settings_service = plugin_settings_service
     self.plugin_discovery_service = (plugin_discovery_service
                                      or PluginDiscoveryService(project))
Exemple #16
0
 def __init__(
     self,
     project,
     session,
     job: Optional[Job] = None,
     extractor: Optional[PluginContext] = None,
     loader: Optional[PluginContext] = None,
     transformer: Optional[PluginContext] = None,
     plugin_discovery_service: PluginDiscoveryService = None,
 ):
     self.project = project
     self.job = job
     self.extractor = extractor
     self.loader = loader
     self.transformer = transformer
     self.plugin_discovery_service = (plugin_discovery_service
                                      or PluginDiscoveryService(project))
Exemple #17
0
    def __init__(
        self,
        project,
        config_service: ConfigService = None,
        plugin_discovery_service: PluginDiscoveryService = None,
        show_hidden=True,
        env_override={},
        config_override={},
    ):
        self.project = project
        self.config_service = config_service or ConfigService(project)
        self.discovery_service = plugin_discovery_service or PluginDiscoveryService(
            project
        )
        self.show_hidden = show_hidden
        self.env_override = env_override
        self.config_override = config_override

        self._env = None
Exemple #18
0
 def __init__(
     self,
     project: Project,
     config_service: ConfigService = None,
     plugin_settings_service: PluginSettingsService = None,
     plugin_discovery_service: PluginDiscoveryService = None,
 ):
     self.project = project
     self.config_service = config_service or ConfigService(project)
     self.plugin_discovery_service = (
         plugin_discovery_service
         or PluginDiscoveryService(project, config_service=config_service))
     self.plugin_settings_service = plugin_settings_service or PluginSettingsService(
         project,
         config_service=config_service,
         plugin_discovery_service=plugin_discovery_service,
     )
     self._extractor = None
     self._loader = None
     self._job = None
Exemple #19
0
def plugin_discovery_service(project, discovery, config_service):
    return PluginDiscoveryService(project,
                                  discovery=discovery,
                                  config_service=config_service)
Exemple #20
0
def add_plugin(
    add_service,
    project: Project,
    plugin_type: PluginType,
    plugin_name: str,
    include_related=False,
):
    try:
        plugin = add_service.add(plugin_type, plugin_name)
        click.secho(f"Added '{plugin_name}' to your Meltano project.",
                    fg="green")
    except PluginAlreadyAddedException as err:
        click.secho(
            f"'{plugin_name}' was found in your Meltano project. Use `meltano install` to install it.",
            fg="yellow",
            err=True,
        )
        plugin = err.plugin
    except (PluginNotSupportedException, PluginNotFoundError):
        click.secho(f"Error: {plugin_type} '{plugin_name}' is not supported.",
                    fg="red")
        raise click.Abort()

    install_service = PluginInstallService(project)

    try:
        click.secho(f"Installing '{plugin_name}'...")
        run = install_service.install_plugin(plugin)
        if run:
            click.secho(run.stdout)
        click.secho(f"Installed '{plugin_name}'.", fg="green")

        click.secho(f"Added and installed {plugin_type} '{plugin_name}'.",
                    fg="green")
    except PluginNotInstallable as install_err:
        logging.info(f"{plugin_type} is not installable, skipping install.")
    except SubprocessError as proc_err:
        click.secho(str(proc_err), fg="red")
        click.secho(proc_err.process.stderr, err=True)
        raise click.Abort()

    docs_link = plugin._extras.get("docs")
    if docs_link:
        click.secho(
            f"Visit {docs_link} for more details about '{plugin.name}'.")

    if include_related:
        discovery_service = PluginDiscoveryService(project)
        plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)

        related_plugins = add_service.add_related(plugin_def)
        if len(related_plugins) == 0:
            click.secho(
                "No related plugins found that are not already installed.")
        else:
            for plugin in related_plugins:
                click.secho(
                    f"Added related plugin '{plugin.name}' to your Meltano project.",
                    fg="green",
                )

            click.secho(
                f"Installing {len(related_plugins)} related plugins...")
            install_status = install_service.install_plugins(related_plugins)

            num_installed = len(install_status["installed"])
            num_failed = len(install_status["errors"])

            fg = "green"
            if num_failed >= 0 and num_installed == 0:
                fg = "red"
            elif num_failed > 0 and num_installed > 0:
                fg = "yellow"

            click.secho(
                f"Installed {num_installed}/{num_installed+num_failed} related plugins.",
                fg=fg,
            )
Exemple #21
0
def install_missing_plugins(project: Project, extractor: str, loader: str,
                            transform: str):
    add_service = ProjectAddService(project)
    config_service = ConfigService(project)

    plugins = []
    if transform != "only":
        try:
            config_service.find_plugin(extractor,
                                       plugin_type=PluginType.EXTRACTORS)
        except PluginMissingError:
            click.secho(
                f"Extractor '{extractor}' is missing, trying to install it...",
                fg="yellow",
            )
            plugin = add_plugin(project,
                                PluginType.EXTRACTORS,
                                extractor,
                                add_service=add_service)
            plugins.append(plugin)

        try:
            config_service.find_plugin(loader, plugin_type=PluginType.LOADERS)
        except PluginMissingError:
            click.secho(
                f"Loader '{loader}' is missing, trying to install it...",
                fg="yellow")
            plugin = add_plugin(project,
                                PluginType.LOADERS,
                                loader,
                                add_service=add_service)
            plugins.append(plugin)

    if transform != "skip":
        try:
            config_service.find_plugin("dbt",
                                       plugin_type=PluginType.TRANSFORMERS)
        except PluginMissingError as e:
            click.secho(
                f"Transformer 'dbt' is missing, trying to install it...",
                fg="yellow")
            plugin = add_plugin(project,
                                PluginType.TRANSFORMERS,
                                "dbt",
                                add_service=add_service)
            plugins.append(plugin)

        discovery_service = PluginDiscoveryService(project)
        extractor_plugin_def = discovery_service.find_plugin(
            PluginType.EXTRACTORS, extractor)
        try:
            transform_plugin = config_service.find_plugin_by_namespace(
                extractor_plugin_def.namespace, PluginType.TRANSFORMS)

            # Update dbt_project.yml in case the vars values have changed in meltano.yml
            transform_add_service = TransformAddService(project)
            transform_add_service.update_dbt_project(transform_plugin)
        except PluginMissingError:
            try:
                # Check if there is a default transform for this extractor
                transform_plugin_def = discovery_service.find_plugin_by_namespace(
                    extractor_plugin_def.namespace, PluginType.TRANSFORMS)

                click.secho(
                    f"Transform '{transform_plugin_def.name}' is missing, trying to install it...",
                    fg="yellow",
                )
                add_plugin(
                    project,
                    PluginType.TRANSFORMS,
                    transform_plugin_def.name,
                    add_service=add_service,
                )
                plugins.append(plugin)
            except PluginNotFoundError:
                # There is no default transform for this extractor..
                # Don't panic, everything is cool - just run custom transforms
                pass

    return install_plugins(project, plugins)
Exemple #22
0
def plugin_discovery_service(project, discovery):
    return PluginDiscoveryService(project, discovery=discovery)