def with_transform(self, transform: str): if transform == "skip": return self if transform not in ("run", "only"): self._transform = PluginRef(PluginType.TRANSFORMS, transform) return self.with_transformer("dbt")
def with_transformer(self, transformer_name: str): self._transformer = PluginRef(PluginType.TRANSFORMERS, transformer_name) return self
def with_loader(self, loader_name: str): self._loader = PluginRef(PluginType.LOADERS, loader_name) return self
def with_extractor(self, extractor_name: str): self._extractor = PluginRef(PluginType.EXTRACTORS, extractor_name) return self
class TestCliAdd: @pytest.mark.parametrize( "plugin_type,plugin_name,default_variant,related_plugin_refs", [ (PluginType.EXTRACTORS, "tap-carbon-intensity", "meltano", []), (PluginType.LOADERS, "target-sqlite", "meltano", []), ( PluginType.TRANSFORMS, "tap-carbon-intensity", "meltano", [ PluginRef(PluginType.TRANSFORMERS, "dbt"), PluginRef(PluginType.FILES, "dbt"), ], ), (PluginType.MODELS, "model-carbon-intensity", "meltano", []), (PluginType.DASHBOARDS, "dashboard-google-analytics", "meltano", []), ( PluginType.ORCHESTRATORS, "airflow", Variant.ORIGINAL_NAME, [PluginRef(PluginType.FILES, "airflow")], ), # Installed automatically because of transform 'tap-carbon-intensity' # (PluginType.TRANSFORMERS, "dbt", Variant.ORIGINAL_NAME, [PluginRef(PluginType.FILES, "dbt")]), ], ) def test_add( self, plugin_type, plugin_name, default_variant, related_plugin_refs, project, cli_runner, project_plugins_service, ): # ensure the plugin is not present with pytest.raises(PluginNotFoundError): project_plugins_service.find_plugin(plugin_name, plugin_type=plugin_type) with mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True res = cli_runner.invoke(cli, ["add", plugin_type.singular, plugin_name]) assert res.exit_code == 0, res.stdout assert f"Added {plugin_type.descriptor} '{plugin_name}'" in res.stdout plugin = project_plugins_service.find_plugin(plugin_name, plugin_type) assert plugin assert plugin.variant == default_variant plugins = [plugin] for related_plugin_ref in related_plugin_refs: plugin = project_plugins_service.get_plugin(related_plugin_ref) assert plugin assert ( f"Added related {plugin.type.descriptor} '{plugin.name}'" in res.stdout ) plugins.append(plugin) plugins.reverse() install_plugin_mock.assert_called_once_with( project, plugins, reason=PluginInstallReason.ADD ) def test_add_multiple(self, project, cli_runner, project_plugins_service): with mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True cli_runner.invoke(cli, ["add", "extractors", "tap-gitlab"]) with mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: res = cli_runner.invoke( cli, ["add", "extractors", "tap-gitlab", "tap-adwords", "tap-facebook"] ) assert res.exit_code == 0, res.stdout assert ( "Extractor 'tap-gitlab' already exists in your Meltano project" in res.stderr ) assert "Added extractor 'tap-adwords'" in res.stdout assert "Added extractor 'tap-facebook'" in res.stdout tap_gitlab = project_plugins_service.find_plugin( "tap-gitlab", PluginType.EXTRACTORS ) assert tap_gitlab tap_adwords = project_plugins_service.find_plugin( "tap-adwords", PluginType.EXTRACTORS ) assert tap_adwords tap_facebook = project_plugins_service.find_plugin( "tap-facebook", PluginType.EXTRACTORS ) assert tap_facebook install_plugin_mock.assert_called_once_with( project, [tap_facebook, tap_adwords, tap_gitlab], reason=PluginInstallReason.ADD, ) def test_add_transform(self, project, cli_runner): cli_runner.invoke(cli, ["add", "files", "dbt"]) cli_runner.invoke(cli, ["install", "files", "dbt"]) res = cli_runner.invoke(cli, ["add", "transform", "tap-google-analytics"]) assert_cli_runner(res) with project.root_dir("transform/packages.yml").open() as packages_file: packages_yaml = yaml.safe_load(packages_file) with project.root_dir("transform/dbt_project.yml").open() as project_file: project_yaml = yaml.safe_load(project_file) assert { "git": "https://gitlab.com/meltano/dbt-tap-google-analytics.git" } in packages_yaml["packages"] assert "tap_google_analytics" in project_yaml["models"] assert project_yaml["models"]["tap_google_analytics"] == { "vars": {"schema": "{{ env_var('DBT_SOURCE_SCHEMA') }}"} } def test_add_dashboard(self, project, cli_runner): def install(): return cli_runner.invoke( cli, ["add", "dashboard", "dashboard-google-analytics"] ) res = install() assert res.exit_code == 0 dashboards_service = DashboardsService(project) dashboards_count = len(dashboards_service.get_dashboards()) assert dashboards_count > 0 reports_service = ReportsService(project) reports_count = len(reports_service.get_reports()) assert reports_count > 0 # Verify that reinstalling doesn't duplicate dashboards and reports res = install() assert res.exit_code == 0 assert len(dashboards_service.get_dashboards()) == dashboards_count assert len(reports_service.get_reports()) == reports_count def test_add_files_with_updates( self, project, cli_runner, project_plugins_service, plugin_settings_service_factory, ): result = cli_runner.invoke(cli, ["add", "files", "airflow"]) assert_cli_runner(result) # Plugin has been added to meltano.yml plugin = project_plugins_service.find_plugin("airflow", PluginType.FILES) assert plugin # Automatic updating is enabled plugin_settings_service = plugin_settings_service_factory(plugin) update_config = plugin_settings_service.get("_update") assert update_config["orchestrate/dags/meltano.py"] == True # File has been created assert "Created orchestrate/dags/meltano.py" in result.output file_path = project.root_dir("orchestrate/dags/meltano.py") assert file_path.is_file() # File has "managed" header assert ( "This file is managed by the 'airflow' file bundle" in file_path.read_text() ) def test_add_files_without_updates( self, project, cli_runner, project_plugins_service ): result = cli_runner.invoke(cli, ["add", "files", "docker-compose"]) assert_cli_runner(result) # Plugin has not been added to meltano.yml with pytest.raises(PluginNotFoundError): project_plugins_service.find_plugin("docker-compose", PluginType.FILES) # File has been created assert "Created docker-compose.yml" in result.output file_path = project.root_dir("docker-compose.yml") assert file_path.is_file() # File does not have "managed" header assert "This file is managed" not in file_path.read_text() def test_add_files_that_already_exists( self, project, cli_runner, project_plugins_service ): project.root_dir("transform/dbt_project.yml").write_text("Exists!") result = cli_runner.invoke(cli, ["add", "files", "dbt"]) assert_cli_runner(result) assert ( "File transform/dbt_project.yml already exists, keeping both versions" in result.output ) assert "Created transform/dbt_project (dbt).yml" in result.output assert project.root_dir("transform/dbt_project (dbt).yml").is_file() def test_add_related(self, project, cli_runner, project_plugins_service): # Add dbt and transform/ files cli_runner.invoke(cli, ["add", "transformer", "dbt"]) cli_runner.invoke(cli, ["add", "files", "dbt"]) with mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True res = cli_runner.invoke( cli, ["add", "--include-related", "extractor", "tap-gitlab"] ) assert res.exit_code == 0 tap = project_plugins_service.find_plugin( "tap-gitlab", PluginType.EXTRACTORS ) assert tap transform = project_plugins_service.find_plugin( "tap-gitlab", PluginType.TRANSFORMS ) assert transform model = project_plugins_service.find_plugin( "model-gitlab", PluginType.MODELS ) assert model dashboard = project_plugins_service.find_plugin( "dashboard-gitlab", PluginType.DASHBOARDS ) assert dashboard install_plugin_mock.assert_called_once_with( project, [dashboard, model, transform, tap], reason=PluginInstallReason.ADD, ) def test_add_missing(self, project, cli_runner, project_plugins_service): res = cli_runner.invoke(cli, ["add", "extractor", "tap-unknown"]) assert res.exit_code == 1 assert res.exception assert str(res.exception) == "Extractor 'tap-unknown' is not known to Meltano" # ensure the plugin is not present with pytest.raises(PluginNotFoundError): project_plugins_service.find_plugin("tap-unknown", PluginType.EXTRACTORS) @pytest.mark.xfail(reason="Uninstall not implemented yet.") def test_add_fails(self, project, cli_runner, project_plugins_service): res = cli_runner.invoke(cli, ["add", "extractor", "tap-mock"]) assert res.exit_code == 1, res.stdout assert "Failed to install plugin 'tap-mock'" in res.stdout assert res.stderr # ensure the plugin is not present with pytest.raises(PluginNotFoundError): project_plugins_service.find_plugin("tap-mock", PluginType.EXTRACTORS) def test_add_variant(self, project, cli_runner, project_plugins_service): with mock.patch( "meltano.cli.add.ProjectPluginsService", return_value=project_plugins_service, ), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True res = cli_runner.invoke( cli, ["add", "extractor", "tap-mock", "--variant", "singer-io"] ) assert_cli_runner(res) plugin = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock" ) assert plugin.variant == "singer-io" def test_add_inherited( self, project, tap, cli_runner, project_plugins_service, plugin_discovery_service, ): # Make sure tap-mock is not in the project as a project plugin project_plugins_service.remove_from_file(tap) with mock.patch( "meltano.cli.add.ProjectPluginsService", return_value=project_plugins_service, ), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True # Inheriting from a BasePlugin using --as res = cli_runner.invoke( cli, [ "add", "extractor", "tap-mock", "--as", "tap-mock-inherited", ], ) assert_cli_runner(res) assert "Inherit from:\ttap-mock, variant meltano (default)\n" in res.stdout inherited = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock-inherited" ) assert inherited.inherit_from == "tap-mock" assert inherited.variant == "meltano" assert inherited.parent == plugin_discovery_service.find_base_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock", variant="meltano", ) # Inheriting from a BasePlugin using --inherit-from and --variant res = cli_runner.invoke( cli, [ "add", "extractor", "tap-mock--singer-io", "--inherit-from", "tap-mock", "--variant", "singer-io", ], ) assert_cli_runner(res) assert ( "Inherit from:\ttap-mock, variant singer-io (deprecated)\n" in res.stdout ) inherited_variant = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock--singer-io" ) assert inherited_variant.inherit_from == "tap-mock" assert inherited_variant.variant == "singer-io" assert ( inherited_variant.parent == plugin_discovery_service.find_base_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock", variant="singer-io", ) ) # Inheriting from a ProjectPlugin using --inherit-from res = cli_runner.invoke( cli, [ "add", "extractor", "tap-mock-inception", "--inherit-from", "tap-mock-inherited", ], ) assert_cli_runner(res) assert "Inherit from:\ttap-mock-inherited\n" in res.stdout inception = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock-inception" ) assert inception.inherit_from == "tap-mock-inherited" assert inception.parent == inherited # Inheriting from a nonexistent plugin res = cli_runner.invoke( cli, [ "add", "extractor", "tap-foo", "--inherit-from", "tap-bar", ], ) assert res.exit_code == 1 assert ( "Could not find parent plugin for extractor 'tap-foo': Extractor 'tap-bar' is not known to Meltano" in str(res.exception) ) def test_add_custom(self, project, cli_runner, project_plugins_service): pip_url = "-e path/to/tap-custom" executable = "tap-custom-bin" stdin = os.linesep.join( # namespace, pip_url, executable, capabilities, settings ["tap_custom", pip_url, executable, "foo,bar", "baz,qux"] ) with mock.patch( "meltano.cli.add.ProjectPluginsService", return_value=project_plugins_service, ), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True res = cli_runner.invoke( cli, ["add", "--custom", "extractor", "tap-custom"], input=stdin ) assert_cli_runner(res) plugin = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-custom" ) assert plugin.name == "tap-custom" assert plugin.pip_url == pip_url plugin_def = plugin.custom_definition plugin_variant = plugin.custom_definition.variants[0] assert plugin_def.type == plugin.type assert plugin_def.name == plugin.name == "tap-custom" assert plugin_def.namespace == plugin.namespace == "tap_custom" assert plugin_variant.name is None assert plugin.pip_url == plugin_variant.pip_url == pip_url assert plugin.executable == plugin_variant.executable == executable assert plugin.capabilities == plugin_variant.capabilities == ["foo", "bar"] assert [s.name for s in plugin_variant.settings] == ["baz", "qux"] assert plugin.settings == plugin_variant.settings install_plugin_mock.assert_called_once_with( project, [plugin], reason=PluginInstallReason.ADD ) def test_add_custom_variant( self, project, cli_runner, project_plugins_service, plugin_discovery_service ): with mock.patch( "meltano.cli.add.ProjectPluginsService", return_value=project_plugins_service, ), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock: install_plugin_mock.return_value = True res = cli_runner.invoke( cli, [ "add", "--custom", "extractor", "tap-custom-variant", "--variant", "personal", ], ) assert_cli_runner(res) plugin = project_plugins_service.find_plugin( plugin_type=PluginType.EXTRACTORS, plugin_name="tap-custom-variant" ) plugin_def = plugin.custom_definition plugin_variant = plugin_def.variants[0] assert plugin.variant == plugin_variant.name == "personal"
def runner(self): return PluginRef(PluginType.TRANSFORMERS, "dbt")
def to_python(self, value): return PluginRef(*value.split("/"))