def test_schedule(self, get, session, project, cli_runner,
schedule_service):
TEST_DATE = "2010-01-01"
get.return_value = TEST_DATE
with mock.patch("meltano.cli.schedule.ScheduleService",
return_value=schedule_service):
res = cli_runner.invoke(
cli,
[
"schedule",
"schedule-mock",
"tap-mock",
"target-mock",
"@eon",
"--transform",
"run",
],
)
assert_cli_runner(res)
schedule = schedule_service.schedules()[0]
assert schedule.name == "schedule-mock"
assert schedule.extractor == "tap-mock"
assert schedule.loader == "target-mock"
assert schedule.transform == "run"
assert schedule.interval == "@eon" # not anytime soon ;)
assert schedule.start_date == iso8601_datetime(TEST_DATE)
def test_ui_setup(self, project, cli_runner, monkeypatch):
monkeypatch.setenv("MELTANO_UI_SECRET_KEY", "existing_secret_key")
with mock.patch("meltano.cli.ui.secrets.token_hex",
return_value="fake_secret"):
result = cli_runner.invoke(cli,
["ui", "setup", "meltano.example.com"])
assert_cli_runner(result)
assert (
"Setting 'ui.secret_key' has already been set in the environment"
in result.stdout)
settings_service = ProjectSettingsService(project)
assert settings_service.get_with_source("ui.server_name") == (
"meltano.example.com",
SettingValueStore.DOTENV,
)
assert settings_service.get_with_source("ui.secret_key") == (
"existing_secret_key",
SettingValueStore.ENV,
)
assert settings_service.get_with_source("ui.password_salt") == (
"fake_secret",
SettingValueStore.DOTENV,
)
def test_carbon_intensity_sqlite(self, cli_runner, monkeypatch, project):
# Directly run `meltano add loader target-sqlite` in a new project
# So that the extractor and the loader are auto installed by meltano
cli_args = ["elt", "tap-carbon-intensity", "target-sqlite"]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
def test_dump_loader_config(
self,
cli_runner,
project,
tap,
target,
project_plugins_service,
plugin_settings_service_factory,
):
job_id = "pytest_test_elt"
args = [
"elt",
"--job_id",
job_id,
tap.name,
target.name,
"--dump",
"loader-config",
]
settings_service = plugin_settings_service_factory(target)
with mock.patch(
"meltano.cli.elt.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch.object(SingerTap, "discover_catalog"), mock.patch.object(
SingerTap, "apply_catalog_rules"):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
assert json.loads(result.stdout) == settings_service.as_dict(
extras=False, process=True)
def test_carbon_intensity_postgres_dbt(
self, cli_runner, monkeypatch, project, elt_context_builder, project_add_service
):
with mock.patch(
"meltano.cli.add.ProjectAddService", return_value=project_add_service
), mock.patch(
"meltano.cli.elt.ELTContextBuilder", return_value=elt_context_builder
):
cli_args = ["add", "extractor", "tap-carbon-intensity"]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
cli_args = ["add", "loader", "target-postgres"]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
cli_args = ["add", "transformer", "dbt"]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
cli_args = ["add", "model", "model-carbon-intensity"]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
cli_args = [
"elt",
"tap-carbon-intensity",
"target-postgres",
"--transform",
"run",
]
result = cli_runner.invoke(cli, cli_args)
assert_cli_runner(result)
def test_elt_transform_only_with_transform(
self,
google_tracker,
cli_runner,
project,
tap,
target,
dbt,
tap_mock_transform,
project_plugins_service,
):
args = ["elt", tap.name, target.name, "--transform", "only"]
with mock.patch(
"meltano.cli.elt.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch(
"meltano.core.transform_add_service.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch.object(DbtRunner, "run", new=CoroutineMock()):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
assert_lines(
result.output,
"meltano | Extract & load skipped.\n",
"meltano | Running transformation...\n",
"meltano | Transformation complete!\n",
)
def test_upgrade_package(self, project, cli_runner):
result = cli_runner.invoke(cli, ["upgrade", "package"])
assert_cli_runner(result)
assert ("The `meltano` package could not be upgraded automatically"
in result.output)
assert "run `meltano upgrade --skip-package`" not in result.output
def test_add_custom_variant(
self, project, cli_runner, project_plugins_service, plugin_discovery_service
):
with mock.patch(
"meltano.cli.add.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
res = cli_runner.invoke(
cli,
[
"add",
"--custom",
"extractor",
"tap-custom-variant",
"--variant",
"personal",
],
)
assert_cli_runner(res)
plugin = project_plugins_service.find_plugin(
plugin_type=PluginType.EXTRACTORS, plugin_name="tap-custom-variant"
)
plugin_def = plugin.custom_definition
plugin_variant = plugin_def.variants[0]
assert plugin.variant == plugin_variant.name == "personal"
def test_add_files_with_updates(
self,
project,
cli_runner,
project_plugins_service,
plugin_settings_service_factory,
):
result = cli_runner.invoke(cli, ["add", "files", "airflow"])
assert_cli_runner(result)
# Plugin has been added to meltano.yml
plugin = project_plugins_service.find_plugin("airflow", PluginType.FILES)
assert plugin
# Automatic updating is enabled
plugin_settings_service = plugin_settings_service_factory(plugin)
update_config = plugin_settings_service.get("_update")
assert update_config["orchestrate/dags/meltano.py"] == True
# File has been created
assert "Created orchestrate/dags/meltano.py" in result.output
file_path = project.root_dir("orchestrate/dags/meltano.py")
assert file_path.is_file()
# File has "managed" header
assert (
"This file is managed by the 'airflow' file bundle" in file_path.read_text()
)
def test_schedule(self, get_value, session, project, cli_runner,
schedule_service):
TEST_DATE = "2010-01-01"
get_value.return_value = (TEST_DATE, 0)
res = cli_runner.invoke(
cli,
[
"schedule",
"schedule-mock",
"tap-mock",
"target-mock",
"@eon",
"--transform",
"run",
],
)
assert_cli_runner(res)
schedule = next(schedule_service.schedules())
assert schedule.name == "schedule-mock"
assert schedule.extractor == "tap-mock"
assert schedule.loader == "target-mock"
assert schedule.transform == "run"
assert schedule.interval == "@eon" # not anytime soon ;)
assert schedule.start_date == iso8601_datetime(TEST_DATE)
def test_dump_catalog(
self,
cli_runner,
project,
tap,
target,
project_plugins_service,
plugin_settings_service_factory,
):
catalog = {"streams": []}
with project.root.joinpath("catalog.json").open("w") as catalog_file:
json.dump(catalog, catalog_file)
job_id = "pytest_test_elt"
args = [
"elt",
"--job_id",
job_id,
tap.name,
target.name,
"--catalog",
"catalog.json",
"--dump",
"catalog",
]
with mock.patch(
"meltano.cli.elt.ProjectPluginsService",
return_value=project_plugins_service,
):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
assert json.loads(result.stdout) == catalog
def test_elt_transform_only(
google_tracker,
cli_runner,
project,
tap,
target,
dbt,
plugin_discovery_service,
plugin_settings_service,
):
# exit cleanly when `meltano elt ... --transform only` runs for
# a tap with no default transforms
args = ["elt", tap.name, target.name, "--transform", "only"]
# fmt: off
with patch("meltano.cli.elt.add_plugin", return_value=None) as add_plugin, \
patch("meltano.cli.elt.PluginDiscoveryService", return_value=plugin_discovery_service), \
patch("meltano.core.elt_context.PluginDiscoveryService", return_value=plugin_discovery_service), \
patch("meltano.core.elt_context.PluginSettingsService", return_value=plugin_settings_service), \
patch.object(DbtRunner, "run", return_value=None):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
add_plugin.assert_called
def test_elt_transform_run(
self,
google_tracker,
cli_runner,
project,
tap,
target,
dbt,
tap_process,
target_process,
silent_dbt_process,
dbt_process,
tap_mock_transform,
project_plugins_service,
):
args = ["elt", tap.name, target.name, "--transform", "run"]
invoke_async = CoroutineMock(side_effect=(
tap_process,
target_process,
silent_dbt_process, # dbt clean
silent_dbt_process, # dbt deps
dbt_process, # dbt run
))
with mock.patch.object(
PluginInvoker, "invoke_async",
new=invoke_async) as invoke_async, mock.patch(
"meltano.cli.elt.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch(
"meltano.core.transform_add_service.ProjectPluginsService",
return_value=project_plugins_service,
):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
assert_lines(
result.stdout,
"meltano | Running extract & load...\n",
"meltano | Extract & load complete!\n",
"meltano | Running transformation...\n",
"meltano | Transformation complete!\n",
)
assert_lines(
result.stderr,
"tap-mock | Starting\n",
"tap-mock | Running\n",
"tap-mock | Done\n",
"target-mock | Starting\n",
"target-mock | Running\n",
"target-mock | Done\n",
"dbt | Starting\n",
"dbt | Running\n",
"dbt | Done\n",
)
def test_config(self, project, cli_runner, tap, project_plugins_service):
with mock.patch(
"meltano.cli.config.ProjectPluginsService",
return_value=project_plugins_service,
):
result = cli_runner.invoke(cli, ["config", tap.name])
assert_cli_runner(result)
json_config = json.loads(result.stdout)
assert json_config["test"] == "mock"
def test_install(self, project, tap, tap_gitlab, target, dbt, cli_runner):
with mock.patch(
"meltano.cli.install.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
result = cli_runner.invoke(cli, ["install"])
assert_cli_runner(result)
install_plugin_mock.assert_called_once_with(
project, [tap, tap_gitlab, target, dbt])
def test_config_env(self, project, cli_runner, tap,
project_plugins_service):
with mock.patch(
"meltano.cli.config.ProjectPluginsService",
return_value=project_plugins_service,
):
result = cli_runner.invoke(cli,
["config", "--format=env", tap.name])
assert_cli_runner(result)
assert 'TAP_MOCK_TEST="mock"' in result.stdout
def test_add_files_that_already_exists(self, project, cli_runner,
config_service):
project.root_dir("transform/dbt_project.yml").write_text("Exists!")
result = cli_runner.invoke(cli, ["add", "files", "dbt"])
assert_cli_runner(result)
assert (
"File transform/dbt_project.yml already exists, keeping both versions"
in result.output)
assert "Created transform/dbt_project (dbt).yml" in result.output
assert project.root_dir("transform/dbt_project (dbt).yml").is_file()
def test_config_meltano(self, project, cli_runner, engine_uri,
project_plugins_service):
with mock.patch(
"meltano.cli.config.ProjectPluginsService",
return_value=project_plugins_service,
):
result = cli_runner.invoke(cli, ["config", "meltano"])
assert_cli_runner(result)
json_config = json.loads(result.stdout)
assert json_config["send_anonymous_usage_stats"] == False
assert json_config["database_uri"] == engine_uri
assert json_config["cli"]["log_level"] == "info"
def test_elt(
self,
google_tracker,
cli_runner,
project,
tap,
target,
plugin_settings_service,
plugin_discovery_service,
job_logging_service,
):
result = cli_runner.invoke(cli, ["elt"])
assert result.exit_code == 2
job_id = "pytest_test_elt"
args = ["elt", "--job_id", job_id, tap.name, target.name]
# exit cleanly when everything is fine
# fmt: off
with patch.object(SingerRunner, "run", return_value=None), \
patch("meltano.cli.elt.install_missing_plugins", return_value=True), \
patch("meltano.core.elt_context.PluginDiscoveryService", return_value=plugin_discovery_service), \
patch("meltano.core.elt_context.PluginSettingsService", return_value=plugin_settings_service):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
# fmt: on
# aborts when there is an exception
with patch("meltano.cli.elt.install_missing_plugins",
return_value=None):
result = cli_runner.invoke(cli, args)
assert result.exit_code == 1
job_logging_service.delete_all_logs(job_id)
with patch.object(
SingerRunner,
"run",
side_effect=Exception("This is a grave danger.")), patch(
"meltano.core.elt_context.PluginDiscoveryService",
return_value=plugin_discovery_service,
), patch(
"meltano.core.elt_context.PluginSettingsService",
return_value=plugin_settings_service,
):
result = cli_runner.invoke(cli, args)
assert result.exit_code == 1
# ensure there is a log of this exception
log = job_logging_service.get_latest_log(job_id)
assert "This is a grave danger.\n" in log
def test_discover_extractors(self, project, cli_runner,
plugin_discovery_service):
with mock.patch(
"meltano.cli.discovery.PluginDiscoveryService",
return_value=plugin_discovery_service,
):
result = cli_runner.invoke(cli, ["discover", "extractors"])
assert_cli_runner(result)
assert "Extractors" in result.output
assert "tap-gitlab" in result.output
assert "tap-mock" in result.output
assert "Loaders" not in result.output
def test_add_variant(self, project, cli_runner, project_plugins_service):
with mock.patch(
"meltano.cli.add.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
res = cli_runner.invoke(
cli, ["add", "extractor", "tap-mock", "--variant", "singer-io"]
)
assert_cli_runner(res)
plugin = project_plugins_service.find_plugin(
plugin_type=PluginType.EXTRACTORS, plugin_name="tap-mock"
)
assert plugin.variant == "singer-io"
def test_install_multiple(
self, project, tap, tap_gitlab, target, dbt, cli_runner, project_plugins_service
):
with mock.patch(
"meltano.cli.install.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch("meltano.cli.install.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
result = cli_runner.invoke(
cli, ["install", "extractors", tap.name, tap_gitlab.name]
)
assert_cli_runner(result)
install_plugin_mock.assert_called_once_with(project, [tap_gitlab, tap])
def test_ui(self, project, cli_runner):
with mock.patch("meltano.cli.ui.APIWorker.start"
) as start_api_worker, mock.patch(
"meltano.cli.ui.MeltanoCompilerWorker.start"
) as start_compiler, mock.patch(
"meltano.cli.ui.UIAvailableWorker.start"
) as start_ui_available_worker, mock.patch.object(
GoogleAnalyticsTracker,
"track_meltano_ui") as track:
result = cli_runner.invoke(cli, "ui")
assert_cli_runner(result)
assert start_api_worker.called
assert start_ui_available_worker.called
assert start_compiler.called
assert track.called
def test_upgrade(self, project, cli_runner):
result = cli_runner.invoke(cli, ["upgrade"])
assert_cli_runner(result)
assert ("The `meltano` package could not be upgraded automatically"
in result.output)
assert "run `meltano upgrade --skip-package`" in result.output
with mock.patch("meltano.cli.upgrade.UpgradeService._upgrade_package"
) as upgrade_package_mock:
upgrade_package_mock.return_value = True
result = cli_runner.invoke(cli, ["upgrade"])
assert_cli_runner(result)
assert ("Meltano and your Meltano project have been upgraded!"
in result.output)
def test_discover(self, project, cli_runner, plugin_discovery_service):
with mock.patch(
"meltano.cli.discovery.PluginDiscoveryService",
return_value=plugin_discovery_service,
):
result = cli_runner.invoke(cli, ["discover"])
assert_cli_runner(result)
assert "Extractors" in result.output
assert "tap-gitlab" in result.output
assert (
"tap-mock, variants: meltano (default), singer-io (deprecated)"
in result.output)
assert "Loaders" in result.output
assert "target-jsonl" in result.output
assert "target-mock" in result.output
def test_add_files_without_updates(self, project, cli_runner,
config_service):
result = cli_runner.invoke(cli, ["add", "files", "docker-compose"])
assert_cli_runner(result)
# Plugin has not been added to meltano.yml
with pytest.raises(PluginMissingError):
config_service.find_plugin("docker-compose", PluginType.FILES)
# File has been created
assert "Created docker-compose.yml" in result.output
file_path = project.root_dir("docker-compose.yml")
assert file_path.is_file()
# File does not have "managed" header
assert "This file is managed" not in file_path.read_text()
def test_install_type_name(self, project, tap, tap_gitlab, target, dbt,
cli_runner):
with mock.patch(
"meltano.cli.install.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
result = cli_runner.invoke(cli, ["install", "extractor", tap.name])
assert_cli_runner(result)
install_plugin_mock.assert_called_once_with(project, [tap])
with mock.patch(
"meltano.cli.install.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
result = cli_runner.invoke(cli, ["install", "loader", target.name])
assert_cli_runner(result)
install_plugin_mock.assert_called_once_with(project, [target])
def test_add_custom(self, project, cli_runner, project_plugins_service):
pip_url = "-e path/to/tap-custom"
executable = "tap-custom-bin"
stdin = os.linesep.join(
# namespace, pip_url, executable, capabilities, settings
["tap_custom", pip_url, executable, "foo,bar", "baz,qux"]
)
with mock.patch(
"meltano.cli.add.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch("meltano.cli.add.install_plugins") as install_plugin_mock:
install_plugin_mock.return_value = True
res = cli_runner.invoke(
cli, ["add", "--custom", "extractor", "tap-custom"], input=stdin
)
assert_cli_runner(res)
plugin = project_plugins_service.find_plugin(
plugin_type=PluginType.EXTRACTORS, plugin_name="tap-custom"
)
assert plugin.name == "tap-custom"
assert plugin.pip_url == pip_url
plugin_def = plugin.custom_definition
plugin_variant = plugin.custom_definition.variants[0]
assert plugin_def.type == plugin.type
assert plugin_def.name == plugin.name == "tap-custom"
assert plugin_def.namespace == plugin.namespace == "tap_custom"
assert plugin_variant.name is None
assert plugin.pip_url == plugin_variant.pip_url == pip_url
assert plugin.executable == plugin_variant.executable == executable
assert plugin.capabilities == plugin_variant.capabilities == ["foo", "bar"]
assert [s.name for s in plugin_variant.settings] == ["baz", "qux"]
assert plugin.settings == plugin_variant.settings
install_plugin_mock.assert_called_once_with(
project, [plugin], reason=PluginInstallReason.ADD
)
def test_add_transform(self, project, cli_runner):
cli_runner.invoke(cli, ["add", "files", "dbt"])
cli_runner.invoke(cli, ["install", "files", "dbt"])
res = cli_runner.invoke(cli, ["add", "transform", "tap-google-analytics"])
assert_cli_runner(res)
with project.root_dir("transform/packages.yml").open() as packages_file:
packages_yaml = yaml.safe_load(packages_file)
with project.root_dir("transform/dbt_project.yml").open() as project_file:
project_yaml = yaml.safe_load(project_file)
assert {
"git": "https://gitlab.com/meltano/dbt-tap-google-analytics.git"
} in packages_yaml["packages"]
assert "tap_google_analytics" in project_yaml["models"]
assert project_yaml["models"]["tap_google_analytics"] == {
"vars": {"schema": "{{ env_var('DBT_SOURCE_SCHEMA') }}"}
}
def test_dump_state(
self,
session,
cli_runner,
project,
tap,
target,
project_plugins_service,
plugin_settings_service_factory,
):
state = {"success": True}
with project.root.joinpath("state.json").open("w") as state_file:
json.dump(state, state_file)
job_id = "pytest_test_elt"
args = [
"elt",
"--job_id",
job_id,
tap.name,
target.name,
"--state",
"state.json",
"--dump",
"state",
]
with mock.patch(
"meltano.cli.elt.ProjectPluginsService",
return_value=project_plugins_service,
), mock.patch.object(SingerTap, "discover_catalog"), mock.patch.object(
SingerTap, "apply_catalog_rules"):
result = cli_runner.invoke(cli, args)
assert_cli_runner(result)
assert json.loads(result.stdout) == state
