def test_configure_with_rest_sink_initializes_graph( self, mock_source, mock_test_connection ): pipeline = Pipeline.create( { "source": { "type": "file", "config": {"filename": "test_events.json"}, }, "sink": { "type": "datahub-rest", "config": { "server": "http://somehost.someplace.some:8080", "token": "foo", }, }, } ) # assert that the default sink config is for a DatahubRestSink assert isinstance(pipeline.config.sink, DynamicTypedConfig) assert pipeline.config.sink.type == "datahub-rest" assert pipeline.config.sink.config == { "server": "http://somehost.someplace.some:8080", "token": "foo", } assert pipeline.ctx.graph is not None, "DataHubGraph should be initialized" assert pipeline.ctx.graph.config.server == pipeline.config.sink.config["server"] assert pipeline.ctx.graph.config.token == pipeline.config.sink.config["token"]
def test_mongodb_ingest(mongodb, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/mongodb" pipeline = Pipeline.create({ "run_id": "mongodb-test", "source": { "type": "mongodb", "config": { "connect_uri": "mongodb://localhost:57017", "username": "******", "password": "******", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/mongodb_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "mongodb_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "mongodb_mce_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def test_lookml_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" pipeline = Pipeline.create({ "run_id": "lookml-test", "source": { "type": "lookml", "config": { "base_folder": str(test_resources_dir), "connection_to_platform_map": { "my_connection": "conn" }, "parse_table_names_from_sql": True, }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/lookml_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "lookml_mces.json")) expected = mce_helpers.load_json_file( str(test_resources_dir / "expected_output.json")) mce_helpers.assert_mces_equal(output, expected)
def test_data_lake_s3_ingest(pytestconfig, s3_populate, source_file, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/s3/" f = open(os.path.join(SOURCE_FILES_PATH, source_file)) source = json.load(f) config_dict = {} config_dict["source"] = source config_dict["sink"] = { "type": "file", "config": { "filename": f"{tmp_path}/{source_file}", }, } config_dict["run_id"] = source_file pipeline = Pipeline.create(config_dict) pipeline.run() pipeline.raise_from_status() # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path=f"{tmp_path}/{source_file}", golden_path= f"{test_resources_dir}/golden-files/s3/golden_mces_{source_file}", )
def test_data_lake_local_ingest(pytestconfig, source_file, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/s3/" f = open(os.path.join(SOURCE_FILES_PATH, source_file)) source = json.load(f) config_dict = {} source["config"]["path_spec"]["include"] = source["config"]["path_spec"][ "include"].replace("s3://my-test-bucket/", "tests/integration/s3/test_data/local_system/") source["config"]["profiling"]["enabled"] = True source["config"].pop("aws_config") config_dict["source"] = source config_dict["sink"] = { "type": "file", "config": { "filename": f"{tmp_path}/{source_file}", }, } config_dict["run_id"] = source_file pipeline = Pipeline.create(config_dict) pipeline.run() pipeline.raise_from_status() # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path=f"{tmp_path}/{source_file}", golden_path= f"{test_resources_dir}/golden-files/local/golden_mces_{source_file}", )
def create_and_run_test_pipeline( events: List[Union[MetadataChangeEventClass, MetadataChangeProposalWrapper]], transformers: List[Dict[str, Any]], path: str, ) -> str: with mock.patch("tests.unit.test_source.TestSource.get_workunits" ) as mock_getworkunits: mock_getworkunits.return_value = [ workunit.MetadataWorkUnit( id=f"test-workunit-mce-{e.proposedSnapshot.urn}", mce=e) if isinstance(e, MetadataChangeEventClass) else workunit.MetadataWorkUnit( id=f"test-workunit-mcp-{e.entityUrn}-{e.aspectName}", mcp=e) for e in events ] events_file = f"{path}/{str(uuid4())}.json" pipeline = Pipeline.create( config_dict={ "source": { "type": "tests.unit.test_source.TestSource", "config": {}, }, "transformers": transformers, "sink": { "type": "file", "config": { "filename": events_file } }, }) pipeline.run() pipeline.raise_from_status() return events_file
def test_ldap_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/ldap" with docker_compose_runner(test_resources_dir / "docker-compose.yml", "ldap") as docker_services: # The openldap container loads the sample data after exposing the port publicly. As such, # we must wait a little bit extra to ensure that the sample data is loaded. wait_for_port(docker_services, "openldap", 389) time.sleep(5) pipeline = Pipeline.create({ "run_id": "ldap-test", "source": { "type": "ldap", "config": { "ldap_server": "ldap://localhost", "ldap_user": "******", "ldap_password": "******", "base_dn": "dc=example,dc=org", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/ldap_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "ldap_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "ldap_mces_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def ingest(config: str): """Main command for ingesting metadata into DataHub""" config_file = pathlib.Path(config) if not config_file.is_file(): raise ConfigurationError(f"Cannot open config file {config}") config_mech: ConfigurationMechanism if config_file.suffix in [".yaml", ".yml"]: config_mech = YamlConfigurationMechanism() elif config_file.suffix == ".toml": config_mech = TomlConfigurationMechanism() else: raise ConfigurationError( "Only .toml and .yml are supported. Cannot process file type {}".format( config_file.suffix ) ) with config_file.open() as fp: pipeline_config = config_mech.load_config(fp) with nicely_formatted_validation_errors(): logger.debug(f"Using config: {pipeline_config}") pipeline = Pipeline.create(pipeline_config) pipeline.run()
def datahub_recipe(): with open("path/to/recipe.yml") as config_file: config = yaml.safe_load(config_file) pipeline = Pipeline.create(config) pipeline.run() pipeline.raise_from_status()
def test_serde_to_json(pytestconfig: PytestConfig, tmp_path: pathlib.Path, json_filename: str) -> None: golden_file = pytestconfig.rootpath / json_filename output_filename = "output.json" output_file = tmp_path / output_filename pipeline = Pipeline.create({ "source": { "type": "file", "config": { "filename": str(golden_file) } }, "sink": { "type": "file", "config": { "filename": str(output_file) } }, "run_id": "serde_test", }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=f"{tmp_path}/{output_filename}", golden_path=golden_file, )
def test_mongodb_ingest(docker_compose_runner, pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/mongodb" with docker_compose_runner(test_resources_dir / "docker-compose.yml", "mongo") as docker_services: wait_for_port(docker_services, "testmongodb", 27017) # Run the metadata ingestion pipeline. pipeline = Pipeline.create({ "run_id": "mongodb-test", "source": { "type": "mongodb", "config": { "connect_uri": "mongodb://localhost:57017", "username": "******", "password": "******", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/mongodb_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "mongodb_mces.json", golden_path=test_resources_dir / "mongodb_mces_golden.json", )
def ingest(config: str): """Main command for ingesting metadata into DataHub""" config_file = pathlib.Path(config) if not config_file.is_file(): raise ConfigurationError(f"Cannot open config file {config}") config_mech: ConfigurationMechanism if config_file.suffix in [".yaml", ".yml"]: config_mech = YamlConfigurationMechanism() elif config_file.suffix == ".toml": config_mech = TomlConfigurationMechanism() else: raise ConfigurationError( "Only .toml and .yml are supported. Cannot process file type {}". format(config_file.suffix)) with config_file.open() as fp: pipeline_config = config_mech.load_config(fp) try: logger.debug(f"Using config: {pipeline_config}") pipeline = Pipeline.create(pipeline_config) except ValidationError as e: click.echo(e, err=True) sys.exit(1) pipeline.run() ret = pipeline.pretty_print_summary() sys.exit(ret)
def run( ctx: click.Context, config: str, dry_run: bool, preview: bool, strict_warnings: bool, preview_workunits: int, ) -> None: """Ingest metadata into DataHub.""" logger.info("DataHub CLI version: %s", datahub_package.nice_version_name()) config_file = pathlib.Path(config) pipeline_config = load_config_file(config_file) try: logger.debug(f"Using config: {pipeline_config}") pipeline = Pipeline.create(pipeline_config, dry_run, preview, preview_workunits) except ValidationError as e: click.echo(e, err=True) sys.exit(1) except Exception as e: # The pipeline_config may contain sensitive information, so we wrap the exception # in a SensitiveError to prevent detailed variable-level information from being logged. raise SensitiveError() from e logger.info("Starting metadata ingestion") pipeline.run() logger.info("Finished metadata ingestion") ret = pipeline.pretty_print_summary(warnings_as_failure=strict_warnings) pipeline.log_ingestion_stats() sys.exit(ret)
def test_ldap_ingest(mysql, pytestconfig, tmp_path): test_resources_dir = pytestconfig.rootpath / "tests/integration/ldap" pipeline = Pipeline.create({ "run_id": "ldap-test", "source": { "type": "ldap", "config": { "ldap_server": "ldap://localhost", "ldap_user": "******", "ldap_password": "******", "base_dn": "dc=example,dc=org", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/ldap_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "ldap_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "ldap_mce_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def test_trino_instance_ingest(loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time): instance = "production_warehouse" platform = "trino" mce_out_file = "trino_instance_mces.json" events_file = tmp_path / mce_out_file pipeline_config = { "run_id": "trino-hive-instance-test", "source": { "type": data_platform, "config": TrinoConfig( host_port="localhost:5300", database="hivedb", username="******", platform_instance="production_warehouse", schema_pattern=AllowDenyPattern(allow=["^db1"]), ).dict(), }, "sink": { "type": "file", "config": FileSinkConfig(filename=str(events_file)).dict(), }, } # Run the metadata ingestion pipeline. pipeline = Pipeline.create(pipeline_config) pipeline.run() pipeline.pretty_print_summary() pipeline.raise_from_status(raise_warnings=True) # Assert that all events generated have instance specific urns urn_pattern = "^" + re.escape( f"urn:li:dataset:(urn:li:dataPlatform:{platform},{instance}.") assert (mce_helpers.assert_mce_entity_urn( "ALL", entity_type="dataset", regex_pattern=urn_pattern, file=events_file, ) >= 0), "There should be at least one match" assert (mce_helpers.assert_mcp_entity_urn( "ALL", entity_type="dataset", regex_pattern=urn_pattern, file=events_file, ) >= 0), "There should be at least one MCP" # all dataset entities emitted must have a dataPlatformInstance aspect emitted # there must be at least one entity emitted assert (mce_helpers.assert_for_each_entity( entity_type="dataset", aspect_name="dataPlatformInstance", aspect_field_matcher={ "instance": f"urn:li:dataPlatformInstance:(urn:li:dataPlatform:{platform},{instance})" }, file=events_file, ) >= 1)
def test_serde_to_json(pytestconfig, tmp_path, json_filename): golden_file = pytestconfig.rootpath / json_filename output_filename = "output.json" output_file = tmp_path / output_filename pipeline = Pipeline.create({ "source": { "type": "file", "config": { "filename": str(golden_file) } }, "sink": { "type": "file", "config": { "filename": str(output_file) } }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(tmp_path / output_filename) golden = mce_helpers.load_json_file(golden_file) assert golden == output
def test_bq_usage_source(pytestconfig, tmp_path): # from google.cloud.logging_v2 import ProtobufEntry test_resources_dir: pathlib.Path = (pytestconfig.rootpath / "tests/integration/bigquery-usage") bigquery_reference_logs_path = test_resources_dir / "bigquery_logs.json" if WRITE_REFERENCE_FILE: source = BigQueryUsageSource.create( dict( projects=[ "harshal-playground-306419", ], start_time=datetime.now(tz=timezone.utc) - timedelta(days=25), ), PipelineContext(run_id="bq-usage-test"), ) entries = list( source._get_bigquery_log_entries_via_gcp_logging( source._make_bigquery_logging_clients())) entries = [entry._replace(logger=None) for entry in entries] log_entries = jsonpickle.encode(entries, indent=4) with bigquery_reference_logs_path.open("w") as logs: logs.write(log_entries) with unittest.mock.patch( "datahub.ingestion.source.usage.bigquery_usage.GCPLoggingClient", autospec=True) as MockClient: # Add mock BigQuery API responses. with bigquery_reference_logs_path.open() as logs: reference_logs = jsonpickle.decode(logs.read()) MockClient().list_entries.return_value = reference_logs # Run a BigQuery usage ingestion run. pipeline = Pipeline.create({ "run_id": "test-bigquery-usage", "source": { "type": "bigquery-usage", "config": { "projects": ["sample-bigquery-project-1234"], "start_time": "2021-01-01T00:00Z", "end_time": "2021-07-01T00:00Z", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/bigquery_usages.json", }, }, }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "bigquery_usages.json", golden_path=test_resources_dir / "bigquery_usages_golden.json", )
def test_dbt_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" pipeline = Pipeline.create({ "run_id": "dbt-test", "source": { "type": "dbt", "config": { "manifest_path": f"{test_resources_dir}/dbt_manifest.json", "catalog_path": f"{test_resources_dir}/dbt_catalog.json", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/dbt_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "dbt_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "dbt_mces_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def test_feast_repository_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/feast" output_path = tmp_path / "feast_repository_mces.json" pipeline = Pipeline.create({ "run_id": "feast-repository-test", "source": { "type": "feast", "config": { "path": str(test_resources_dir / "feature_store"), "environment": "PROD", }, }, "sink": { "type": "file", "config": { "filename": str(output_path), }, }, }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=output_path, golden_path=test_resources_dir / "feast_repository_mces_golden.json", )
def test_mode_ingest_success(pytestconfig, tmp_path): with patch( "datahub.ingestion.source.mode.requests.session", side_effect=mocked_requests_sucess, ): global test_resources_dir test_resources_dir = pytestconfig.rootpath / "tests/integration/mode" pipeline = Pipeline.create({ "run_id": "mode-test", "source": { "type": "mode", "config": { "token": "xxxx", "password": "******", "connect_uri": "https://app.mode.com/", "workspace": "acryl", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/mode_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=f"{tmp_path}/mode_mces.json", golden_path=test_resources_dir / "mode_mces_golden.json", ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, )
def test_serde_large(pytestconfig, tmp_path): json_filename = "test_serde_large.json" output_filename = "output.json" test_resources_dir = pytestconfig.rootpath / "tests/unit/serde" golden_file = test_resources_dir / json_filename output_file = tmp_path / output_filename pipeline = Pipeline.create({ "source": { "type": "file", "config": { "filename": str(golden_file) } }, "sink": { "type": "file", "config": { "filename": str(output_file) } }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(tmp_path / output_filename) golden = mce_helpers.load_json_file(golden_file) mce_helpers.assert_mces_equal(output, golden)
def test_run_including_fake_transformation(self): pipeline = Pipeline.create({ "source": { "type": "tests.unit.test_pipeline.FakeSource" }, "transformers": [{ "type": "tests.unit.test_pipeline.AddStatusRemovedTransformer" }], "sink": { "type": "tests.test_helpers.sink_helpers.RecordingSink" }, "run_id": "pipeline_test", }) pipeline.run() pipeline.raise_from_status() expected_mce = get_initial_mce() dataset_snapshot = cast(DatasetSnapshotClass, expected_mce.proposedSnapshot) dataset_snapshot.aspects.append(get_status_removed_aspect()) sink_report: RecordingSinkReport = cast(RecordingSinkReport, pipeline.sink.get_report()) self.assertEqual(len(sink_report.received_records), 1) self.assertEqual(expected_mce, sink_report.received_records[0].record)
def test_lookml_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/lookml" pipeline = Pipeline.create({ "run_id": "lookml-test", "source": { "type": "lookml", "config": { "base_folder": str(test_resources_dir / "lkml_samples"), "connection_to_platform_map": { "my_connection": "conn" }, "parse_table_names_from_sql": True, }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/lookml_mces.json", }, }, }) pipeline.run() pipeline.pretty_print_summary() pipeline.raise_from_status(raise_warnings=True) mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "lookml_mces.json", golden_path=test_resources_dir / "expected_output.json", )
def test_feast_ingest(docker_compose_runner, pytestconfig, tmp_path): test_resources_dir = pytestconfig.rootpath / "tests/integration/feast" with docker_compose_runner(test_resources_dir / "docker-compose.yml", "feast") as docker_services: wait_for_port(docker_services, "testfeast", 6565) # container listens to this port once test cases have been setup wait_for_port(docker_services, "testfeast_setup", 6789) # Run the metadata ingestion pipeline. pipeline = Pipeline.create({ "run_id": "feast-test", "source": { "type": "feast", "config": { "core_url": "localhost:6565", "use_local_build": True, }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/feast_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() # Verify the output. output = mce_helpers.load_json_file(str(tmp_path / "feast_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "feast_mce_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def test_pipeline_process_commits(self, commit_policy, source, should_commit): pipeline = Pipeline.create({ "source": { "type": f"tests.unit.test_pipeline.{source}" }, "sink": { "type": "console" }, "run_id": "pipeline_test", }) class FakeCommittable(Committable): def __init__(self, commit_policy: CommitPolicy): self.name = "test_checkpointer" self.commit_policy = commit_policy def commit(self) -> None: pass fake_committable: Committable = FakeCommittable(commit_policy) with patch.object(FakeCommittable, "commit", wraps=fake_committable.commit) as mock_commit: pipeline.ctx.register_reporter(fake_committable) pipeline.run() # check that we called the commit method once only if should_commit is True if should_commit: mock_commit.assert_called_once() else: mock_commit.assert_not_called()
def test_dbt_ingest(pytestconfig, tmp_path, mock_time): test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt" # test manifest, catalog, sources are generated from https://github.com/kevinhu/sample-dbt pipeline = Pipeline.create({ "run_id": "dbt-test", "source": { "type": "dbt", "config": { "manifest_path": f"{test_resources_dir}/dbt_manifest.json", "catalog_path": f"{test_resources_dir}/dbt_catalog.json", "sources_path": f"{test_resources_dir}/dbt_sources.json", "target_platform": "dbt", "load_schemas": True, }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/dbt_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() output = mce_helpers.load_json_file(str(tmp_path / "dbt_mces.json")) golden = mce_helpers.load_json_file( str(test_resources_dir / "dbt_mces_golden.json")) mce_helpers.assert_mces_equal(output, golden)
def test_azure_ad_source_nested_groups(pytestconfig, tmp_path): test_resources_dir: pathlib.Path = (pytestconfig.rootpath / "tests/integration/azure_ad") with patch( "datahub.ingestion.source.identity.azure_ad.AzureADSource.get_token" ) as mock_token, patch( "datahub.ingestion.source.identity.azure_ad.AzureADSource._get_azure_ad_users" ) as mock_users, patch( "datahub.ingestion.source.identity.azure_ad.AzureADSource._get_azure_ad_groups" ) as mock_groups, patch( "datahub.ingestion.source.identity.azure_ad.AzureADSource._get_azure_ad_group_members" ) as mock_group_users: mocked_functions( test_resources_dir, mock_token, mock_users, mock_groups, mock_group_users, True, ) # Run an azure usage ingestion run. pipeline = Pipeline.create({ "run_id": "test-azure-ad", "source": { "type": "azure-ad", "config": { "client_id": "00000000-0000-0000-0000-000000000000", "tenant_id": "00000000-0000-0000-0000-000000000000", "client_secret": "client_secret", "redirect": "https://login.microsoftonline.com/common/oauth2/nativeclient", "authority": "https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000", "token_url": "https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token", "graph_url": "https://graph.microsoft.com/v1.0", "ingest_group_membership": True, "ingest_groups": True, "ingest_users": False, }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/azure_ad_mces_nested_groups.json", }, }, }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "azure_ad_mces_nested_groups.json", golden_path=test_resources_dir / "azure_ad_mces_golden_nested_groups.json", )
def get_current_checkpoint_from_pipeline( pipeline_config_dict: Dict[str, Any]) -> Optional[Checkpoint]: pipeline = Pipeline.create(pipeline_config_dict) pipeline.run() pipeline.raise_from_status() mysql_source = cast(MySQLSource, pipeline.source) return mysql_source.get_current_checkpoint( mysql_source.get_default_ingestion_job_id())
def test_looker_ingest(pytestconfig, tmp_path, mock_time): mocked_client = mock.MagicMock() with mock.patch( "datahub.ingestion.source.looker.LookerDashboardSource._get_looker_client", mocked_client, ): mocked_client.return_value.all_dashboards.return_value = [ Dashboard(id="1") ] mocked_client.return_value.dashboard.return_value = Dashboard( id="1", title="foo", created_at=datetime.utcfromtimestamp(time.time()), description="lorem ipsum", dashboard_elements=[ DashboardElement( id="2", type="", subtitle_text="Some text", query=Query( model="data", view="my_view", dynamic_fields= '[{"table_calculation":"calc","label":"foobar","expression":"offset(${my_table.value},1)","value_format":null,"value_format_name":"eur","_kind_hint":"measure","_type_hint":"number"}]', ), ) ], ) test_resources_dir = pytestconfig.rootpath / "tests/integration/looker" pipeline = Pipeline.create({ "run_id": "looker-test", "source": { "type": "looker", "config": { "base_url": "https://looker.company.com", "client_id": "foo", "client_secret": "bar", }, }, "sink": { "type": "file", "config": { "filename": f"{tmp_path}/looker_mces.json", }, }, }) pipeline.run() pipeline.raise_from_status() mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "looker_mces.json", golden_path=test_resources_dir / "expected_output.json", )
def test_trino_ingest(loaded_trino, test_resources_dir, pytestconfig, tmp_path, mock_time): # Run the metadata ingestion pipeline. with fs_helpers.isolated_filesystem(tmp_path): # Run the metadata ingestion pipeline for trino catalog referring to postgres database mce_out_file = "trino_mces.json" events_file = tmp_path / mce_out_file pipeline_config = { "run_id": "trino-test", "source": { "type": data_platform, "config": TrinoConfig( host_port="localhost:5300", database="postgresqldb", database_alias="library_catalog", username="******", schema_pattern=AllowDenyPattern(allow=["^librarydb"]), profile_pattern=AllowDenyPattern( allow=["library_catalog.librarydb.*"]), profiling=GEProfilingConfig( enabled=True, include_field_null_count=True, include_field_min_value=True, include_field_max_value=True, include_field_mean_value=True, include_field_median_value=True, include_field_stddev_value=True, include_field_quantiles=True, include_field_distinct_value_frequencies=True, include_field_histogram=True, include_field_sample_values=True, ), ).dict(), }, "sink": { "type": "file", "config": FileSinkConfig(filename=str(events_file)).dict(), }, } # Run the metadata ingestion pipeline. pipeline = Pipeline.create(pipeline_config) pipeline.run() pipeline.pretty_print_summary() pipeline.raise_from_status(raise_warnings=True) # Verify the output. mce_helpers.check_golden_file( pytestconfig, output_path="trino_mces.json", golden_path=test_resources_dir / "trino_mces_golden.json", )