def test_serde_to_json(pytestconfig, tmp_path, json_filename):
golden_file = pytestconfig.rootpath / json_filename
output_filename = "output.json"
output_file = tmp_path / output_filename
pipeline = Pipeline.create({
"source": {
"type": "file",
"config": {
"filename": str(golden_file)
}
},
"sink": {
"type": "file",
"config": {
"filename": str(output_file)
}
},
})
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(tmp_path / output_filename)
golden = mce_helpers.load_json_file(golden_file)
assert golden == output
def test_mssql_ingest(sql_server, pytestconfig, tmp_path):
test_resources_dir = pytestconfig.rootpath / "tests/integration/sql_server"
# Run the setup.sql file to populate the database.
docker = "docker"
command = f"{docker} exec testsqlserver /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'test!Password' -d master -i /setup/setup.sql"
ret = subprocess.run(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
assert ret.returncode == 0
# Run the metadata ingestion pipeline.
config_file = (test_resources_dir / "mssql_to_file.yml").resolve()
runner = CliRunner()
with fs_helpers.isolated_filesystem(tmp_path):
result = runner.invoke(datahub, ["ingest", "-c", f"{config_file}"])
assert result.exit_code == 0
output = mce_helpers.load_json_file("mssql_mces.json")
# Verify the output.
golden = mce_helpers.load_json_file(
str(test_resources_dir / "mssql_mces_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
def test_serde_large(pytestconfig, tmp_path):
json_filename = "test_serde_large.json"
output_filename = "output.json"
test_resources_dir = pytestconfig.rootpath / "tests/unit/serde"
golden_file = test_resources_dir / json_filename
output_file = tmp_path / output_filename
pipeline = Pipeline.create({
"source": {
"type": "file",
"config": {
"filename": str(golden_file)
}
},
"sink": {
"type": "file",
"config": {
"filename": str(output_file)
}
},
})
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(tmp_path / output_filename)
golden = mce_helpers.load_json_file(golden_file)
mce_helpers.assert_mces_equal(output, golden)
def test_ldap_ingest(mysql, pytestconfig, tmp_path):
test_resources_dir = pytestconfig.rootpath / "tests/integration/ldap"
pipeline = Pipeline.create({
"run_id": "ldap-test",
"source": {
"type": "ldap",
"config": {
"ldap_server": "ldap://localhost",
"ldap_user": "******",
"ldap_password": "******",
"base_dn": "dc=example,dc=org",
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/ldap_mces.json",
},
},
})
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(str(tmp_path / "ldap_mces.json"))
golden = mce_helpers.load_json_file(
str(test_resources_dir / "ldap_mce_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
def test_mongodb_ingest(mongodb, pytestconfig, tmp_path, mock_time):
test_resources_dir = pytestconfig.rootpath / "tests/integration/mongodb"
pipeline = Pipeline.create({
"run_id": "mongodb-test",
"source": {
"type": "mongodb",
"config": {
"connect_uri": "mongodb://localhost:57017",
"username": "******",
"password": "******",
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/mongodb_mces.json",
},
},
})
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(str(tmp_path / "mongodb_mces.json"))
golden = mce_helpers.load_json_file(
str(test_resources_dir / "mongodb_mce_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
def test_dbt_ingest(pytestconfig, tmp_path, mock_time):
test_resources_dir = pytestconfig.rootpath / "tests/integration/dbt"
pipeline = Pipeline.create({
"run_id": "dbt-test",
"source": {
"type": "dbt",
"config": {
"manifest_path": f"{test_resources_dir}/dbt_manifest.json",
"catalog_path": f"{test_resources_dir}/dbt_catalog.json",
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/dbt_mces.json",
},
},
})
pipeline.run()
pipeline.raise_from_status()
output = mce_helpers.load_json_file(str(tmp_path / "dbt_mces.json"))
golden = mce_helpers.load_json_file(
str(test_resources_dir / "dbt_mces_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
def test_ingest(mysql, pytestconfig, tmp_path):
test_resources_dir = pytestconfig.rootpath / "tests/integration/mysql"
config_file = (test_resources_dir / "mysql_to_file.yml").resolve()
ingest_command = f'cd {tmp_path} && datahub ingest -c {config_file}'
ret = os.system(ingest_command)
assert ret == 0
output = mce_helpers.load_json_file(str(tmp_path / "mysql_mces.json"))
golden = mce_helpers.load_json_file(
str(test_resources_dir / "mysql_mce_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
def test_mysql_ingest(mysql, pytestconfig, tmp_path):
test_resources_dir = pytestconfig.rootpath / "tests/integration/mysql"
config_file = (test_resources_dir / "mysql_to_file.yml").resolve()
# Run the metadata ingestion pipeline.
runner = CliRunner()
with fs_helpers.isolated_filesystem(tmp_path):
result = runner.invoke(datahub, ["ingest", "-c", f"{config_file}"])
assert result.exit_code == 0
output = mce_helpers.load_json_file("mysql_mces.json")
# Verify the output.
golden = mce_helpers.load_json_file(
str(test_resources_dir / "mysql_mce_golden.json"))
mce_helpers.assert_mces_equal(output, golden)
