def test_dbt_parse_dbt_test_event(mock_datetime, mock_uuid,
parent_run_metadata):
mock_datetime.now.return_value.isoformat.return_value = '2021-08-25T11:00:25.277467+00:00'
mock_uuid.side_effect = [
'6edf42ed-d8d0-454a-b819-d09b9067ff99',
'1a69c0a7-04bb-408b-980e-cbbfb1831ef7',
'f99310b4-339a-4381-ad3e-c1b95c24ff11',
'c11f2efd-4415-45fc-8081-10d2aaa594d2',
]
processor = DbtArtifactProcessor(
producer=
'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt',
project_dir='tests/dbt/test',
)
processor.dbt_run_metadata = parent_run_metadata
dbt_events = processor.parse()
events = [
attr.asdict(event, value_serializer=serialize)
for event in dbt_events.starts + dbt_events.completes +
dbt_events.fails
]
with open('tests/dbt/test/result.json', 'r') as f:
assert match(json.load(f), events)
def test_dbt_parse_and_compare_event(path, parent_run_metadata):
processor = DbtArtifactProcessor(
producer=
'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt',
project_dir=path)
processor.dbt_run_metadata = parent_run_metadata
dbt_events = processor.parse()
events = [
attr.asdict(event, value_serializer=serialize)
for event in dbt_events.starts + dbt_events.completes +
dbt_events.fails
]
with open(f'{path}/result.json', 'r') as f:
assert match(json.load(f), events)
def check_matches_ordered(expected_events, actual_events) -> bool:
for index, expected in enumerate(expected_events):
# Actual events have to be in the same order as expected events
actual = actual_events[index]
if expected['eventType'] == actual['eventType'] and \
expected['job']['name'] == actual['job']['name']:
if not match(expected, actual):
log.info(
f"failed to compare expected {expected}\nwith actual {actual}"
)
return False
break
log.info(
f"Wrong order of events: expected {expected['eventType']} - "
f"{expected['job']['name']}\ngot {actual['eventType']} - {actual['job']['name']}"
)
return False
return True
def check_matches(expected_events, actual_events) -> bool:
for expected in expected_events:
is_compared = False
for actual in actual_events:
# Try to find matching event by eventType and job name
if expected['eventType'] == actual['eventType'] and \
expected['job']['name'] == actual['job']['name']:
is_compared = True
if not match(expected, actual):
log.info(
f"failed to compare expected {expected}\nwith actual {actual}"
)
return False
break
if not is_compared:
log.info(f"not found event comparable to {expected['eventType']} "
f"- {expected['job']['name']}")
return False
return True
def test_dbt_parse_profile_with_env_vars(mock_uuid, parent_run_metadata):
mock_uuid.side_effect = [
'6edf42ed-d8d0-454a-b819-d09b9067ff99',
]
processor = DbtArtifactProcessor(
producer=
'https://github.com/OpenLineage/OpenLineage/tree/0.0.1/integration/dbt',
project_dir='tests/dbt/env_vars',
target='prod',
)
processor.dbt_run_metadata = parent_run_metadata
dbt_events = processor.parse()
events = [
attr.asdict(event, value_serializer=serialize)
for event in dbt_events.starts + dbt_events.completes +
dbt_events.fails
]
with open('tests/dbt/env_vars/result.json', 'r') as f:
assert match(json.load(f), events)