def test_fivetran_sync_op():
ft_resource = fivetran_resource.configured({
"api_key": "foo",
"api_secret": "bar"
})
final_data = {"succeeded_at": "2021-01-01T02:00:00.0Z"}
api_prefix = f"{FIVETRAN_API_BASE}/{FIVETRAN_CONNECTOR_PATH}{DEFAULT_CONNECTOR_ID}"
@op
def foo_op():
pass
@job(
resource_defs={"fivetran": ft_resource},
config={
"ops": {
"fivetran_sync_op": {
"config": {
"connector_id": DEFAULT_CONNECTOR_ID,
"poll_interval": 0.1,
"poll_timeout": 10,
}
}
}
},
)
def fivetran_sync_job():
fivetran_sync_op(start_after=foo_op())
with responses.RequestsMock() as rsps:
rsps.add(rsps.PATCH, api_prefix, json=get_sample_update_response())
rsps.add(rsps.POST,
f"{api_prefix}/force",
json=get_sample_sync_response())
# connector schema
rsps.add(rsps.GET,
f"{api_prefix}/schemas",
json=get_sample_connector_schema_config())
# initial state
rsps.add(rsps.GET, api_prefix, json=get_sample_connector_response())
# n polls before updating
for _ in range(2):
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response())
# final state will be updated
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response(data=final_data))
result = fivetran_sync_job.execute_in_process()
assert result.output_for_node("fivetran_sync_op") == FivetranOutput(
connector_details=get_sample_connector_response(
data=final_data)["data"],
schema_config=get_sample_connector_schema_config()["data"],
)
asset_materializations = [
event for event in result.events_for_node("fivetran_sync_op")
if event.event_type_value == "ASSET_MATERIALIZATION"
]
assert len(asset_materializations) == 3
asset_keys = set(mat.event_specific_data.materialization.asset_key
for mat in asset_materializations)
assert asset_keys == set([
AssetKey(["fivetran", "xyz1", "abc1"]),
AssetKey(["fivetran", "xyz1", "abc2"]),
AssetKey(["fivetran", "abc", "xyz"]),
])
def test_fivetran_asset_run(asset_keys, should_error):
ft_resource = fivetran_resource.configured({
"api_key": "foo",
"api_secret": "bar"
})
final_data = {"succeeded_at": "2021-01-01T02:00:00.0Z"}
api_prefix = f"{FIVETRAN_API_BASE}/{FIVETRAN_CONNECTOR_PATH}{DEFAULT_CONNECTOR_ID}"
fivetran_assets = build_fivetran_assets(
name="my_cool_ft_assets",
connector_id=DEFAULT_CONNECTOR_ID,
asset_keys=asset_keys,
poll_interval=0.1,
poll_timeout=10,
)
# expect the multi asset to have one asset key and one output for each specified asset key
assert fivetran_assets.asset_keys == set(asset_keys)
assert len(fivetran_assets.op.output_defs) == len(asset_keys)
fivetran_assets_job = build_assets_job(
name="fivetran_assets_job",
assets=[fivetran_assets],
resource_defs={"fivetran": ft_resource},
)
with responses.RequestsMock() as rsps:
rsps.add(rsps.PATCH, api_prefix, json=get_sample_update_response())
rsps.add(rsps.POST,
f"{api_prefix}/force",
json=get_sample_sync_response())
# connector schema
rsps.add(
rsps.GET,
f"{api_prefix}/schemas",
json=get_sample_connector_schema_config(tables=[
("schema1", "tracked"),
("schema1", "untracked"),
("schema2", "tracked"),
]),
)
# initial state
rsps.add(rsps.GET, api_prefix, json=get_sample_connector_response())
# final state will be updated
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response(data=final_data))
if should_error:
with pytest.raises(DagsterStepOutputNotFoundError):
fivetran_assets_job.execute_in_process()
else:
result = fivetran_assets_job.execute_in_process()
assert result.success
# make sure we only have outputs for the explicit asset keys
outputs = [
event for event in result.events_for_node("my_cool_ft_assets")
if event.event_type_value == "STEP_OUTPUT"
]
assert len(outputs) == len(asset_keys)
# make sure we have asset materializations for all the schemas/tables that were actually sync'd
asset_materializations = [
event for event in result.events_for_node("my_cool_ft_assets")
if event.event_type_value == "ASSET_MATERIALIZATION"
]
assert len(asset_materializations) == 3
found_asset_keys = set(
mat.event_specific_data.materialization.asset_key
for mat in asset_materializations)
assert found_asset_keys == {
AssetKey(["schema1", "tracked"]),
AssetKey(["schema1", "untracked"]),
AssetKey(["schema2", "tracked"]),
}