def test_resync_and_poll(n_polls, succeed_at_end):
ft_resource = fivetran_resource(
build_init_resource_context(config={
"api_key": "some_key",
"api_secret": "some_secret",
}))
api_prefix = f"{ft_resource.api_base_url}{DEFAULT_CONNECTOR_ID}"
final_data = ({
"succeeded_at": "2021-01-01T02:00:00.0Z"
} if succeed_at_end else {
"failed_at": "2021-01-01T02:00:00.0Z"
})
def _mock_interaction():
with responses.RequestsMock() as rsps:
rsps.add(
rsps.GET,
f"{ft_resource.api_base_url}{DEFAULT_CONNECTOR_ID}/schemas",
json=get_complex_sample_connector_schema_config(),
)
rsps.add(rsps.PATCH, api_prefix, json=get_sample_update_response())
rsps.add(rsps.POST,
f"{api_prefix}/schemas/tables/resync",
json=get_sample_resync_response())
# initial state
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response())
# n polls before updating
for _ in range(n_polls):
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response())
# final state will be updated
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response(data=final_data))
return ft_resource.resync_and_poll(
DEFAULT_CONNECTOR_ID,
resync_parameters={"xyz1": ["abc1", "abc2"]},
poll_interval=0.1,
)
if succeed_at_end:
assert _mock_interaction() == FivetranOutput(
connector_details=get_sample_connector_response(
data=final_data)["data"],
schema_config=get_complex_sample_connector_schema_config()["data"],
)
else:
with pytest.raises(Failure, match="failed!"):
_mock_interaction()
def test_fivetran_sync_op():
ft_resource = fivetran_resource.configured({
"api_key": "foo",
"api_secret": "bar"
})
final_data = {"succeeded_at": "2021-01-01T02:00:00.0Z"}
api_prefix = f"{FIVETRAN_API_BASE}/{FIVETRAN_CONNECTOR_PATH}{DEFAULT_CONNECTOR_ID}"
@op
def foo_op():
pass
@job(
resource_defs={"fivetran": ft_resource},
config={
"ops": {
"fivetran_sync_op": {
"config": {
"connector_id": DEFAULT_CONNECTOR_ID,
"poll_interval": 0.1,
"poll_timeout": 10,
}
}
}
},
)
def fivetran_sync_job():
fivetran_sync_op(start_after=foo_op())
with responses.RequestsMock() as rsps:
rsps.add(rsps.PATCH, api_prefix, json=get_sample_update_response())
rsps.add(rsps.POST,
f"{api_prefix}/force",
json=get_sample_sync_response())
# connector schema
rsps.add(rsps.GET,
f"{api_prefix}/schemas",
json=get_sample_connector_schema_config())
# initial state
rsps.add(rsps.GET, api_prefix, json=get_sample_connector_response())
# n polls before updating
for _ in range(2):
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response())
# final state will be updated
rsps.add(rsps.GET,
api_prefix,
json=get_sample_connector_response(data=final_data))
result = fivetran_sync_job.execute_in_process()
assert result.output_for_node("fivetran_sync_op") == FivetranOutput(
connector_details=get_sample_connector_response(
data=final_data)["data"],
schema_config=get_sample_connector_schema_config()["data"],
)
asset_materializations = [
event for event in result.events_for_node("fivetran_sync_op")
if event.event_type_value == "ASSET_MATERIALIZATION"
]
assert len(asset_materializations) == 3
asset_keys = set(mat.event_specific_data.materialization.asset_key
for mat in asset_materializations)
assert asset_keys == set([
AssetKey(["fivetran", "xyz1", "abc1"]),
AssetKey(["fivetran", "xyz1", "abc2"]),
AssetKey(["fivetran", "abc", "xyz"]),
])