def setUp(self):
class TestSubstream(ResponseSubstream):
tap_stream_id = "test_response_substream"
key_properties = []
path = ("", )
transformer_class = MagicMock()
class TestStream(Stream):
tap_stream_id = "test_stream"
substream_definitions = [TestSubstream]
key_properties = []
request_handler = MagicMock()
valid_replication_keys = ["updated_date"]
transformer_class = MagicMock()
self.TestSubstream = TestSubstream
self.TestStream = TestStream
self.test_catalog = generate_catalog([
{
"tap_stream_id": "test_stream",
"selected": True
},
{
"tap_stream_id": "test_response_substream",
"selected": True
},
])
self.test_stream = self.TestStream(self.test_catalog, {})
def test_updates_replication_config_from_catalog_entry(self):
test_catalog = generate_catalog([{
"tap_stream_id": "test_stream",
"selected": True,
"replication_key": "modified_at",
"replication_method": "FULL_TABLE",
}])
self.TestStream.valid_replication_keys = ["modified_at"]
test_stream = self.TestStream(catalog=test_catalog, config={})
self.assertEqual(test_stream.replication_key, "modified_at")
self.assertEqual(test_stream.replication_method, "FULL_TABLE")
def test_with_conflicting_parent_and_child_stream(self):
test_catalog = generate_catalog([
{
"tap_stream_id": "parent",
"selected": False
},
{
"tap_stream_id": "child",
"selected": True
},
])
with self.assertRaises(DependencyConflict):
check_dependency_conflicts(test_catalog)
def test_with_non_conflicting_streams(self):
test_catalog = generate_catalog([
{
"tap_stream_id": "parent",
"selected": True
},
{
"tap_stream_id": "child",
"selected": False
},
])
try:
check_dependency_conflicts(test_catalog)
except DependencyConflict:
self.fail(
"check_dependency_conflicts raised a DependencyConflict exception"
)
test_catalog_2 = generate_catalog([
{
"tap_stream_id": "parent",
"selected": True
},
{
"tap_stream_id": "child",
"selected": True
},
])
try:
check_dependency_conflicts(test_catalog_2)
except DependencyConflict:
self.fail(
"check_dependency_conflicts raised a DependencyConflict exception"
)
def test_is_first_run_false_full_table(self, mock_write_activate_version):
"""Ensure a FULL_TABLE run with `wrote_initial_activate_version` being set to True in
bookmarks DOES NOT result in an initial ACTIVATE_VERSION message being sent
"""
prepare_stream(
tap_stream_id="webhooks",
stream_defs={},
stream_versions={},
catalog=generate_catalog([
{"tap_stream_id": "webhooks", "selected": True, "replication_key": None, "replication_method": "FULL_TABLE"},
]),
config={"start_date": "2021-01-01"},
state={"bookmarks": {"webhooks": {"wrote_initial_activate_version": True}}},
)
mock_write_activate_version.assert_not_called()
def test_is_first_run_true_incremental(self, mock_write_activate_version):
"""Ensure an INCREMENTAL run with `wrote_initial_activate_version` being set to True in
bookmarks DOES NOT result in an initial ACTIVATE_VERSION message being sent
(expected not to be sent in general for INCREMENTAL)
"""
prepare_stream(
tap_stream_id="invoices",
stream_defs={},
stream_versions={},
catalog=generate_catalog([
{"tap_stream_id": "invoices", "selected": True, "replication_key": "updated_date", "replication_method": "INCREMENTAL"},
]),
config={"start_date": "2021-01-01"},
state={"bookmarks": {"invoices": {"wrote_initial_activate_version": True}}},
)
mock_write_activate_version.assert_not_called()
def test_with_substream_instances(self):
test_catalog = generate_catalog([
{
"tap_stream_id": "parent",
"selected": True
},
{
"tap_stream_id": "child",
"selected": True
},
{
"tap_stream_id": "child2",
"selected": True
},
])
self.assertTrue(is_substream(ChildStream(test_catalog, {})))
self.assertTrue(is_substream(Child2Stream(test_catalog, {})))
self.assertFalse(is_substream(ParentStream(test_catalog, {})))
def test_is_first_run_full_table_as_substream(self, mock_write_activate_version, mock_get_full_table_version):
"""Ensure a FULL_TABLE run without `wrote_initial_activate_version` being set to True in
bookmarks results in an initial ACTIVATE_VERSION message being sent for SUBSTREAMS
"""
mock_get_full_table_version.return_value = 123
prepare_stream(
tap_stream_id="plans",
stream_defs={},
stream_versions={},
catalog=generate_catalog([
{"tap_stream_id": "plans", "selected": True, "replication_key": None, "replication_method": "FULL_TABLE"},
{"tap_stream_id": "charges", "selected": True, "replication_key": None, "replication_method": "FULL_TABLE"},
]),
config={"start_date": "2021-01-01"},
state={"bookmarks": {"charges": {}}},
)
mock_write_activate_version.assert_any_call(
"charges",
123
)