def do_sync_incremental(conn_config, stream, state, desired_columns, md_map):
"""
Runs Incremental sync
"""
replication_key = md_map.get((), {}).get('replication-key')
LOGGER.info(
"Stream %s is using incremental replication with replication key %s",
stream['tap_stream_id'], replication_key)
stream_state = state.get('bookmarks', {}).get(stream['tap_stream_id'])
illegal_bk_keys = set(stream_state.keys()).difference({
'replication_key', 'replication_key_value', 'version',
'last_replication_method'
})
if len(illegal_bk_keys) != 0:
raise Exception(f"invalid keys found in state: {illegal_bk_keys}")
state = singer.write_bookmark(state, stream['tap_stream_id'],
'replication_key', replication_key)
sync_common.send_schema_message(stream, [replication_key])
state = incremental.sync_table(conn_config, stream, state, desired_columns,
md_map)
return state
def do_sync_incremental(conn_config, stream, state, desired_columns, md_map):
replication_key = md_map.get((), {}).get('replication-key')
LOGGER.info(
"Stream %s is using incremental replication with replication key %s",
stream.tap_stream_id, replication_key)
stream_state = state.get('bookmarks', {}).get(stream.tap_stream_id)
illegal_bk_keys = set(stream_state.keys()).difference(
set([
'replication_key', 'replication_key_value', 'version',
'last_replication_method'
]))
if len(illegal_bk_keys) != 0:
raise Exception(
"invalid keys found in state: {}".format(illegal_bk_keys))
state = singer.write_bookmark(state, stream.tap_stream_id,
'replication_key', replication_key)
if md_map.get((), {}).get('is-view'):
pks = md_map.get(()).get('table-key-properties')
else:
pks = md_map.get(()).get('table-key-properties')
send_schema_message(stream, pks)
state = incremental.sync_table(conn_config, stream, state, desired_columns,
md_map)
return state
def do_sync_incremental(conn_config, stream, state, desired_columns, md_map):
replication_key = md_map.get((), {}).get("replication-key")
LOGGER.info(
"Stream %s is using incremental replication with replication key %s",
stream["tap_stream_id"],
replication_key,
)
stream_state = state.get("bookmarks", {}).get(stream["tap_stream_id"])
illegal_bk_keys = set(stream_state.keys()).difference(
set([
"replication_key", "replication_key_value", "version",
"last_replication_method"
]))
if len(illegal_bk_keys) != 0:
raise Exception(
"invalid keys found in state: {}".format(illegal_bk_keys))
state = singer.write_bookmark(state, stream["tap_stream_id"],
"replication_key", replication_key)
sync_common.send_schema_message(stream, [replication_key])
state = incremental.sync_table(conn_config, stream, state, desired_columns,
md_map)
return state
def test_sync_table_if_not_hstore_available(self, _,
mocked_hstore_available):
"""Test for sync_table_ if hstore is unavailable"""
desired_columns = ['foo_key']
expected_state_replication_key_value = MockedConnect.cursor.return_value
mocked_hstore_available.return_value = False
actual_state = incremental.sync_table(self.conn_config, self.stream,
self.state, desired_columns,
self.md_map)
self.assertEqual(
expected_state_replication_key_value, actual_state['bookmarks'][
self.stream['tap_stream_id']]['replication_key_value'])
def test_sync_table(self, mocked_register_hstore):
"""Test for sync_table if it works correctly"""
desired_columns = ['foo_key']
self.state['bookmarks'] = {}
expected_state_replication_key_value = MockedConnect.cursor.return_value
actual_state = incremental.sync_table(self.conn_config, self.stream,
self.state, desired_columns,
self.md_map)
mocked_register_hstore.assert_called()
self.assertEqual(
expected_state_replication_key_value,
actual_state['bookmarks'][self.stream['tap_stream_id']]
['replication_key_value'],
)
def test_sync_table_if_rows_saved_is_a_multiply_of_update_bookmark_period(
self, mocked_register_hstore, mocked_singer_write):
"""Test for sync_table if rows_saved is a multiply of UPDATE_BOOKMARK_PERION"""
original_update_bookmark_period = incremental.UPDATE_BOOKMARK_PERIOD
incremental.UPDATE_BOOKMARK_PERIOD = MockedConnect.cursor.counter_limit - 1
desired_columns = ['foo_key']
expected_state_replication_key_value = MockedConnect.cursor.return_value
actual_state = incremental.sync_table(self.conn_config, self.stream,
self.state, desired_columns,
self.md_map)
mocked_register_hstore.assert_called()
self.assertEqual(
expected_state_replication_key_value,
actual_state['bookmarks'][self.stream['tap_stream_id']]
['replication_key_value'],
)
incremental.UPDATE_BOOKMARK_PERIOD = original_update_bookmark_period
mocked_singer_write.assert_called_with(
singer.StateMessage(value=self.state))
