def do_sync_incremental(conn_config, stream, state, desired_columns, md_map): """ Runs Incremental sync """ replication_key = md_map.get((), {}).get('replication-key') LOGGER.info( "Stream %s is using incremental replication with replication key %s", stream['tap_stream_id'], replication_key) stream_state = state.get('bookmarks', {}).get(stream['tap_stream_id']) illegal_bk_keys = set(stream_state.keys()).difference({ 'replication_key', 'replication_key_value', 'version', 'last_replication_method' }) if len(illegal_bk_keys) != 0: raise Exception(f"invalid keys found in state: {illegal_bk_keys}") state = singer.write_bookmark(state, stream['tap_stream_id'], 'replication_key', replication_key) sync_common.send_schema_message(stream, [replication_key]) state = incremental.sync_table(conn_config, stream, state, desired_columns, md_map) return state
def do_sync_incremental(conn_config, stream, state, desired_columns, md_map): replication_key = md_map.get((), {}).get('replication-key') LOGGER.info( "Stream %s is using incremental replication with replication key %s", stream.tap_stream_id, replication_key) stream_state = state.get('bookmarks', {}).get(stream.tap_stream_id) illegal_bk_keys = set(stream_state.keys()).difference( set([ 'replication_key', 'replication_key_value', 'version', 'last_replication_method' ])) if len(illegal_bk_keys) != 0: raise Exception( "invalid keys found in state: {}".format(illegal_bk_keys)) state = singer.write_bookmark(state, stream.tap_stream_id, 'replication_key', replication_key) if md_map.get((), {}).get('is-view'): pks = md_map.get(()).get('table-key-properties') else: pks = md_map.get(()).get('table-key-properties') send_schema_message(stream, pks) state = incremental.sync_table(conn_config, stream, state, desired_columns, md_map) return state
def do_sync_incremental(conn_config, stream, state, desired_columns, md_map): replication_key = md_map.get((), {}).get("replication-key") LOGGER.info( "Stream %s is using incremental replication with replication key %s", stream["tap_stream_id"], replication_key, ) stream_state = state.get("bookmarks", {}).get(stream["tap_stream_id"]) illegal_bk_keys = set(stream_state.keys()).difference( set([ "replication_key", "replication_key_value", "version", "last_replication_method" ])) if len(illegal_bk_keys) != 0: raise Exception( "invalid keys found in state: {}".format(illegal_bk_keys)) state = singer.write_bookmark(state, stream["tap_stream_id"], "replication_key", replication_key) sync_common.send_schema_message(stream, [replication_key]) state = incremental.sync_table(conn_config, stream, state, desired_columns, md_map) return state
def test_sync_table_if_not_hstore_available(self, _, mocked_hstore_available): """Test for sync_table_ if hstore is unavailable""" desired_columns = ['foo_key'] expected_state_replication_key_value = MockedConnect.cursor.return_value mocked_hstore_available.return_value = False actual_state = incremental.sync_table(self.conn_config, self.stream, self.state, desired_columns, self.md_map) self.assertEqual( expected_state_replication_key_value, actual_state['bookmarks'][ self.stream['tap_stream_id']]['replication_key_value'])
def test_sync_table(self, mocked_register_hstore): """Test for sync_table if it works correctly""" desired_columns = ['foo_key'] self.state['bookmarks'] = {} expected_state_replication_key_value = MockedConnect.cursor.return_value actual_state = incremental.sync_table(self.conn_config, self.stream, self.state, desired_columns, self.md_map) mocked_register_hstore.assert_called() self.assertEqual( expected_state_replication_key_value, actual_state['bookmarks'][self.stream['tap_stream_id']] ['replication_key_value'], )
def test_sync_table_if_rows_saved_is_a_multiply_of_update_bookmark_period( self, mocked_register_hstore, mocked_singer_write): """Test for sync_table if rows_saved is a multiply of UPDATE_BOOKMARK_PERION""" original_update_bookmark_period = incremental.UPDATE_BOOKMARK_PERIOD incremental.UPDATE_BOOKMARK_PERIOD = MockedConnect.cursor.counter_limit - 1 desired_columns = ['foo_key'] expected_state_replication_key_value = MockedConnect.cursor.return_value actual_state = incremental.sync_table(self.conn_config, self.stream, self.state, desired_columns, self.md_map) mocked_register_hstore.assert_called() self.assertEqual( expected_state_replication_key_value, actual_state['bookmarks'][self.stream['tap_stream_id']] ['replication_key_value'], ) incremental.UPDATE_BOOKMARK_PERIOD = original_update_bookmark_period mocked_singer_write.assert_called_with( singer.StateMessage(value=self.state))