def clear_state_on_replication_change(stream, state): md_map = metadata.to_map(stream['metadata']) tap_stream_id = stream['tap_stream_id'] # replication method changed current_replication_method = metadata.get(md_map, (), 'replication-method') last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method') if last_replication_method is not None and (current_replication_method != last_replication_method): log_msg = 'Replication method changed from %s to %s, will re-replicate entire collection %s' LOGGER.info(log_msg, last_replication_method, current_replication_method, tap_stream_id) state = singer.reset_stream(state, tap_stream_id) # replication key changed if current_replication_method == 'INCREMENTAL': last_replication_key = singer.get_bookmark(state, tap_stream_id, 'replication_key_name') current_replication_key = metadata.get(md_map, (), 'replication-key') if last_replication_key is not None and (current_replication_key != last_replication_key): log_msg = 'Replication Key changed from %s to %s, will re-replicate entire collection %s' LOGGER.info(log_msg, last_replication_key, current_replication_key, tap_stream_id) state = singer.reset_stream(state, tap_stream_id) state = singer.write_bookmark(state, tap_stream_id, 'replication_key_name', current_replication_key) state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', current_replication_method) return state
def clear_state_on_replication_change(state, tap_stream_id, replication_key, replication_method): #user changed replication, nuke state last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method') if last_replication_method is not None and (replication_method != last_replication_method): state = singer.reset_stream(state, tap_stream_id) #key changed if replication_method == 'INCREMENTAL': if replication_key != singer.get_bookmark(state, tap_stream_id, 'replication_key'): state = singer.reset_stream(state, tap_stream_id) state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', replication_method) return state
def clear_state_on_replication_change(stream, state): md_map = metadata.to_map(stream['metadata']) tap_stream_id = stream['tap_stream_id'] # replication method changed current_replication_method = metadata.get(md_map, (), 'replication-method') last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method') if last_replication_method is not None and (current_replication_method != last_replication_method): log_msg = 'Replication method changed from %s to %s, will re-replicate entire table %s' LOGGER.info(log_msg, last_replication_method, current_replication_method, tap_stream_id) state = singer.reset_stream(state, tap_stream_id) state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', current_replication_method) return state
def clear_state_on_replication_change(state: Dict, tap_stream_id: str, replication_key: str, replication_method: str) -> Dict: """ Update state if replication method change is detected Returns: new state dictionary """ # user changed replication, nuke state last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method') if last_replication_method is not None and (replication_method != last_replication_method): state = singer.reset_stream(state, tap_stream_id) # key changed if replication_method == 'INCREMENTAL' and \ replication_key != singer.get_bookmark(state, tap_stream_id, 'replication_key'): state = singer.reset_stream(state, tap_stream_id) state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', replication_method) return state