def clear_state_on_replication_change(stream, state):
md_map = metadata.to_map(stream['metadata'])
tap_stream_id = stream['tap_stream_id']
# replication method changed
current_replication_method = metadata.get(md_map, (), 'replication-method')
last_replication_method = singer.get_bookmark(state, tap_stream_id,
'last_replication_method')
if last_replication_method is not None and (current_replication_method !=
last_replication_method):
log_msg = 'Replication method changed from %s to %s, will re-replicate entire collection %s'
LOGGER.info(log_msg, last_replication_method,
current_replication_method, tap_stream_id)
state = singer.reset_stream(state, tap_stream_id)
# replication key changed
if current_replication_method == 'INCREMENTAL':
last_replication_key = singer.get_bookmark(state, tap_stream_id,
'replication_key_name')
current_replication_key = metadata.get(md_map, (), 'replication-key')
if last_replication_key is not None and (current_replication_key !=
last_replication_key):
log_msg = 'Replication Key changed from %s to %s, will re-replicate entire collection %s'
LOGGER.info(log_msg, last_replication_key, current_replication_key,
tap_stream_id)
state = singer.reset_stream(state, tap_stream_id)
state = singer.write_bookmark(state, tap_stream_id,
'replication_key_name',
current_replication_key)
state = singer.write_bookmark(state, tap_stream_id,
'last_replication_method',
current_replication_method)
return state
def clear_state_on_replication_change(state, tap_stream_id, replication_key, replication_method):
#user changed replication, nuke state
last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method')
if last_replication_method is not None and (replication_method != last_replication_method):
state = singer.reset_stream(state, tap_stream_id)
#key changed
if replication_method == 'INCREMENTAL':
if replication_key != singer.get_bookmark(state, tap_stream_id, 'replication_key'):
state = singer.reset_stream(state, tap_stream_id)
state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', replication_method)
return state
def clear_state_on_replication_change(stream, state):
md_map = metadata.to_map(stream['metadata'])
tap_stream_id = stream['tap_stream_id']
# replication method changed
current_replication_method = metadata.get(md_map, (), 'replication-method')
last_replication_method = singer.get_bookmark(state, tap_stream_id, 'last_replication_method')
if last_replication_method is not None and (current_replication_method != last_replication_method):
log_msg = 'Replication method changed from %s to %s, will re-replicate entire table %s'
LOGGER.info(log_msg, last_replication_method, current_replication_method, tap_stream_id)
state = singer.reset_stream(state, tap_stream_id)
state = singer.write_bookmark(state, tap_stream_id, 'last_replication_method', current_replication_method)
return state
def clear_state_on_replication_change(state: Dict, tap_stream_id: str,
replication_key: str,
replication_method: str) -> Dict:
"""
Update state if replication method change is detected
Returns: new state dictionary
"""
# user changed replication, nuke state
last_replication_method = singer.get_bookmark(state, tap_stream_id,
'last_replication_method')
if last_replication_method is not None and (replication_method !=
last_replication_method):
state = singer.reset_stream(state, tap_stream_id)
# key changed
if replication_method == 'INCREMENTAL' and \
replication_key != singer.get_bookmark(state, tap_stream_id, 'replication_key'):
state = singer.reset_stream(state, tap_stream_id)
state = singer.write_bookmark(state, tap_stream_id,
'last_replication_method',
replication_method)
return state
