def do_sync_logical_replication(conn_config, stream, state, desired_columns,
md_map):
if get_bookmark(state, stream.tap_stream_id, 'lsn'):
LOGGER.info("Stream %s is using logical replication. end lsn %s",
stream.tap_stream_id,
logical_replication.fetch_current_lsn(conn_config))
logical_replication.add_automatic_properties(stream)
send_schema_message(stream, ['lsn'])
state = logical_replication.sync_table(conn_config, stream, state,
desired_columns, md_map)
else:
#start off with full-table replication
end_lsn = logical_replication.fetch_current_lsn(conn_config)
LOGGER.info(
"Stream %s is using logical replication. performing initial full table sync",
stream.tap_stream_id)
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
#once we are done with full table, write the lsn to the state
state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn',
end_lsn)
return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns,
md_map):
LOGGER.info("Stream %s is using logical replication", stream.tap_stream_id)
if get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#finishing previously interrupted full-table (first stage of logical replication)
LOGGER.info(
"Initial stage of full table sync was interrupted. resuming...")
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
state = singer.write_bookmark(state, stream.tap_stream_id,
'initial_logical_replication_complete',
False)
#inconsistent state
elif get_bookmark(
state, stream.tap_stream_id,
'xmin') and not get_bookmark(state, stream.tap_stream_id, 'lsn'):
raise Exception(
"Xmin found(%s) in state implying full-table replication but no lsn is present"
)
elif not get_bookmark(state,
stream.tap_stream_id, 'xmin') and not get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#initial full-table phase of logical replication
end_lsn = logical_replication.fetch_current_lsn(conn_config)
LOGGER.info("Performing initial full table sync")
state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn',
end_lsn)
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
state = singer.write_bookmark(state, stream.tap_stream_id,
'initial_logical_replication_complete',
False)
elif not get_bookmark(state,
stream.tap_stream_id, 'xmin') and get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#initial stage of logical replication(full-table) has been completed. moving onto pure logical replication
LOGGER.info("Pure Logical Replication upto lsn %s",
logical_replication.fetch_current_lsn(conn_config))
logical_replication.add_automatic_properties(stream)
send_schema_message(stream, ['lsn'])
state = logical_replication.sync_table(conn_config, stream, state,
desired_columns, md_map)
return state
def sync_logical_streams(conn_config, logical_streams, state, end_lsn,
state_file):
"""
Sync streams that use LOG_BASED method
"""
if logical_streams:
LOGGER.info("Pure Logical Replication upto lsn %s for (%s)", end_lsn,
[s['tap_stream_id'] for s in logical_streams])
logical_streams = [
logical_replication.add_automatic_properties(
s, conn_config.get('debug_lsn', False))
for s in logical_streams
]
# Remove LOG_BASED stream bookmarks from state if it has been de-selected
# This is to avoid sending very old starting and flushing positions to source
selected_streams = set()
for stream in logical_streams:
selected_streams.add(stream['tap_stream_id'])
new_state = dict(currently_syncing=state['currently_syncing'],
bookmarks={})
for stream, bookmark in state['bookmarks'].items():
if bookmark == {} or bookmark[
'last_replication_method'] != 'LOG_BASED' or stream in selected_streams:
new_state['bookmarks'][stream] = bookmark
state = new_state
state = logical_replication.sync_tables(conn_config, logical_streams,
state, end_lsn, state_file)
return state
def sync_logical_streams(conn_config, logical_streams, state, end_lsn):
if logical_streams:
LOGGER.info("Pure Logical Replication upto lsn %s for (%s)", end_lsn, list(map(lambda s: s['tap_stream_id'], logical_streams)))
logical_streams = list(map(lambda s: logical_replication.add_automatic_properties(s, conn_config), logical_streams))
state = logical_replication.sync_tables(conn_config, logical_streams, state, end_lsn)
return state
