def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn):
"""
Sync INCREMENTAL and FULL_TABLE streams
"""
LOGGER.info("Beginning sync of stream(%s) with sync method(%s)",
stream['tap_stream_id'], sync_method)
md_map = metadata.to_map(stream['metadata'])
conn_config['dbname'] = md_map.get(()).get('database-name')
desired_columns = [
c for c in stream['schema']['properties'].keys()
if sync_common.should_sync_column(md_map, c)
]
desired_columns.sort()
if len(desired_columns) == 0:
LOGGER.warning(
'There are no columns selected for stream %s, skipping it',
stream['tap_stream_id'])
return state
register_type_adapters(conn_config)
if sync_method == 'full':
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
state = do_sync_full_table(conn_config, stream, state, desired_columns,
md_map)
elif sync_method == 'incremental':
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
state = do_sync_incremental(conn_config, stream, state,
desired_columns, md_map)
elif sync_method == 'logical_initial':
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
LOGGER.info("Performing initial full table sync")
state = singer.write_bookmark(state, stream['tap_stream_id'], 'lsn',
end_lsn)
sync_common.send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream['tap_stream_id'], 'xmin',
None)
elif sync_method == 'logical_initial_interrupted':
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
LOGGER.info(
"Initial stage of full table sync was interrupted. resuming...")
sync_common.send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
else:
raise Exception(
f"unknown sync method {sync_method} for stream {stream['tap_stream_id']}"
)
state = singer.set_currently_syncing(state, None)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns,
md_map):
LOGGER.info("Stream %s is using logical replication", stream.tap_stream_id)
if get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#finishing previously interrupted full-table (first stage of logical replication)
LOGGER.info(
"Initial stage of full table sync was interrupted. resuming...")
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
state = singer.write_bookmark(state, stream.tap_stream_id,
'initial_logical_replication_complete',
False)
#inconsistent state
elif get_bookmark(
state, stream.tap_stream_id,
'xmin') and not get_bookmark(state, stream.tap_stream_id, 'lsn'):
raise Exception(
"Xmin found(%s) in state implying full-table replication but no lsn is present"
)
elif not get_bookmark(state,
stream.tap_stream_id, 'xmin') and not get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#initial full-table phase of logical replication
end_lsn = logical_replication.fetch_current_lsn(conn_config)
LOGGER.info("Performing initial full table sync")
state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn',
end_lsn)
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
state = singer.write_bookmark(state, stream.tap_stream_id,
'initial_logical_replication_complete',
False)
elif not get_bookmark(state,
stream.tap_stream_id, 'xmin') and get_bookmark(
state, stream.tap_stream_id, 'lsn'):
#initial stage of logical replication(full-table) has been completed. moving onto pure logical replication
LOGGER.info("Pure Logical Replication upto lsn %s",
logical_replication.fetch_current_lsn(conn_config))
logical_replication.add_automatic_properties(stream)
send_schema_message(stream, ['lsn'])
state = logical_replication.sync_table(conn_config, stream, state,
desired_columns, md_map)
return state
def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn):
LOGGER.info("Beginning sync of stream(%s) with sync method(%s)",
stream["tap_stream_id"], sync_method)
md_map = metadata.to_map(stream["metadata"])
conn_config["dbname"] = md_map.get(()).get("database-name")
desired_columns = [
c for c in stream["schema"]["properties"].keys()
if sync_common.should_sync_column(md_map, c)
]
desired_columns.sort()
if len(desired_columns) == 0:
LOGGER.warning(
"There are no columns selected for stream %s, skipping it",
stream["tap_stream_id"])
return state
register_type_adapters(conn_config)
if sync_method == "full":
state = singer.set_currently_syncing(state, stream["tap_stream_id"])
state = do_sync_full_table(conn_config, stream, state, desired_columns,
md_map)
elif sync_method == "incremental":
state = singer.set_currently_syncing(state, stream["tap_stream_id"])
state = do_sync_incremental(conn_config, stream, state,
desired_columns, md_map)
elif sync_method == "logical_initial":
state = singer.set_currently_syncing(state, stream["tap_stream_id"])
LOGGER.info("Performing initial full table sync")
state = singer.write_bookmark(state, stream["tap_stream_id"], "lsn",
end_lsn)
sync_common.send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream["tap_stream_id"], "xmin",
None)
elif sync_method == "logical_initial_interrupted":
state = singer.set_currently_syncing(state, stream["tap_stream_id"])
LOGGER.info(
"Initial stage of full table sync was interrupted. resuming...")
sync_common.send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
else:
raise Exception("unknown sync method {} for stream {}".format(
sync_method, stream["tap_stream_id"]))
state = singer.set_currently_syncing(state, None)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns,
md_map):
if get_bookmark(state, stream.tap_stream_id, 'lsn'):
LOGGER.info("Stream %s is using logical replication. end lsn %s",
stream.tap_stream_id,
logical_replication.fetch_current_lsn(conn_config))
logical_replication.add_automatic_properties(stream)
send_schema_message(stream, ['lsn'])
state = logical_replication.sync_table(conn_config, stream, state,
desired_columns, md_map)
else:
#start off with full-table replication
end_lsn = logical_replication.fetch_current_lsn(conn_config)
LOGGER.info(
"Stream %s is using logical replication. performing initial full table sync",
stream.tap_stream_id)
send_schema_message(stream, [])
state = full_table.sync_table(conn_config, stream, state,
desired_columns, md_map)
state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin',
None)
#once we are done with full table, write the lsn to the state
state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn',
end_lsn)
return state
def do_sync_full_table(conn_config, stream, state, desired_columns, md_map):
LOGGER.info("Stream %s is using full_table replication", stream['tap_stream_id'])
sync_common.send_schema_message(stream, [])
if md_map.get((), {}).get('is-view'):
state = full_table.sync_view(conn_config, stream, state, desired_columns, md_map)
else:
state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map)
return state
