def sync_traditional_stream(client: MongoClient, stream: Dict, state: Dict):
"""
Sync given stream
Args:
client: MongoDb client instance
stream: stream to sync
state: state
"""
tap_stream_id = stream['tap_stream_id']
common.COUNTS[tap_stream_id] = 0
common.TIMES[tap_stream_id] = 0
common.SCHEMA_COUNT[tap_stream_id] = 0
common.SCHEMA_TIMES[tap_stream_id] = 0
md_map = metadata.to_map(stream['metadata'])
replication_method = metadata.get(md_map, (), 'replication-method')
if replication_method not in {INCREMENTAL_METHOD, FULL_TABLE_METHOD}:
raise InvalidReplicationMethodException(replication_method,
'replication method needs to be either FULL_TABLE or INCREMENTAL')
database_name = metadata.get(md_map, (), 'database-name')
# Emit a state message to indicate that we've started this stream
state = clear_state_on_replication_change(stream, state)
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
write_schema_message(stream)
common.SCHEMA_COUNT[tap_stream_id] += 1
with metrics.job_timer('sync_table') as timer:
timer.tags['database'] = database_name
timer.tags['table'] = stream['table_name']
collection = client[database_name][stream["table_name"]]
if replication_method == 'FULL_TABLE':
full_table.sync_collection(collection, stream, state)
else:
incremental.sync_collection(collection, stream, state)
state = singer.set_currently_syncing(state, None)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
def sync_stream(client, stream, state):
tap_stream_id = stream['tap_stream_id']
common.COUNTS[tap_stream_id] = 0
common.TIMES[tap_stream_id] = 0
common.SCHEMA_COUNT[tap_stream_id] = 0
common.SCHEMA_TIMES[tap_stream_id] = 0
md_map = metadata.to_map(stream['metadata'])
replication_method = metadata.get(md_map, (), 'replication-method')
database_name = metadata.get(md_map, (), 'database-name')
stream_projection = load_stream_projection(stream)
# Emit a state message to indicate that we've started this stream
state = clear_state_on_replication_change(stream, state)
state = singer.set_currently_syncing(state, stream['tap_stream_id'])
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
write_schema_message(stream)
common.SCHEMA_COUNT[tap_stream_id] += 1
with metrics.job_timer('sync_table') as timer:
timer.tags['database'] = database_name
timer.tags['table'] = stream['table_name']
if replication_method == 'LOG_BASED':
if oplog.oplog_has_aged_out(client, state, tap_stream_id):
# remove all state for stream
# then it will do a full sync and start oplog again.
LOGGER.info("Clearing state because Oplog has aged out")
state.get('bookmarks', {}).pop(tap_stream_id)
# make sure initial full table sync has been completed
if not singer.get_bookmark(state, tap_stream_id,
'initial_full_table_complete'):
msg = 'Must complete full table sync before starting oplog replication for %s'
LOGGER.info(msg, tap_stream_id)
# only mark current ts in oplog on first sync so tap has a
# starting point after the full table sync
if singer.get_bookmark(state, tap_stream_id,
'version') is None:
collection_oplog_ts = oplog.get_latest_ts(client)
oplog.update_bookmarks(state, tap_stream_id,
collection_oplog_ts)
full_table.sync_collection(client, stream, state,
stream_projection)
oplog.sync_collection(client, stream, state, stream_projection)
elif replication_method == 'FULL_TABLE':
full_table.sync_collection(client, stream, state,
stream_projection)
elif replication_method == 'INCREMENTAL':
incremental.sync_collection(client, stream, state,
stream_projection)
else:
raise Exception(
"only FULL_TABLE, LOG_BASED, and INCREMENTAL replication \
methods are supported (you passed {})".format(
replication_method))
state = singer.set_currently_syncing(state, None)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))