def sync_table(mssql_conn, config, catalog_entry, state, columns):
mssql_conn = MSSQLConnection(config)
common.whitelist_bookmark_keys(BOOKMARK_KEYS, catalog_entry.tap_stream_id,
state)
catalog_metadata = metadata.to_map(catalog_entry.metadata)
stream_metadata = catalog_metadata.get((), {})
replication_key_metadata = stream_metadata.get("replication-key")
replication_key_state = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
"replication_key")
replication_key_value = None
if replication_key_metadata == replication_key_state:
replication_key_value = singer.get_bookmark(
state, catalog_entry.tap_stream_id, "replication_key_value")
else:
state = singer.write_bookmark(state, catalog_entry.tap_stream_id,
"replication_key",
replication_key_metadata)
state = singer.clear_bookmark(state, catalog_entry.tap_stream_id,
"replication_key_value")
stream_version = common.get_stream_version(catalog_entry.tap_stream_id,
state)
state = singer.write_bookmark(state, catalog_entry.tap_stream_id,
"version", stream_version)
activate_version_message = singer.ActivateVersionMessage(
stream=catalog_entry.stream, version=stream_version)
singer.write_message(activate_version_message)
LOGGER.info("Beginning SQL")
with connect_with_backoff(mssql_conn) as open_conn:
with open_conn.cursor() as cur:
select_sql = common.generate_select_sql(catalog_entry, columns)
params = {}
if replication_key_value is not None:
if catalog_entry.schema.properties[
replication_key_metadata].format == "date-time":
replication_key_value = pendulum.parse(
replication_key_value)
select_sql += " WHERE \"{}\" >= %(replication_key_value)s ORDER BY \"{}\" ASC".format(
replication_key_metadata, replication_key_metadata)
params["replication_key_value"] = replication_key_value
elif replication_key_metadata is not None:
select_sql += " ORDER BY \"{}\" ASC".format(
replication_key_metadata)
common.sync_query(cur, catalog_entry, state, select_sql, columns,
stream_version, params)
def do_sync_incremental(mssql_conn, config, catalog_entry, state, columns):
mssql_conn = MSSQLConnection(config)
md_map = metadata.to_map(catalog_entry.metadata)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)
replication_key = md_map.get((), {}).get("replication-key")
write_schema_message(catalog_entry=catalog_entry, bookmark_properties=[replication_key])
LOGGER.info("Schema written")
incremental.sync_table(mssql_conn, config, catalog_entry, state, columns)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
def do_sync_full_table(mssql_conn, config, catalog_entry, state, columns):
key_properties = common.get_key_properties(catalog_entry)
mssql_conn = MSSQLConnection(config)
write_schema_message(catalog_entry)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)
full_table.sync_table(mssql_conn, config, catalog_entry, state, columns, stream_version)
# Prefer initial_full_table_complete going forward
singer.clear_bookmark(state, catalog_entry.tap_stream_id, "version")
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "initial_full_table_complete", True
)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))
def do_sync_full_table(mssql_conn, config, catalog_entry, state, columns):
LOGGER.info("Stream %s is using full table replication",
catalog_entry.stream)
key_properties = common.get_key_properties(catalog_entry)
write_schema_message(catalog_entry)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id,
state)
full_table.sync_table(mssql_conn, catalog_entry, state, columns,
stream_version)
singer.clear_bookmark(state, catalog_entry.tap_stream_id, 'version')
state = singer.write_bookmark(state, catalog_entry.tap_stream_id,
'initial_full_table_complete', True)
singer.write_message(singer.StateMessage(value=copy.deepcopy(state)))