def get_max_pk_values(cursor, catalog_entry):
"""Get actual max primary key values from database"""
database_name = common.get_database_name(catalog_entry)
escaped_db = common.escape(database_name)
escaped_table = common.escape(catalog_entry.table)
key_properties = common.get_key_properties(catalog_entry)
escaped_columns = [common.escape(c) for c in key_properties]
sql = """SELECT {}
FROM {}.{}
ORDER BY {}
LIMIT 1
"""
select_column_clause = ', '.join(escaped_columns)
order_column_clause = ', '.join([pk + ' DESC' for pk in escaped_columns])
cursor.execute(
sql.format(select_column_clause, escaped_db, escaped_table,
order_column_clause))
result = cursor.fetchone()
if result:
max_pk_values = dict(zip(key_properties, result))
else:
max_pk_values = {}
return max_pk_values
def generate_pk_clause(catalog_entry, state):
"""Generate primary key where clause to SQL select"""
key_properties = common.get_key_properties(catalog_entry)
escaped_columns = [common.escape(c) for c in key_properties]
max_pk_values = singer.get_bookmark(state, catalog_entry.tap_stream_id,
'max_pk_values')
last_pk_fetched = singer.get_bookmark(state, catalog_entry.tap_stream_id,
'last_pk_fetched')
if last_pk_fetched:
pk_comparisons = [
'({} > {} AND {} <= {})'.format(common.escape(pk),
last_pk_fetched[pk],
common.escape(pk),
max_pk_values[pk])
for pk in key_properties
]
else:
pk_comparisons = [
f'{common.escape(pk)} <= {max_pk_values[pk]}'
for pk in key_properties
]
sql = ' WHERE {} ORDER BY {} ASC'.format(' AND '.join(pk_comparisons),
', '.join(escaped_columns))
return sql
def write_schema_message(catalog_entry, bookmark_properties=None):
key_properties = common.get_key_properties(catalog_entry)
singer.write_message(
singer.SchemaMessage(stream=catalog_entry.stream,
schema=catalog_entry.schema.to_dict(),
key_properties=key_properties,
bookmark_properties=bookmark_properties))