def get_max_pk_values(cursor, catalog_entry): """Get actual max primary key values from database""" database_name = common.get_database_name(catalog_entry) escaped_db = common.escape(database_name) escaped_table = common.escape(catalog_entry.table) key_properties = common.get_key_properties(catalog_entry) escaped_columns = [common.escape(c) for c in key_properties] sql = """SELECT {} FROM {}.{} ORDER BY {} LIMIT 1 """ select_column_clause = ', '.join(escaped_columns) order_column_clause = ', '.join([pk + ' DESC' for pk in escaped_columns]) cursor.execute( sql.format(select_column_clause, escaped_db, escaped_table, order_column_clause)) result = cursor.fetchone() if result: max_pk_values = dict(zip(key_properties, result)) else: max_pk_values = {} return max_pk_values
def generate_pk_clause(catalog_entry, state): """Generate primary key where clause to SQL select""" key_properties = common.get_key_properties(catalog_entry) escaped_columns = [common.escape(c) for c in key_properties] max_pk_values = singer.get_bookmark(state, catalog_entry.tap_stream_id, 'max_pk_values') last_pk_fetched = singer.get_bookmark(state, catalog_entry.tap_stream_id, 'last_pk_fetched') if last_pk_fetched: pk_comparisons = [ '({} > {} AND {} <= {})'.format(common.escape(pk), last_pk_fetched[pk], common.escape(pk), max_pk_values[pk]) for pk in key_properties ] else: pk_comparisons = [ f'{common.escape(pk)} <= {max_pk_values[pk]}' for pk in key_properties ] sql = ' WHERE {} ORDER BY {} ASC'.format(' AND '.join(pk_comparisons), ', '.join(escaped_columns)) return sql
def write_schema_message(catalog_entry, bookmark_properties=None): key_properties = common.get_key_properties(catalog_entry) singer.write_message( singer.SchemaMessage(stream=catalog_entry.stream, schema=catalog_entry.schema.to_dict(), key_properties=key_properties, bookmark_properties=bookmark_properties))