def row_to_singer_record(catalog_entry, version, db_column_map, row,
time_extracted):
row_to_persist = {}
for column_name, val in row.items():
property_type = catalog_entry.schema.properties[column_name].type
db_column_type = db_column_map.get(column_name)
if isinstance(val,
(datetime.datetime, datetime.date, datetime.timedelta)):
the_utc_date = common.to_utc_datetime_str(val)
row_to_persist[column_name] = the_utc_date
elif db_column_type == FIELD_TYPE.JSON:
row_to_persist[column_name] = json.dumps(json_bytes_to_string(val))
elif 'boolean' in property_type or property_type == 'boolean':
if val is None:
boolean_representation = None
elif val == 0:
boolean_representation = False
elif db_column_type == FIELD_TYPE.BIT:
boolean_representation = int(val) != 0
else:
boolean_representation = True
row_to_persist[column_name] = boolean_representation
else:
row_to_persist[column_name] = val
return singer.RecordMessage(stream=catalog_entry.stream,
record=row_to_persist,
version=version,
time_extracted=time_extracted)
def get_max_pk_values(cursor, catalog_entry):
database_name = common.get_database_name(catalog_entry)
escaped_db = common.escape(database_name)
escaped_table = common.escape(catalog_entry.table)
key_properties = common.get_key_properties(catalog_entry)
escaped_columns = [common.escape(c) for c in key_properties]
sql = """SELECT {}
FROM {}.{}
"""
select_column_clause = ", ".join(
["max(" + pk + ")" for pk in escaped_columns])
cursor.execute(sql.format(select_column_clause, escaped_db, escaped_table))
result = cursor.fetchone()
processed_results = []
for bm in result:
if isinstance(bm,
(datetime.date, datetime.datetime, datetime.timedelta)):
processed_results += [common.to_utc_datetime_str(bm)]
elif bm is not None:
processed_results += [bm]
max_pk_values = {}
if processed_results:
max_pk_values = dict(zip(key_properties, processed_results))
return max_pk_values