def primary_column_names(stream_schema_message):
try:
return [
sql_utils.safe_column_name(p)
for p in stream_schema_message['key_properties']
]
except KeyError:
return []
def column_schema_avro(name, schema_property):
property_type = schema_property['type']
property_format = schema_property.get('format', None)
result = {"name": sql_utils.safe_column_name(name, quotes=False)}
if 'array' in property_type:
try:
items_type = column_schema_avro(name, schema_property['items'])
result_type = {'type': 'array', 'items': items_type['type']}
except KeyError:
result_type = 'string'
elif 'object' in property_type:
items_types = [
column_schema_avro(col, schema_property) for col, schema_property
in schema_property.get('properties', {}).items()
]
if items_types:
result_type = {
'type': 'record',
'name': name + '_properties',
'fields': items_types
}
else:
result_type = 'string'
elif property_format == 'date-time':
result_type = {'type': 'long', 'logicalType': 'timestamp-micros'}
elif property_format == 'date':
result_type = {'type': 'int', 'logicalType': 'date'}
elif property_format == 'time':
result_type = {'type': 'int', 'logicalType': 'time-millis'}
elif 'number' in property_type:
result_type = {
'type': 'bytes',
'logicalType': 'decimal',
'scale': 9,
'precision': 38
}
elif 'integer' in property_type and 'string' in property_type:
result_type = 'string'
elif 'integer' in property_type:
result_type = 'long'
elif 'boolean' in property_type:
result_type = 'boolean'
else:
result_type = 'string'
result['type'] = ['null', result_type]
return result
def record_primary_key_string(self, record):
if len(self.stream_schema_message['key_properties']) == 0:
return None
flatten = flattening.flatten_record(
record, max_level=self.data_flattening_max_level)
primary_keys = [
sql_utils.safe_column_name(p, quotes=False)
for p in self.stream_schema_message['key_properties']
]
try:
key_props = [str(flatten[p]) for p in primary_keys]
except Exception as exc:
logger.info("Cannot find {} primary key(s) in record: {}".format(
primary_keys, flatten))
raise exc
return ','.join(key_props)
def column_names(self):
return [
sql_utils.safe_column_name(name) for name in self.flatten_schema
]