def get_primary_key_from_path(self, column_names: Dict[str, Tuple[str,
str]],
path: List[str]) -> str:
if path and len(path) == 1:
field = path[0]
if not is_airbyte_column(field):
if "type" in self.properties[field]:
property_type = self.properties[field]["type"]
else:
property_type = "object"
if is_number(property_type) or is_boolean(
property_type) or is_array(property_type) or is_object(
property_type):
# some destinations don't handle float columns (or other types) as primary keys, turn everything to string
return f"cast({self.safe_cast_to_string(field, self.properties[field], column_names[field][1])} as {jinja_call('dbt_utils.type_string()')})"
else:
return field
else:
# using an airbyte generated column
return f"cast({field} as {jinja_call('dbt_utils.type_string()')})"
else:
if path:
raise ValueError(
f"Unsupported nested path {'.'.join(path)} for stream {self.stream_name}"
)
else:
raise ValueError(
f"No path specified for stream {self.stream_name}")
def cast_property_type(self, property_name: str, column_name: str,
jinja_column: str) -> str:
definition = self.properties[property_name]
if "type" not in definition:
print(
f"WARN: Unknown type for column {property_name} at {self.current_json_path()}"
)
return column_name
elif is_array(definition["type"]):
return self.cast_property_type_as_array(property_name, column_name)
elif is_object(definition["type"]):
sql_type = self.cast_property_type_as_object(
property_name, column_name)
# Treat simple types from narrower to wider scope type: boolean < integer < number < string
elif is_boolean(definition["type"]):
cast_operation = jinja_call(f"cast_to_boolean({jinja_column})")
return f"{cast_operation} as {column_name}"
elif is_integer(definition["type"]):
sql_type = jinja_call("dbt_utils.type_bigint()")
elif is_number(definition["type"]):
sql_type = jinja_call("dbt_utils.type_float()")
elif is_string(definition["type"]):
sql_type = jinja_call("dbt_utils.type_string()")
else:
print(
f"WARN: Unknown type {definition['type']} for column {property_name} at {self.current_json_path()}"
)
return column_name
return f"cast({column_name} as {sql_type}) as {column_name}"
def safe_cast_to_string(property_name: str, definition: Dict,
column_name: str) -> str:
if "type" not in definition:
return column_name
elif is_boolean(definition["type"]):
return f"boolean_to_string({column_name})"
elif is_array(definition["type"]):
return f"array_to_string({column_name})"
else:
return column_name
def safe_cast_to_string(definition: Dict, column_name: str) -> str:
"""
Note that the result from this static method should always be used within a jinja context (for example, from jinja macro surrogate_key call)
"""
if "type" not in definition:
return column_name
elif is_boolean(definition["type"]):
return f"boolean_to_string({column_name})"
elif is_array(definition["type"]):
return f"array_to_string({column_name})"
else:
return column_name
