def get_primary_key_from_path(self, column_names: Dict[str, Tuple[str, str]], path: List[str]) -> str: if path and len(path) == 1: field = path[0] if not is_airbyte_column(field): if "type" in self.properties[field]: property_type = self.properties[field]["type"] else: property_type = "object" if is_number(property_type) or is_boolean( property_type) or is_array(property_type) or is_object( property_type): # some destinations don't handle float columns (or other types) as primary keys, turn everything to string return f"cast({self.safe_cast_to_string(field, self.properties[field], column_names[field][1])} as {jinja_call('dbt_utils.type_string()')})" else: return field else: # using an airbyte generated column return f"cast({field} as {jinja_call('dbt_utils.type_string()')})" else: if path: raise ValueError( f"Unsupported nested path {'.'.join(path)} for stream {self.stream_name}" ) else: raise ValueError( f"No path specified for stream {self.stream_name}")
def cast_property_type(self, property_name: str, column_name: str, jinja_column: str) -> str: definition = self.properties[property_name] if "type" not in definition: print( f"WARN: Unknown type for column {property_name} at {self.current_json_path()}" ) return column_name elif is_array(definition["type"]): return self.cast_property_type_as_array(property_name, column_name) elif is_object(definition["type"]): sql_type = self.cast_property_type_as_object( property_name, column_name) # Treat simple types from narrower to wider scope type: boolean < integer < number < string elif is_boolean(definition["type"]): cast_operation = jinja_call(f"cast_to_boolean({jinja_column})") return f"{cast_operation} as {column_name}" elif is_integer(definition["type"]): sql_type = jinja_call("dbt_utils.type_bigint()") elif is_number(definition["type"]): sql_type = jinja_call("dbt_utils.type_float()") elif is_string(definition["type"]): sql_type = jinja_call("dbt_utils.type_string()") else: print( f"WARN: Unknown type {definition['type']} for column {property_name} at {self.current_json_path()}" ) return column_name return f"cast({column_name} as {sql_type}) as {column_name}"
def safe_cast_to_string(property_name: str, definition: Dict, column_name: str) -> str: if "type" not in definition: return column_name elif is_boolean(definition["type"]): return f"boolean_to_string({column_name})" elif is_array(definition["type"]): return f"array_to_string({column_name})" else: return column_name
def safe_cast_to_string(definition: Dict, column_name: str) -> str: """ Note that the result from this static method should always be used within a jinja context (for example, from jinja macro surrogate_key call) """ if "type" not in definition: return column_name elif is_boolean(definition["type"]): return f"boolean_to_string({column_name})" elif is_array(definition["type"]): return f"array_to_string({column_name})" else: return column_name