def convert_col_type(self, coltype: str) -> str: """Converts our metadata types to Athena/Glue versions Args: coltype (str): str representation of our metadata column types Returns: str: String representation of athena column type version of `coltype` """ data_type = _unpack_complex_data_type(coltype) return _flatten_and_convert_complex_data_type( data_type, self.convert_basic_col_type)
def reverse_convert_col_type(self, coltype: str): """Converts etl-manager metadata col types to Metadata col types Args: coltype ([str]): str representation of etl-manager column types Returns: [type]: str representation of Metadata col type """ data_type = _unpack_complex_etl_type(coltype) return _flatten_and_convert_complex_data_type( data_type, self.reverse_convert_basic_col_type, complex_dtype_names=("array", "struct"), )
def reverse_convert_col_type(self, arrow_type: pa.lib.DataType) -> str: """Converts an arrow type to a metadata col type Args: arrow_field (pa.Field): an arrow field to convert Returns: str: str representation of Metadata col type """ data_type_dict = _convert_complex_pa_to_data_type(arrow_type) # Types in are data_type_dict have already beenm converted # to our agnostic metadata type names so can flatten with # no converter (aka use this lambda fun) data_type = _flatten_and_convert_complex_data_type( data_type=data_type_dict, converter_fun=lambda x: x) return data_type
def convert_col_type(self, col_type: str): data_type = _unpack_complex_data_type(col_type) return _flatten_and_convert_complex_data_type( data_type, self.convert_basic_col_type, field_sep=",")