def convert_col_type(self, coltype: str) -> str:
"""Converts our metadata types to Athena/Glue versions
Args:
coltype (str): str representation of our metadata column types
Returns:
str: String representation of athena column type version of `coltype`
"""
data_type = _unpack_complex_data_type(coltype)
return _flatten_and_convert_complex_data_type(
data_type, self.convert_basic_col_type)
def reverse_convert_col_type(self, coltype: str):
"""Converts etl-manager metadata col types to Metadata col types
Args:
coltype ([str]): str representation of etl-manager column types
Returns:
[type]: str representation of Metadata col type
"""
data_type = _unpack_complex_etl_type(coltype)
return _flatten_and_convert_complex_data_type(
data_type,
self.reverse_convert_basic_col_type,
complex_dtype_names=("array", "struct"),
)
def reverse_convert_col_type(self, arrow_type: pa.lib.DataType) -> str:
"""Converts an arrow type to a metadata col type
Args:
arrow_field (pa.Field): an arrow field to convert
Returns:
str: str representation of Metadata col type
"""
data_type_dict = _convert_complex_pa_to_data_type(arrow_type)
# Types in are data_type_dict have already beenm converted
# to our agnostic metadata type names so can flatten with
# no converter (aka use this lambda fun)
data_type = _flatten_and_convert_complex_data_type(
data_type=data_type_dict, converter_fun=lambda x: x)
return data_type
def convert_col_type(self, col_type: str):
data_type = _unpack_complex_data_type(col_type)
return _flatten_and_convert_complex_data_type(
data_type, self.convert_basic_col_type, field_sep=",")
