def pa_column_to_proto_column(
feast_value_type: ValueType, column: pa.lib.ChunkedArray
) -> List[ProtoValue]:
type_map: Dict[ValueType, Union[str, Dict[str, Any]]] = {
ValueType.INT32: "int32_val",
ValueType.INT64: "int64_val",
ValueType.FLOAT: "float_val",
ValueType.DOUBLE: "double_val",
ValueType.STRING: "string_val",
ValueType.BYTES: "bytes_val",
ValueType.BOOL: "bool_val",
ValueType.BOOL_LIST: {"bool_list_val": BoolList},
ValueType.BYTES_LIST: {"bytes_list_val": BytesList},
ValueType.STRING_LIST: {"string_list_val": StringList},
ValueType.FLOAT_LIST: {"float_list_val": FloatList},
ValueType.DOUBLE_LIST: {"double_list_val": DoubleList},
ValueType.INT32_LIST: {"int32_list_val": Int32List},
ValueType.INT64_LIST: {"int64_list_val": Int64List},
}
value: Union[str, Dict[str, Any]] = type_map[feast_value_type]
# Process list types
if isinstance(value, dict):
list_param_name = list(value.keys())[0]
return [
ProtoValue(**{list_param_name: value[list_param_name](val=x.as_py())})
for x in column
]
else:
return [ProtoValue(**{value: x.as_py()}) for x in column]
def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
"""
Converts a Python (native, pandas) value to a Feast Proto Value based
on a provided value type
Args:
feast_value_type: The target value type
value: Value that will be converted
Returns:
Feast Value Proto
"""
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
if feast_value_type == ValueType.FLOAT_LIST:
return ProtoValue(float_list_val=FloatList(val=[
item if type(item) in
[np.float32, np.float64] else _type_err(item, np.float32)
for item in value
]))
if feast_value_type == ValueType.DOUBLE_LIST:
return ProtoValue(double_list_val=DoubleList(val=[
item if type(item) in
[np.float64, np.float32] else _type_err(item, np.float64)
for item in value
]))
if feast_value_type == ValueType.INT32_LIST:
return ProtoValue(int32_list_val=Int32List(val=[
item if type(item) is np.int32 else _type_err(item, np.int32)
for item in value
]))
if feast_value_type == ValueType.INT64_LIST:
return ProtoValue(int64_list_val=Int64List(val=[
item if type(item) in
[np.int64, np.int32] else _type_err(item, np.int64)
for item in value
]))
if feast_value_type == ValueType.STRING_LIST:
return ProtoValue(string_list_val=StringList(val=[
item if type(item) in
[np.str_, str] else _type_err(item, np.str_) for item in value
]))
if feast_value_type == ValueType.BOOL_LIST:
return ProtoValue(bool_list_val=BoolList(val=[
item if type(item) in
[np.bool_, bool] else _type_err(item, np.bool_)
for item in value
]))
if feast_value_type == ValueType.BYTES_LIST:
return ProtoValue(bytes_list_val=BytesList(val=[
item if type(item) in
[np.bytes_, bytes] else _type_err(item, np.bytes_)
for item in value
]))
# Handle scalar types below
else:
if pd.isnull(value):
return ProtoValue()
elif feast_value_type == ValueType.INT32:
return ProtoValue(int32_val=int(value))
elif feast_value_type == ValueType.INT64:
return ProtoValue(int64_val=int(value))
elif feast_value_type == ValueType.FLOAT:
return ProtoValue(float_val=float(value))
elif feast_value_type == ValueType.DOUBLE:
assert type(value) is float or np.float64
return ProtoValue(double_val=value)
elif feast_value_type == ValueType.STRING:
return ProtoValue(string_val=str(value))
elif feast_value_type == ValueType.BYTES:
assert type(value) is bytes
return ProtoValue(bytes_val=value)
elif feast_value_type == ValueType.BOOL:
assert type(value) is bool
return ProtoValue(bool_val=value)
raise Exception(f"Unsupported data type: ${str(type(value))}")
def pd_value_to_proto_value(feast_value_type, value) -> ProtoValue:
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
if feast_value_type == ValueType.FLOAT_LIST:
return ProtoValue(float_list_val=FloatList(val=[
item if type(item) is np.float32 else type_err(
item, np.float32) for item in value
]))
if feast_value_type == ValueType.DOUBLE_LIST:
return ProtoValue(double_list_val=DoubleList(val=[
item if type(item) is np.float64 else type_err(
item, np.float64) for item in value
]))
if feast_value_type == ValueType.INT32_LIST:
return ProtoValue(int32_list_val=Int32List(val=[
item if type(item) is np.int32 else type_err(item, np.int32)
for item in value
]))
if feast_value_type == ValueType.INT64_LIST:
return ProtoValue(int64_list_val=Int64List(val=[
item if type(item) is np.int64 else type_err(item, np.int64)
for item in value
]))
if feast_value_type == ValueType.STRING_LIST:
return ProtoValue(string_list_val=StringList(val=[
item if type(item) is np.str_ else type_err(item, np.str_)
for item in value
]))
if feast_value_type == ValueType.BOOL_LIST:
return ProtoValue(bool_list_val=BoolList(val=[
item if type(item) is np.bool_ else type_err(item, np.bool_)
for item in value
]))
if feast_value_type == ValueType.BYTES_LIST:
return ProtoValue(bytes_list_val=BytesList(val=[
item if type(item) is np.bytes_ else type_err(item, np.bytes_)
for item in value
]))
# Handle scalar types below
else:
if pd.isnull(value):
return ProtoValue()
elif feast_value_type == ValueType.INT32:
return ProtoValue(int32_val=int(value))
elif feast_value_type == ValueType.INT64:
return ProtoValue(int64_val=int(value))
elif feast_value_type == ValueType.FLOAT:
return ProtoValue(float_val=float(value))
elif feast_value_type == ValueType.DOUBLE:
assert type(value) is float
return ProtoValue(double_val=value)
elif feast_value_type == ValueType.STRING:
return ProtoValue(string_val=str(value))
elif feast_value_type == ValueType.BYTES:
assert type(value) is bytes
return ProtoValue(bytes_val=value)
elif feast_value_type == ValueType.BOOL:
assert type(value) is bool
return ProtoValue(bool_val=value)
raise Exception(f"Unsupported data type: ${str(type(value))}")