def pa_column_to_proto_column(
feast_value_type: ValueType, column: pa.lib.ChunkedArray
) -> List[ProtoValue]:
type_map: Dict[ValueType, Union[str, Dict[str, Any]]] = {
ValueType.INT32: "int32_val",
ValueType.INT64: "int64_val",
ValueType.FLOAT: "float_val",
ValueType.DOUBLE: "double_val",
ValueType.STRING: "string_val",
ValueType.BYTES: "bytes_val",
ValueType.BOOL: "bool_val",
ValueType.BOOL_LIST: {"bool_list_val": BoolList},
ValueType.BYTES_LIST: {"bytes_list_val": BytesList},
ValueType.STRING_LIST: {"string_list_val": StringList},
ValueType.FLOAT_LIST: {"float_list_val": FloatList},
ValueType.DOUBLE_LIST: {"double_list_val": DoubleList},
ValueType.INT32_LIST: {"int32_list_val": Int32List},
ValueType.INT64_LIST: {"int64_list_val": Int64List},
}
value: Union[str, Dict[str, Any]] = type_map[feast_value_type]
# Process list types
if isinstance(value, dict):
list_param_name = list(value.keys())[0]
return [
ProtoValue(**{list_param_name: value[list_param_name](val=x.as_py())})
for x in column
]
else:
return [ProtoValue(**{value: x.as_py()}) for x in column]
def _python_value_to_proto_value(feast_value_type: ValueType, value: Any) -> ProtoValue:
"""
Converts a Python (native, pandas) value to a Feast Proto Value based
on a provided value type
Args:
feast_value_type: The target value type
value: Value that will be converted
Returns:
Feast Value Proto
"""
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
# Feature can be list but None is still valid
if value is None:
return ProtoValue()
if feast_value_type in PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE:
proto_type, field_name, valid_types = PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE[
feast_value_type
]
f = {
field_name: proto_type(
val=[
item
if type(item) in valid_types
else _type_err(item, valid_types[0])
for item in value
]
)
}
return ProtoValue(**f)
# Handle scalar types below
else:
if pd.isnull(value):
return ProtoValue()
if feast_value_type == ValueType.UNIX_TIMESTAMP:
if isinstance(value, datetime):
return ProtoValue(int64_val=int(value.timestamp()))
elif isinstance(value, Timestamp):
return ProtoValue(int64_val=int(value.ToSeconds()))
return ProtoValue(int64_val=int(value))
if feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE:
(
field_name,
func,
valid_scalar_types,
) = PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE[feast_value_type]
if valid_scalar_types:
assert type(value) in valid_scalar_types
kwargs = {field_name: func(value)}
return ProtoValue(**kwargs)
raise Exception(f"Unsupported data type: ${str(type(value))}")
def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
"""
Converts a Python (native, pandas) value to a Feast Proto Value based
on a provided value type
Args:
feast_value_type: The target value type
value: Value that will be converted
Returns:
Feast Value Proto
"""
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
if feast_value_type == ValueType.FLOAT_LIST:
return ProtoValue(
float_list_val=FloatList(
val=[
item
if type(item) in [np.float32, np.float64]
else _type_err(item, np.float32)
for item in value
]
)
)
if feast_value_type == ValueType.DOUBLE_LIST:
return ProtoValue(
double_list_val=DoubleList(
val=[
item
if type(item) in [np.float64, np.float32]
else _type_err(item, np.float64)
for item in value
]
)
)
if feast_value_type == ValueType.INT32_LIST:
return ProtoValue(
int32_list_val=Int32List(
val=[
item if type(item) is np.int32 else _type_err(item, np.int32)
for item in value
]
)
)
if feast_value_type == ValueType.INT64_LIST:
return ProtoValue(
int64_list_val=Int64List(
val=[
item
if type(item) in [np.int64, np.int32]
else _type_err(item, np.int64)
for item in value
]
)
)
if feast_value_type == ValueType.UNIX_TIMESTAMP_LIST:
return ProtoValue(
int64_list_val=Int64List(
val=[
item
if type(item) in [np.int64, np.int32]
else _type_err(item, np.int64)
for item in value
]
)
)
if feast_value_type == ValueType.STRING_LIST:
return ProtoValue(
string_list_val=StringList(
val=[
item
if type(item) in [np.str_, str]
else _type_err(item, np.str_)
for item in value
]
)
)
if feast_value_type == ValueType.BOOL_LIST:
return ProtoValue(
bool_list_val=BoolList(
val=[
item
if type(item) in [np.bool_, bool]
else _type_err(item, np.bool_)
for item in value
]
)
)
if feast_value_type == ValueType.BYTES_LIST:
return ProtoValue(
bytes_list_val=BytesList(
val=[
item
if type(item) in [np.bytes_, bytes]
else _type_err(item, np.bytes_)
for item in value
]
)
)
# Handle scalar types below
else:
if pd.isnull(value):
return ProtoValue()
elif feast_value_type == ValueType.INT32:
return ProtoValue(int32_val=int(value))
elif feast_value_type == ValueType.INT64:
return ProtoValue(int64_val=int(value))
elif feast_value_type == ValueType.UNIX_TIMESTAMP:
return ProtoValue(int64_val=int(value))
elif feast_value_type == ValueType.FLOAT:
return ProtoValue(float_val=float(value))
elif feast_value_type == ValueType.DOUBLE:
assert type(value) is float or np.float64
return ProtoValue(double_val=value)
elif feast_value_type == ValueType.STRING:
return ProtoValue(string_val=str(value))
elif feast_value_type == ValueType.BYTES:
assert type(value) is bytes
return ProtoValue(bytes_val=value)
elif feast_value_type == ValueType.BOOL:
assert type(value) is bool
return ProtoValue(bool_val=value)
raise Exception(f"Unsupported data type: ${str(type(value))}")
def _python_value_to_proto_value(feast_value_type: ValueType,
values: List[Any]) -> List[ProtoValue]:
"""
Converts a Python (native, pandas) value to a Feast Proto Value based
on a provided value type
Args:
feast_value_type: The target value type
values: List of Values that will be converted
Returns:
List of Feast Value Proto
"""
# ToDo: make a better sample for type checks (more than one element)
sample = next(filter(_non_empty_value, values),
None) # first not empty value
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
# Feature can be list but None is still valid
if feast_value_type in PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE:
proto_type, field_name, valid_types = PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE[
feast_value_type]
if sample is not None and not all(
type(item) in valid_types for item in sample):
first_invalid = next(item for item in sample
if type(item) not in valid_types)
raise _type_err(first_invalid, valid_types[0])
if feast_value_type == ValueType.UNIX_TIMESTAMP_LIST:
int_timestamps_lists = (
_python_datetime_to_int_timestamp(value)
for value in values)
return [
# ProtoValue does actually accept `np.int_` but the typing complains.
ProtoValue(unix_timestamp_list_val=Int64List(val=ts)
) # type: ignore
for ts in int_timestamps_lists
]
if feast_value_type == ValueType.BOOL_LIST:
# ProtoValue does not support conversion of np.bool_ so we need to convert it to support np.bool_.
return [
ProtoValue(
**
{field_name: proto_type(
val=[bool(e) for e in value])}) # type: ignore
if value is not None else ProtoValue() for value in values
]
return [
ProtoValue(**{field_name: proto_type(
val=value)}) # type: ignore
if value is not None else ProtoValue() for value in values
]
# Handle scalar types below
else:
if sample is None:
# all input values are None
return [ProtoValue()] * len(values)
if feast_value_type == ValueType.UNIX_TIMESTAMP:
int_timestamps = _python_datetime_to_int_timestamp(values)
# ProtoValue does actually accept `np.int_` but the typing complains.
return [
ProtoValue(unix_timestamp_val=ts) for ts in int_timestamps
] # type: ignore
(
field_name,
func,
valid_scalar_types,
) = PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE[feast_value_type]
if valid_scalar_types:
assert type(sample) in valid_scalar_types
if feast_value_type == ValueType.BOOL:
# ProtoValue does not support conversion of np.bool_ so we need to convert it to support np.bool_.
return [
ProtoValue(
**{
field_name:
func(
bool(value) if type(value) is np.bool_ else
value # type: ignore
)
}) if not pd.isnull(value) else ProtoValue()
for value in values
]
if feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE:
return [
ProtoValue(**{field_name: func(value)})
if not pd.isnull(value) else ProtoValue() for value in values
]
raise Exception(f"Unsupported data type: ${str(type(values[0]))}")
def _python_value_to_proto_value(
feast_value_type: ValueType, values: List[Any]
) -> List[ProtoValue]:
"""
Converts a Python (native, pandas) value to a Feast Proto Value based
on a provided value type
Args:
feast_value_type: The target value type
values: List of Values that will be converted
Returns:
List of Feast Value Proto
"""
# ToDo: make a better sample for type checks (more than one element)
sample = next(filter(_non_empty_value, values), None) # first not empty value
if sample is None:
# all input values are None or empty lists
return [ProtoValue()] * len(values)
# Detect list type and handle separately
if "list" in feast_value_type.name.lower():
# Feature can be list but None is still valid
if feast_value_type in PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE:
proto_type, field_name, valid_types = PYTHON_LIST_VALUE_TYPE_TO_PROTO_VALUE[
feast_value_type
]
if not all(type(item) in valid_types for item in sample):
first_invalid = next(
item for item in sample if type(item) not in valid_types
)
raise _type_err(first_invalid, valid_types[0])
return [
ProtoValue(**{field_name: proto_type(val=value)})
if value is not None
else ProtoValue()
for value in values
]
# Handle scalar types below
else:
if feast_value_type == ValueType.UNIX_TIMESTAMP:
if isinstance(sample, datetime):
return [
ProtoValue(int64_val=int(value.timestamp())) for value in values
]
elif isinstance(sample, Timestamp):
return [
ProtoValue(int64_val=int(value.ToSeconds())) for value in values
]
return [ProtoValue(int64_val=int(value)) for value in values]
if feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE:
(
field_name,
func,
valid_scalar_types,
) = PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE[feast_value_type]
if valid_scalar_types:
assert type(sample) in valid_scalar_types
return [
ProtoValue(**{field_name: func(value)})
if not pd.isnull(value)
else ProtoValue()
for value in values
]
raise Exception(f"Unsupported data type: ${str(type(values[0]))}")