def test_get_online_features(self, mocked_client, auth_metadata, mocker,
get_online_features_fields_statuses):
ROW_COUNT = 100
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
request = GetOnlineFeaturesRequestV2(project="driver_project")
request.features.extend([
FeatureRefProto(feature_table="driver", name="age"),
FeatureRefProto(feature_table="driver", name="rating"),
FeatureRefProto(feature_table="driver", name="null_value"),
])
receive_response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(0, ROW_COUNT):
fields = get_online_features_fields_statuses[row_number][0]
statuses = get_online_features_fields_statuses[row_number][1]
request.entity_rows.append(
GetOnlineFeaturesRequestV2.EntityRow(
fields={
"driver_id": ValueProto.Value(int64_val=row_number)
}))
entity_rows.append(
{"driver_id": ValueProto.Value(int64_val=row_number)})
receive_response.field_values.append(
GetOnlineFeaturesResponse.FieldValues(fields=fields,
statuses=statuses))
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeaturesV2",
return_value=receive_response,
)
got_response = mocked_client.get_online_features(
entity_rows=entity_rows,
feature_refs=["driver:age", "driver:rating", "driver:null_value"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
mocked_client._serving_service_stub.GetOnlineFeaturesV2.assert_called_with(
request, metadata=auth_metadata, timeout=10)
got_fields = got_response.field_values[1].fields
got_statuses = got_response.field_values[1].statuses
assert (got_fields["driver_id"] == ValueProto.Value(int64_val=1)
and got_statuses["driver_id"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:age"] == ValueProto.Value(int64_val=1)
and got_statuses["driver:age"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT and
got_fields["driver:rating"] == ValueProto.Value(string_val="9")
and got_statuses["driver:rating"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:null_value"] == ValueProto.Value()
and got_statuses["driver:null_value"]
== GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE)
def test_get_online_features(self, mock_client, mocker):
ROW_COUNT = 300
mock_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel("")
)
fields = dict()
for feature_num in range(1, 10):
fields["feature_set_1:1:feature_" + str(feature_num)] = ValueProto.Value(
int64_val=feature_num
)
field_values = GetOnlineFeaturesResponse.FieldValues(fields=fields)
response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(1, ROW_COUNT + 1):
response.field_values.append(field_values)
entity_rows.append(
GetOnlineFeaturesRequest.EntityRow(
fields={"customer_id": ValueProto.Value(int64_val=row_number)}
)
)
mocker.patch.object(
mock_client._serving_service_stub,
"GetOnlineFeatures",
return_value=response,
)
response = mock_client.get_online_features(
entity_rows=entity_rows,
feature_ids=[
"feature_set_1:1:feature_1",
"feature_set_1:1:feature_2",
"feature_set_1:1:feature_3",
"feature_set_1:1:feature_4",
"feature_set_1:1:feature_5",
"feature_set_1:1:feature_6",
"feature_set_1:1:feature_7",
"feature_set_1:1:feature_8",
"feature_set_1:1:feature_9",
],
) # type: GetOnlineFeaturesResponse
assert (
response.field_values[0].fields["feature_set_1:1:feature_1"].int64_val == 1
and response.field_values[0].fields["feature_set_1:1:feature_9"].int64_val
== 9
)
def test_get_online_features(self, mocked_client, mocker):
ROW_COUNT = 300
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
def int_val(x):
return ValueProto.Value(int64_val=x)
request = GetOnlineFeaturesRequest()
request.features.extend([
FeatureRefProto(project="driver_project",
feature_set="driver",
name="age"),
FeatureRefProto(project="driver_project", name="rating"),
])
recieve_response = GetOnlineFeaturesResponse()
for row_number in range(1, ROW_COUNT + 1):
request.entity_rows.append(
GetOnlineFeaturesRequest.EntityRow(
fields={"driver_id": int_val(row_number)})),
field_values = GetOnlineFeaturesResponse.FieldValues(
fields={
"driver_id": int_val(row_number),
"driver_project/driver:age": int_val(1),
"driver_project/rating": int_val(9),
})
recieve_response.field_values.append(field_values)
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeatures",
return_value=recieve_response,
)
got_response = mocked_client.get_online_features(
entity_rows=request.entity_rows,
feature_refs=["driver:age", "rating"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with(
request)
got_fields = got_response.field_values[0].fields
assert (got_fields["driver_id"] == int_val(1)
and got_fields["driver:age"] == int_val(1)
and got_fields["rating"] == int_val(9))
def test_get_online_features(self, mocked_client, mocker):
ROW_COUNT = 300
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
def int_val(x):
return ValueProto.Value(int64_val=x)
# serving can return feature references with projects,
# get_online_features() should strip the project part.
field_values = GetOnlineFeaturesResponse.FieldValues(
fields={
"driver_project/driver:driver_id": int_val(1),
"driver_project/driver_id": int_val(9),
})
response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(1, ROW_COUNT + 1):
response.field_values.append(field_values)
entity_rows.append(
GetOnlineFeaturesRequest.EntityRow(
fields={"customer_id": int_val(row_number)}))
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeatures",
return_value=response,
)
# NOTE: Feast Serving does not allow for feature references
# that specify the same feature in the same request
response = mocked_client.get_online_features(
entity_rows=entity_rows,
feature_refs=["driver:driver_id", "driver_id"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
assert (
response.field_values[0].fields["driver:driver_id"].int64_val == 1
and response.field_values[0].fields["driver_id"].int64_val == 9)
def GetOnlineFeatures(self, request: GetOnlineFeaturesRequest, context):
response = GetOnlineFeaturesResponse(feature_data_sets=[
GetOnlineFeaturesResponse.FeatureDataSet(
name="feature_set_1",
feature_rows=[
FeatureRowProto.FeatureRow(
feature_set="feature_set_1",
event_timestamp=Timestamp(),
fields=[
FieldProto.Field(
name="feature_1",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_2",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_3",
value=ValueProto.Value(float_val=1.2),
),
],
),
FeatureRowProto.FeatureRow(
feature_set="feature_set_1",
event_timestamp=Timestamp(),
fields=[
FieldProto.Field(
name="feature_1",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_2",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_3",
value=ValueProto.Value(float_val=1.2),
),
],
),
FeatureRowProto.FeatureRow(
feature_set="feature_set_1",
event_timestamp=Timestamp(),
fields=[
FieldProto.Field(
name="feature_1",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_2",
value=ValueProto.Value(float_val=1.2),
),
FieldProto.Field(
name="feature_3",
value=ValueProto.Value(float_val=1.2),
),
],
),
],
)
])
return response
def get_online_features(
self,
feature_refs: List[str],
entity_rows: List[GetOnlineFeaturesRequest.EntityRow],
project: Optional[str] = None,
) -> GetOnlineFeaturesResponse:
"""
Retrieves the latest online feature data from Feast Serving
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_set:feature" where "feature_set" & "feature" refer to
the feature and feature set names respectively.
Only the feature name is required.
entity_rows: List of GetFeaturesRequest.EntityRow where each row
contains entities. Timestamp should not be set for online
retrieval. All entity types within a feature
project: Specifies the project which contain the FeatureSets
which the requested features belong to.
Returns:
Returns a list of maps where each item in the list contains the
latest feature values for the provided entities
"""
self._connect_serving()
try:
response = self._serving_service_stub.GetOnlineFeatures(
GetOnlineFeaturesRequest(
features=_build_feature_references(
feature_ref_strs=feature_refs,
project=project
if project is not None else self.project,
),
entity_rows=entity_rows,
))
# collect entity row refs
entity_refs = set()
for entity_row in entity_rows:
entity_refs.update(entity_row.fields.keys())
strip_field_values = []
for field_value in response.field_values:
# strip the project part the string feature references returned from serving
strip_fields = {}
for ref_str, value in field_value.fields.items():
# find and ignore entities
if ref_str in entity_refs:
strip_fields[ref_str] = value
else:
strip_ref_str = repr(
FeatureRef.from_str(ref_str, ignore_project=True))
strip_fields[strip_ref_str] = value
strip_field_values.append(
GetOnlineFeaturesResponse.FieldValues(fields=strip_fields))
del response.field_values[:]
response.field_values.extend(strip_field_values)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return response
def get_online_features(
self,
feature_refs: List[str],
entity_rows: List[GetOnlineFeaturesRequest.EntityRow],
project: Optional[str] = None,
omit_entities: bool = False,
) -> GetOnlineFeaturesResponse:
"""
Retrieves the latest online feature data from Feast Serving
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_set:feature" where "feature_set" & "feature" refer to
the feature and feature set names respectively.
Only the feature name is required.
entity_rows: List of GetFeaturesRequest.EntityRow where each row
contains entities. Timestamp should not be set for online
retrieval. All entity types within a feature
project: Specifies the project which contain the FeatureSets
which the requested features belong to.
omit_entities: If true will omit entity values in the returned feature data.
Returns:
GetOnlineFeaturesResponse containing the feature data in records.
Each EntityRow provided will yield one record, which contains
data fields with data value and field status metadata (if included).
"""
try:
response = self._serving_service.GetOnlineFeatures(
GetOnlineFeaturesRequest(
omit_entities_in_response=omit_entities,
features=_build_feature_references(
feature_ref_strs=feature_refs,
project=project
if project is not None else self.project,
),
entity_rows=entity_rows,
))
entity_refs = {
key
for entity_row in entity_rows
for key in entity_row.fields.keys()
}
# strip the project part the string feature references returned from serving
strip = (
lambda ref: repr(FeatureRef.from_str(ref, ignore_project=True))
if ref not in entity_refs else ref)
strip_field_values = []
for field_value in response.field_values:
keys, fields, statuses = (
field_value.fields.keys(),
field_value.fields,
field_value.statuses,
)
fields_and_statuses = [(strip(key), fields[key], statuses[key])
for key in keys]
keys, fields, statuses = zip(*fields_and_statuses)
strip_field_values.append(
GetOnlineFeaturesResponse.FieldValues(
fields=dict(zip(keys, fields)),
statuses=dict(zip(keys, statuses)),
))
del response.field_values[:]
response.field_values.extend(strip_field_values)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return response
def test_get_online_features(self, mocked_client, auth_metadata, mocker):
ROW_COUNT = 300
mocked_client._serving_service_stub = Serving.ServingServiceStub(
grpc.insecure_channel(""))
def int_val(x):
return ValueProto.Value(int64_val=x)
request = GetOnlineFeaturesRequest(project="driver_project")
request.features.extend([
FeatureRefProto(feature_set="driver", name="age"),
FeatureRefProto(name="rating"),
FeatureRefProto(name="null_value"),
])
recieve_response = GetOnlineFeaturesResponse()
entity_rows = []
for row_number in range(1, ROW_COUNT + 1):
request.entity_rows.append(
GetOnlineFeaturesRequest.EntityRow(
fields={"driver_id": int_val(row_number)}))
entity_rows.append({"driver_id": int_val(row_number)})
field_values = GetOnlineFeaturesResponse.FieldValues(
fields={
"driver_id": int_val(row_number),
"driver:age": int_val(1),
"rating": int_val(9),
"null_value": ValueProto.Value(),
},
statuses={
"driver_id": GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"driver:age":
GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"rating": GetOnlineFeaturesResponse.FieldStatus.PRESENT,
"null_value":
GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE,
},
)
recieve_response.field_values.append(field_values)
mocker.patch.object(
mocked_client._serving_service_stub,
"GetOnlineFeatures",
return_value=recieve_response,
)
got_response = mocked_client.get_online_features(
entity_rows=entity_rows,
feature_refs=["driver:age", "rating", "null_value"],
project="driver_project",
) # type: GetOnlineFeaturesResponse
mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with(
request, metadata=auth_metadata)
got_fields = got_response.field_values[0].fields
got_statuses = got_response.field_values[0].statuses
assert (got_fields["driver_id"] == int_val(1)
and got_statuses["driver_id"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["driver:age"] == int_val(1)
and got_statuses["driver:age"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["rating"] == int_val(9)
and got_statuses["rating"]
== GetOnlineFeaturesResponse.FieldStatus.PRESENT
and got_fields["null_value"] == ValueProto.Value()
and got_statuses["null_value"]
== GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE)
