def get_online_features( self, feature_ids: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving :param feature_ids: List of feature Ids in the following format [feature_set_name]:[version]:[feature_name] example: ["feature_set_1:6:my_feature_1", "feature_set_1:6:my_feature_2",] :param entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature set must be provided for each entity key. :return: Returns a list of maps where each item in the list contains the latest feature values for the provided entities """ self._connect_serving() try: response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( feature_sets=_build_feature_set_request(feature_ids), entity_rows=entity_rows, )) # type: GetOnlineFeaturesResponse except grpc.RpcError as e: print( format_grpc_exception("GetOnlineFeatures", e.code(), e.details())) else: return response
def test_basic_retrieve_online_success(client, cust_trans_df): # Poll serving for feature values until the correct values are returned while True: time.sleep(1) response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=cust_trans_df.iloc[0]["customer_id"]) }) ], # Test retrieve with different variations of the string feature refs feature_refs=[ "daily_transactions", "total_transactions", ]) # type: GetOnlineFeaturesResponse if response is None: continue returned_daily_transactions = float( response.field_values[0].fields["daily_transactions"].float_val) sent_daily_transactions = float( cust_trans_df.iloc[0]["daily_transactions"]) if math.isclose( sent_daily_transactions, returned_daily_transactions, abs_tol=FLOAT_TOLERANCE, ): break
def test_large_volume_retrieve_online_success(client, large_volume_dataframe): # Poll serving for feature values until the correct values are returned feature_refs = [ "daily_transactions_large", "total_transactions_large", ] while True: response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=large_volume_dataframe.iloc[0] ["customer_id"]) }) ], feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse is_ok = all([ check_online_response(ref, large_volume_dataframe, response) for ref in feature_refs ]) return None, is_ok wait_retry_backoff( retry_fn=try_get_features, timeout_secs=90, timeout_msg="Timed out trying to get online feature values")
def test_large_volume_retrieve_online_success(client, large_volume_dataframe): # Poll serving for feature values until the correct values are returned while True: time.sleep(1) response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=large_volume_dataframe.iloc[0] ["customer_id"]) }) ], feature_ids=[ "customer_transactions_large:1:daily_transactions", "customer_transactions_large:1:total_transactions", ], ) # type: GetOnlineFeaturesResponse if response is None: continue returned_daily_transactions = float(response.field_values[0].fields[ "customer_transactions_large:1:daily_transactions"].float_val) sent_daily_transactions = float( large_volume_dataframe.iloc[0]["daily_transactions"]) if math.isclose( sent_daily_transactions, returned_daily_transactions, abs_tol=FLOAT_TOLERANCE, ): break
def get_online_features( self, feature_ids: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_ids: List of feature Ids in the following format [feature_set_name]:[version]:[feature_name] example: ["feature_set_1:6:my_feature_1", "feature_set_1:6:my_feature_2",] entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature Returns: Returns a list of maps where each item in the list contains the latest feature values for the provided entities """ self._connect_serving() return self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( feature_sets=_build_feature_set_request(feature_ids), entity_rows=entity_rows, )) # type: GetOnlineFeaturesResponse
def test_get_online_features(self, mocked_client, mocker): ROW_COUNT = 300 mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("")) def int_val(x): return ValueProto.Value(int64_val=x) request = GetOnlineFeaturesRequest() request.features.extend([ FeatureRefProto(project="driver_project", feature_set="driver", name="age"), FeatureRefProto(project="driver_project", name="rating"), ]) recieve_response = GetOnlineFeaturesResponse() for row_number in range(1, ROW_COUNT + 1): request.entity_rows.append( GetOnlineFeaturesRequest.EntityRow( fields={"driver_id": int_val(row_number)})), field_values = GetOnlineFeaturesResponse.FieldValues( fields={ "driver_id": int_val(row_number), "driver_project/driver:age": int_val(1), "driver_project/rating": int_val(9), }) recieve_response.field_values.append(field_values) mocker.patch.object( mocked_client._serving_service_stub, "GetOnlineFeatures", return_value=recieve_response, ) got_response = mocked_client.get_online_features( entity_rows=request.entity_rows, feature_refs=["driver:age", "rating"], project="driver_project", ) # type: GetOnlineFeaturesResponse mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with( request) got_fields = got_response.field_values[0].fields assert (got_fields["driver_id"] == int_val(1) and got_fields["driver:age"] == int_val(1) and got_fields["rating"] == int_val(9))
def get_online_features( self, feature_refs: List[str], entity_rows: List[Union[GetOnlineFeaturesRequest.EntityRow, Dict[str, Any]]], project: Optional[str] = None, omit_entities: bool = False, ) -> OnlineResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references that will be returned for each entity. Each feature reference should have the following format: "feature_set:feature" where "feature_set" & "feature" refer to the feature and feature set names respectively. Only the feature name is required. entity_rows: A list of dictionaries where each key is an entity and each value is feast.types.Value or Python native form. project: Optionally specify the the project override. If specified, uses given project for retrieval. Overrides the projects specified in Feature References if also are specified. omit_entities: If true will omit entity values in the returned feature data. Returns: GetOnlineFeaturesResponse containing the feature data in records. Each EntityRow provided will yield one record, which contains data fields with data value and field status metadata (if included). Examples: >>> from feast import Client >>> >>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566") >>> feature_refs = ["daily_transactions"] >>> entity_rows = [{"customer_id": 0},{"customer_id": 1}] >>> >>> online_response = feast_client.get_online_features( >>> feature_refs, entity_rows, project="my_project") >>> online_response_dict = online_response.to_dict() >>> print(online_response_dict) {'daily_transactions': [1.1,1.2], 'customer_id': [0,1]} """ try: response = self._serving_service.GetOnlineFeatures( GetOnlineFeaturesRequest( omit_entities_in_response=omit_entities, features=_build_feature_references( feature_ref_strs=feature_refs), entity_rows=_infer_online_entity_rows(entity_rows), project=project if project is not None else self.project, ), metadata=self._get_grpc_metadata(), ) except grpc.RpcError as e: raise grpc.RpcError(e.details()) response = OnlineResponse(response) return response
def _infer_online_entity_rows( entity_rows: List[Union[GetOnlineFeaturesRequest.EntityRow, Dict[str, Any]]], ) -> List[GetOnlineFeaturesRequest.EntityRow]: """ Builds a list of EntityRow protos from Python native type format passed by user. Args: entity_rows: A list of dictionaries where each key is an entity and each value is feast.types.Value or Python native form. Returns: A list of EntityRow protos parsed from args. """ # Maintain backward compatibility with users providing EntityRow Proto if entity_rows and isinstance(entity_rows[0], GetOnlineFeaturesRequest.EntityRow): warnings.warn( "entity_rows parameter will only be accepting Dict format from Feast v0.7 onwards", DeprecationWarning, ) entity_rows_proto = cast( List[Union[GetOnlineFeaturesRequest.EntityRow]], entity_rows) return entity_rows_proto entity_rows_dicts = cast(List[Dict[str, Any]], entity_rows) entity_row_list = [] entity_type_map = dict() for entity in entity_rows_dicts: fields = {} for key, value in entity.items(): # Allow for feast.types.Value if isinstance(value, Value): proto_value = value else: # Infer the specific type for this row current_dtype = python_type_to_feast_value_type(name=key, value=value) if key not in entity_type_map: entity_type_map[key] = current_dtype else: if current_dtype != entity_type_map[key]: raise TypeError( f"Input entity {key} has mixed types, {current_dtype} and {entity_type_map[key]}. That is not allowed. " ) proto_value = _python_value_to_proto_value( current_dtype, value) fields[key] = proto_value entity_row_list.append( GetOnlineFeaturesRequest.EntityRow(fields=fields)) return entity_row_list
def try_get_features(): response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={"user_id": Value( int64_val=all_types_dataframe.iloc[0]["user_id"])} ) ], feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse is_ok = check_online_response("float_feature", all_types_dataframe, response) return response, is_ok
def test_basic_retrieve_online_multiple_featureset(client, cust_trans_df, driver_df): # Poll serving for feature values until the correct values are returned while True: time.sleep(1) # Test retrieve with different variations of the string feature refs # ie feature set inference for feature refs without specified feature set feature_ref_df_mapping = [ ("customer_transactions:daily_transactions", cust_trans_df), ("driver:rating", driver_df), ("total_transactions", cust_trans_df), ] response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value( int64_val=cust_trans_df.iloc[0]["customer_id"] ), "driver_id": Value( int64_val=driver_df.iloc[0]["driver_id"] ) } ) ], feature_refs=[mapping[0] for mapping in feature_ref_df_mapping], ) # type: GetOnlineFeaturesResponse if response is None: continue def check_response(ingest_df, response, feature_ref): returned_value = float( response.field_values[0] .fields[feature_ref] .float_val ) feature_ref_splits = feature_ref.split(":") if len(feature_ref_splits) == 1: feature_name = feature_ref else: _, feature_name = feature_ref_splits sent_value = float( ingest_df.iloc[0][feature_name]) return math.isclose( sent_value, returned_value, abs_tol=FLOAT_TOLERANCE, ) if all([check_response(df, response, ref) for ref, df in feature_ref_df_mapping]): break
def test_get_online_features(self, mock_client, mocker): ROW_COUNT = 300 mock_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("") ) fields = dict() for feature_num in range(1, 10): fields["feature_set_1:1:feature_" + str(feature_num)] = ValueProto.Value( int64_val=feature_num ) field_values = GetOnlineFeaturesResponse.FieldValues(fields=fields) response = GetOnlineFeaturesResponse() entity_rows = [] for row_number in range(1, ROW_COUNT + 1): response.field_values.append(field_values) entity_rows.append( GetOnlineFeaturesRequest.EntityRow( fields={"customer_id": ValueProto.Value(int64_val=row_number)} ) ) mocker.patch.object( mock_client._serving_service_stub, "GetOnlineFeatures", return_value=response, ) response = mock_client.get_online_features( entity_rows=entity_rows, feature_ids=[ "feature_set_1:1:feature_1", "feature_set_1:1:feature_2", "feature_set_1:1:feature_3", "feature_set_1:1:feature_4", "feature_set_1:1:feature_5", "feature_set_1:1:feature_6", "feature_set_1:1:feature_7", "feature_set_1:1:feature_8", "feature_set_1:1:feature_9", ], ) # type: GetOnlineFeaturesResponse assert ( response.field_values[0].fields["feature_set_1:1:feature_1"].int64_val == 1 and response.field_values[0].fields["feature_set_1:1:feature_9"].int64_val == 9 )
def try_get_features(): response = client.get_online_features(entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=cust_trans_df.iloc[0]["customer_id"]) }) ], feature_refs=feature_refs) # type: GetOnlineFeaturesResponse is_ok = all([ check_online_response(ref, cust_trans_df, response) for ref in feature_refs ]) return response, is_ok
def test_all_types_retrieve_online_success(client, all_types_dataframe): # Poll serving for feature values until the correct values are returned while True: time.sleep(1) response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={"user_id": Value( int64_val=all_types_dataframe.iloc[0]["user_id"])} ) ], feature_refs=[ "float_feature", "int64_feature", "int32_feature", "string_feature", "bytes_feature", "bool_feature", "double_feature", "float_list_feature", "int64_list_feature", "int32_list_feature", "string_list_feature", "bytes_list_feature", "double_list_feature", ], ) # type: GetOnlineFeaturesResponse if response is None: continue returned_float_list = ( response.field_values[0] .fields[PROJECT_NAME+"/float_list_feature"] .float_list_val.val ) sent_float_list = all_types_dataframe.iloc[0]["float_list_feature"] if math.isclose( returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE ): break
def get_online_features( self, feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], default_project: Optional[str] = None, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references in the following format [project]/[feature_name]:[version]. Only the feature name is a required component in the reference. example: ["my_project/my_feature_1:3", "my_project3/my_feature_4:1",] entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature default_project: This project will be used if the project name is not provided in the feature reference Returns: Returns a list of maps where each item in the list contains the latest feature values for the provided entities """ self._connect_serving() try: response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( features=_build_feature_references( feature_refs=feature_refs, default_project=( default_project if not self.project else self.project ), ), entity_rows=entity_rows, ) ) except grpc.RpcError as e: raise grpc.RpcError(e.details()) return response
def try_get_features(): feature_refs = [mapping[0] for mapping in feature_ref_df_mapping] response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=cust_trans_df.iloc[0]["customer_id"]), "driver_id": Value(int64_val=driver_df.iloc[0]["driver_id"]), }) ], feature_refs=feature_refs, ) # type: GetOnlineFeaturesResponse is_ok = all([ check_online_response(ref, df, response) for ref, df in feature_ref_df_mapping ]) return response, is_ok
def test_get_online_features(self, mocked_client, mocker): ROW_COUNT = 300 mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("")) def int_val(x): return ValueProto.Value(int64_val=x) # serving can return feature references with projects, # get_online_features() should strip the project part. field_values = GetOnlineFeaturesResponse.FieldValues( fields={ "driver_project/driver:driver_id": int_val(1), "driver_project/driver_id": int_val(9), }) response = GetOnlineFeaturesResponse() entity_rows = [] for row_number in range(1, ROW_COUNT + 1): response.field_values.append(field_values) entity_rows.append( GetOnlineFeaturesRequest.EntityRow( fields={"customer_id": int_val(row_number)})) mocker.patch.object( mocked_client._serving_service_stub, "GetOnlineFeatures", return_value=response, ) # NOTE: Feast Serving does not allow for feature references # that specify the same feature in the same request response = mocked_client.get_online_features( entity_rows=entity_rows, feature_refs=["driver:driver_id", "driver_id"], project="driver_project", ) # type: GetOnlineFeaturesResponse assert ( response.field_values[0].fields["driver:driver_id"].int64_val == 1 and response.field_values[0].fields["driver_id"].int64_val == 9)
def _infer_online_entity_rows( entity_rows: List[Dict[str, Any]], ) -> List[GetOnlineFeaturesRequest.EntityRow]: """ Builds a list of EntityRow protos from Python native type format passed by user. Args: entity_rows: A list of dictionaries where each key is an entity and each value is feast.types.Value or Python native form. Returns: A list of EntityRow protos parsed from args. """ entity_rows_dicts = cast(List[Dict[str, Any]], entity_rows) entity_row_list = [] entity_type_map = dict() for entity in entity_rows_dicts: fields = {} for key, value in entity.items(): # Allow for feast.types.Value if isinstance(value, Value): proto_value = value else: # Infer the specific type for this row current_dtype = python_type_to_feast_value_type(name=key, value=value) if key not in entity_type_map: entity_type_map[key] = current_dtype else: if current_dtype != entity_type_map[key]: raise TypeError( f"Input entity {key} has mixed types, {current_dtype} and {entity_type_map[key]}. That is not allowed. " ) proto_value = _python_value_to_proto_value( current_dtype, value) fields[key] = proto_value entity_row_list.append( GetOnlineFeaturesRequest.EntityRow(fields=fields)) return entity_row_list
def test_get_online_features(self, mocked_client, auth_metadata, mocker): ROW_COUNT = 300 mocked_client._serving_service_stub = Serving.ServingServiceStub( grpc.insecure_channel("")) def int_val(x): return ValueProto.Value(int64_val=x) request = GetOnlineFeaturesRequest(project="driver_project") request.features.extend([ FeatureRefProto(feature_set="driver", name="age"), FeatureRefProto(name="rating"), FeatureRefProto(name="null_value"), ]) recieve_response = GetOnlineFeaturesResponse() entity_rows = [] for row_number in range(1, ROW_COUNT + 1): request.entity_rows.append( GetOnlineFeaturesRequest.EntityRow( fields={"driver_id": int_val(row_number)})) entity_rows.append({"driver_id": int_val(row_number)}) field_values = GetOnlineFeaturesResponse.FieldValues( fields={ "driver_id": int_val(row_number), "driver:age": int_val(1), "rating": int_val(9), "null_value": ValueProto.Value(), }, statuses={ "driver_id": GetOnlineFeaturesResponse.FieldStatus.PRESENT, "driver:age": GetOnlineFeaturesResponse.FieldStatus.PRESENT, "rating": GetOnlineFeaturesResponse.FieldStatus.PRESENT, "null_value": GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE, }, ) recieve_response.field_values.append(field_values) mocker.patch.object( mocked_client._serving_service_stub, "GetOnlineFeatures", return_value=recieve_response, ) got_response = mocked_client.get_online_features( entity_rows=entity_rows, feature_refs=["driver:age", "rating", "null_value"], project="driver_project", ) # type: GetOnlineFeaturesResponse mocked_client._serving_service_stub.GetOnlineFeatures.assert_called_with( request, metadata=auth_metadata) got_fields = got_response.field_values[0].fields got_statuses = got_response.field_values[0].statuses assert (got_fields["driver_id"] == int_val(1) and got_statuses["driver_id"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["driver:age"] == int_val(1) and got_statuses["driver:age"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["rating"] == int_val(9) and got_statuses["rating"] == GetOnlineFeaturesResponse.FieldStatus.PRESENT and got_fields["null_value"] == ValueProto.Value() and got_statuses["null_value"] == GetOnlineFeaturesResponse.FieldStatus.NULL_VALUE)
def test_all_types(client): all_types_fs = client.get_feature_set(name="all_types", version=1) if all_types_fs is None: # Register new feature set if it doesnt exist all_types_fs = FeatureSet( name="all_types", entities=[Entity(name="user_id", dtype=ValueType.INT64)], features=[ Feature(name="float_feature", dtype=ValueType.FLOAT), Feature(name="int64_feature", dtype=ValueType.INT64), Feature(name="int32_feature", dtype=ValueType.INT32), Feature(name="string_feature", dtype=ValueType.STRING), Feature(name="bytes_feature", dtype=ValueType.BYTES), Feature(name="bool_feature", dtype=ValueType.BOOL), Feature(name="double_feature", dtype=ValueType.DOUBLE), Feature(name="float_list_feature", dtype=ValueType.FLOAT_LIST), Feature(name="int64_list_feature", dtype=ValueType.INT64_LIST), Feature(name="int32_list_feature", dtype=ValueType.INT32_LIST), Feature(name="string_list_feature", dtype=ValueType.STRING_LIST), Feature(name="bytes_list_feature", dtype=ValueType.BYTES_LIST), Feature(name="bool_list_feature", dtype=ValueType.BOOL_LIST), Feature(name="double_list_feature", dtype=ValueType.DOUBLE_LIST), ], max_age=Duration(seconds=3600), ) # Register feature set client.apply(all_types_fs) # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(10) all_types_fs = client.get_feature_set(name="all_types", version=1) if all_types_fs is None: raise Exception( "Client cannot retrieve 'all_types_fs' FeatureSet " "after registration. Either Feast Core does not save the " "FeatureSet correctly or the client needs to wait longer for FeatureSet " "to be committed.") all_types_df = pd.DataFrame({ "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(3)], "user_id": [1001, 1002, 1003], "int32_feature": [np.int32(1), np.int32(2), np.int32(3)], "int64_feature": [np.int64(1), np.int64(2), np.int64(3)], "float_feature": [np.float(0.1), np.float(0.2), np.float(0.3)], "double_feature": [np.float64(0.1), np.float64(0.2), np.float64(0.3)], "string_feature": ["one", "two", "three"], "bytes_feature": [b"one", b"two", b"three"], "bool_feature": [True, False, False], "int32_list_feature": [ np.array([1, 2, 3, 4], dtype=np.int32), np.array([1, 2, 3, 4], dtype=np.int32), np.array([1, 2, 3, 4], dtype=np.int32), ], "int64_list_feature": [ np.array([1, 2, 3, 4], dtype=np.int64), np.array([1, 2, 3, 4], dtype=np.int64), np.array([1, 2, 3, 4], dtype=np.int64), ], "float_list_feature": [ np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float32), np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float32), np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float32), ], "double_list_feature": [ np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float64), np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float64), np.array([1.1, 1.2, 1.3, 1.4], dtype=np.float64), ], "string_list_feature": [ np.array(["one", "two", "three"]), np.array(["one", "two", "three"]), np.array(["one", "two", "three"]), ], "bytes_list_feature": [ np.array([b"one", b"two", b"three"]), np.array([b"one", b"two", b"three"]), np.array([b"one", b"two", b"three"]), ], "bool_list_feature": [ np.array([True, False, True]), np.array([True, False, True]), np.array([True, False, True]), ], }) # Ingest user embedding data all_types_fs.ingest(dataframe=all_types_df) time.sleep(3) # Poll serving for feature values until the correct values are returned while True: time.sleep(1) response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "user_id": Value( int64_val=all_types_df.iloc[0]["user_id"]) }) ], feature_ids=[ "all_types:1:float_feature", "all_types:1:int64_feature", "all_types:1:int32_feature", "all_types:1:string_feature", "all_types:1:bytes_feature", "all_types:1:bool_feature", "all_types:1:double_feature", "all_types:1:float_list_feature", "all_types:1:int64_list_feature", "all_types:1:int32_list_feature", "all_types:1:string_list_feature", "all_types:1:bytes_list_feature", "all_types:1:bool_list_feature", "all_types:1:double_list_feature", ], ) # type: GetOnlineFeaturesResponse if response is None: continue returned_float_list = ( response.field_values[0].fields["all_types:1:float_list_feature"]. float_list_val.val) sent_float_list = all_types_df.iloc[0]["float_list_feature"] # TODO: Add tests for each value and type if math.isclose(returned_float_list[0], sent_float_list[0], abs_tol=FLOAT_TOLERANCE): break # Wait for values to appear in Serving time.sleep(1)
def test_basic(client): cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) # TODO: Fix source handling in Feast Core to support true idempotent # applies. In this case, applying a feature set without a source will # create a new feature set every time. if cust_trans_fs is None: # Load feature set from file cust_trans_fs = FeatureSet.from_yaml("basic/cust_trans_fs.yaml") # Register feature set client.apply(cust_trans_fs) # Feast Core needs some time to fully commit the FeatureSet applied # when there is no existing job yet for the Featureset time.sleep(15) cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1) if cust_trans_fs is None: raise Exception( "Client cannot retrieve 'customer_transactions' FeatureSet " "after registration. Either Feast Core does not save the " "FeatureSet correctly or the client needs to wait longer for FeatureSet " "to be committed.") offset = random.randint(1000, 100000) # ensure a unique key space is used customer_data = pd.DataFrame({ "datetime": [datetime.utcnow().replace(tzinfo=pytz.utc) for _ in range(5)], "customer_id": [offset + inc for inc in range(5)], "daily_transactions": [np.random.rand() for _ in range(5)], "total_transactions": [512 for _ in range(5)], }) # Ingest customer transaction data cust_trans_fs.ingest(dataframe=customer_data) # Poll serving for feature values until the correct values are returned while True: time.sleep(1) response = client.get_online_features( entity_rows=[ GetOnlineFeaturesRequest.EntityRow( fields={ "customer_id": Value(int64_val=customer_data.iloc[0]["customer_id"]) }) ], feature_ids=[ "customer_transactions:1:daily_transactions", "customer_transactions:1:total_transactions", ], ) # type: GetOnlineFeaturesResponse if response is None: continue returned_daily_transactions = float( response.field_values[0]. fields["customer_transactions:1:daily_transactions"].float_val) sent_daily_transactions = float( customer_data.iloc[0]["daily_transactions"]) if math.isclose( sent_daily_transactions, returned_daily_transactions, abs_tol=FLOAT_TOLERANCE, ): break
def get_online_features( self, feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, omit_entities: bool = False, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references that will be returned for each entity. Each feature reference should have the following format: "feature_set:feature" where "feature_set" & "feature" refer to the feature and feature set names respectively. Only the feature name is required. entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature project: Specifies the project which contain the FeatureSets which the requested features belong to. omit_entities: If true will omit entity values in the returned feature data. Returns: GetOnlineFeaturesResponse containing the feature data in records. Each EntityRow provided will yield one record, which contains data fields with data value and field status metadata (if included). """ try: response = self._serving_service.GetOnlineFeatures( GetOnlineFeaturesRequest( omit_entities_in_response=omit_entities, features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, ), entity_rows=entity_rows, )) entity_refs = { key for entity_row in entity_rows for key in entity_row.fields.keys() } # strip the project part the string feature references returned from serving strip = ( lambda ref: repr(FeatureRef.from_str(ref, ignore_project=True)) if ref not in entity_refs else ref) strip_field_values = [] for field_value in response.field_values: keys, fields, statuses = ( field_value.fields.keys(), field_value.fields, field_value.statuses, ) fields_and_statuses = [(strip(key), fields[key], statuses[key]) for key in keys] keys, fields, statuses = zip(*fields_and_statuses) strip_field_values.append( GetOnlineFeaturesResponse.FieldValues( fields=dict(zip(keys, fields)), statuses=dict(zip(keys, statuses)), )) del response.field_values[:] response.field_values.extend(strip_field_values) except grpc.RpcError as e: raise grpc.RpcError(e.details()) return response
def get_online_features( self, feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references that will be returned for each entity. Each feature reference should have the following format: "feature_set:feature" where "feature_set" & "feature" refer to the feature and feature set names respectively. Only the feature name is required. entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature project: Specifies the project which contain the FeatureSets which the requested features belong to. Returns: Returns a list of maps where each item in the list contains the latest feature values for the provided entities """ self._connect_serving() try: response = self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, ), entity_rows=entity_rows, )) # collect entity row refs entity_refs = set() for entity_row in entity_rows: entity_refs.update(entity_row.fields.keys()) strip_field_values = [] for field_value in response.field_values: # strip the project part the string feature references returned from serving strip_fields = {} for ref_str, value in field_value.fields.items(): # find and ignore entities if ref_str in entity_refs: strip_fields[ref_str] = value else: strip_ref_str = repr( FeatureRef.from_str(ref_str, ignore_project=True)) strip_fields[strip_ref_str] = value strip_field_values.append( GetOnlineFeaturesResponse.FieldValues(fields=strip_fields)) del response.field_values[:] response.field_values.extend(strip_field_values) except grpc.RpcError as e: raise grpc.RpcError(e.details()) return response