def test_get_list_alltypes(client: Client, alltypes_entity: Entity, alltypes_featuretable: FeatureTable): # ApplyEntity client.apply_entity(alltypes_entity) # GetEntity Check assert client.get_entity(name="alltypes_id") == alltypes_entity # ListEntities Check alltypes_filtering_labels = {"cat": "alltypes"} actual_alltypes_entities = client.list_entities( labels=alltypes_filtering_labels) assert len(actual_alltypes_entities) == 1 # ApplyFeatureTable client.apply_feature_table(alltypes_featuretable) # GetFeatureTable Check actual_get_feature_table = client.get_feature_table(name="alltypes") assert actual_get_feature_table == alltypes_featuretable # ListFeatureTables Check actual_list_feature_table = [ ft for ft in client.list_feature_tables() if ft.name == "alltypes" ][0] assert actual_list_feature_table == alltypes_featuretable
def test_ingest_into_bq( feast_client: Client, customer_entity: Entity, driver_entity: Entity, bq_dataframe: pd.DataFrame, bq_dataset: str, pytestconfig, ): bq_project = pytestconfig.getoption("bq_project") bq_table_id = f"bq_staging_{datetime.now():%Y%m%d%H%M%s}" ft = FeatureTable( name="basic_featuretable", entities=["driver_id", "customer_id"], features=[ Feature(name="dev_feature_float", dtype=ValueType.FLOAT), Feature(name="dev_feature_string", dtype=ValueType.STRING), ], max_age=Duration(seconds=3600), batch_source=BigQuerySource( table_ref=f"{bq_project}:{bq_dataset}.{bq_table_id}", event_timestamp_column="datetime", created_timestamp_column="timestamp", ), ) # ApplyEntity feast_client.apply_entity(customer_entity) feast_client.apply_entity(driver_entity) # ApplyFeatureTable feast_client.apply_feature_table(ft) feast_client.ingest(ft, bq_dataframe, timeout=120) bq_client = bigquery.Client(project=bq_project) # Poll BQ for table until the table has been created def try_get_table(): try: table = bq_client.get_table( bigquery.TableReference( bigquery.DatasetReference(bq_project, bq_dataset), bq_table_id ) ) except NotFound: return None, False else: return table, True wait_retry_backoff( retry_fn=try_get_table, timeout_secs=30, timeout_msg="Timed out trying to get bigquery table", ) query_string = f"SELECT * FROM `{bq_project}.{bq_dataset}.{bq_table_id}`" job = bq_client.query(query_string) query_df = job.to_dataframe() assert_frame_equal(query_df, bq_dataframe)
def entity_create(filename, project): """ Create or update an entity """ entities = [ Entity.from_dict(entity_dict) for entity_dict in yaml_loader(filename) ] feast_client = Client() # type: Client feast_client.apply_entity(entities, project)
def test_ingest( client: Client, customer_entity: Entity, driver_entity: Entity, bq_featuretable: FeatureTable, bq_dataset: pd.DataFrame, bq_table_id: str, ): gcp_project, _ = bq_table_id.split(":") bq_table_id = bq_table_id.replace(":", ".") # ApplyEntity client.apply_entity(customer_entity) client.apply_entity(driver_entity) # ApplyFeatureTable client.apply_feature_table(bq_featuretable) client.ingest(bq_featuretable, bq_dataset, timeout=120) from google.api_core.exceptions import NotFound from google.cloud import bigquery bq_client = bigquery.Client(project=gcp_project) # Poll BQ for table until the table has been created def try_get_table(): table_exist = False table_resp = None try: table_resp = bq_client.get_table(bq_table_id) if table_resp and table_resp.table_id == bq_table_id.split( ".")[-1]: table_exist = True except NotFound: pass return table_resp, table_exist wait_retry_backoff( retry_fn=try_get_table, timeout_secs=30, timeout_msg="Timed out trying to get bigquery table", ) query_string = f"SELECT * FROM `{bq_table_id}`" job = bq_client.query(query_string) query_df = job.to_dataframe() assert_frame_equal(query_df, bq_dataset) bq_client.delete_table(bq_table_id, not_found_ok=True)
def test_get_list_basic( client: Client, customer_entity: Entity, driver_entity: Entity, basic_featuretable: FeatureTable, ): # ApplyEntity client.apply_entity(customer_entity) client.apply_entity(driver_entity) # GetEntity Check assert client.get_entity(name="customer_id") == customer_entity assert client.get_entity(name="driver_id") == driver_entity # ListEntities Check common_filtering_labels = {"common_key": "common_val"} matchmaking_filtering_labels = {"team": "matchmaking"} actual_common_entities = client.list_entities( labels=common_filtering_labels) actual_matchmaking_entities = client.list_entities( labels=matchmaking_filtering_labels) assert len(actual_common_entities) == 2 assert len(actual_matchmaking_entities) == 1 # ApplyFeatureTable client.apply_feature_table(basic_featuretable) # GetFeatureTable Check actual_get_feature_table = client.get_feature_table( name="basic_featuretable") assert actual_get_feature_table == basic_featuretable # ListFeatureTables Check actual_list_feature_table = [ ft for ft in client.list_feature_tables() if ft.name == "basic_featuretable" ][0] assert actual_list_feature_table == basic_featuretable
def _create_ft(self, client: Client, features) -> None: entity = Entity( name="driver_car_id", description="Car driver id", value_type=ValueType.STRING, labels={"team": "matchmaking"}, ) # Register Entity with Core client.apply_entity(entity) # Create Feature Tables batch_source = FileSource( file_format=ParquetFormat(), file_url="file://feast/*", event_timestamp_column="ts_col", created_timestamp_column="timestamp", date_partition_column="date_partition_col", ) stream_source = KafkaSource( bootstrap_servers="localhost:9094", message_format=ProtoFormat("class.path"), topic="test_topic", event_timestamp_column="ts_col", created_timestamp_column="timestamp", ) ft1 = FeatureTable( name=self.table_name, features=features, entities=["driver_car_id"], labels={"team": "matchmaking"}, batch_source=batch_source, stream_source=stream_source, ) # Register Feature Table with Core client.apply_feature_table(ft1)