def test_get_list_alltypes(client: Client, alltypes_entity: Entity,
alltypes_featuretable: FeatureTable):
# ApplyEntity
client.apply_entity(alltypes_entity)
# GetEntity Check
assert client.get_entity(name="alltypes_id") == alltypes_entity
# ListEntities Check
alltypes_filtering_labels = {"cat": "alltypes"}
actual_alltypes_entities = client.list_entities(
labels=alltypes_filtering_labels)
assert len(actual_alltypes_entities) == 1
# ApplyFeatureTable
client.apply_feature_table(alltypes_featuretable)
# GetFeatureTable Check
actual_get_feature_table = client.get_feature_table(name="alltypes")
assert actual_get_feature_table == alltypes_featuretable
# ListFeatureTables Check
actual_list_feature_table = [
ft for ft in client.list_feature_tables() if ft.name == "alltypes"
][0]
assert actual_list_feature_table == alltypes_featuretable
def test_ingest_into_bq(
feast_client: Client,
customer_entity: Entity,
driver_entity: Entity,
bq_dataframe: pd.DataFrame,
bq_dataset: str,
pytestconfig,
):
bq_project = pytestconfig.getoption("bq_project")
bq_table_id = f"bq_staging_{datetime.now():%Y%m%d%H%M%s}"
ft = FeatureTable(
name="basic_featuretable",
entities=["driver_id", "customer_id"],
features=[
Feature(name="dev_feature_float", dtype=ValueType.FLOAT),
Feature(name="dev_feature_string", dtype=ValueType.STRING),
],
max_age=Duration(seconds=3600),
batch_source=BigQuerySource(
table_ref=f"{bq_project}:{bq_dataset}.{bq_table_id}",
event_timestamp_column="datetime",
created_timestamp_column="timestamp",
),
)
# ApplyEntity
feast_client.apply_entity(customer_entity)
feast_client.apply_entity(driver_entity)
# ApplyFeatureTable
feast_client.apply_feature_table(ft)
feast_client.ingest(ft, bq_dataframe, timeout=120)
bq_client = bigquery.Client(project=bq_project)
# Poll BQ for table until the table has been created
def try_get_table():
try:
table = bq_client.get_table(
bigquery.TableReference(
bigquery.DatasetReference(bq_project, bq_dataset), bq_table_id
)
)
except NotFound:
return None, False
else:
return table, True
wait_retry_backoff(
retry_fn=try_get_table,
timeout_secs=30,
timeout_msg="Timed out trying to get bigquery table",
)
query_string = f"SELECT * FROM `{bq_project}.{bq_dataset}.{bq_table_id}`"
job = bq_client.query(query_string)
query_df = job.to_dataframe()
assert_frame_equal(query_df, bq_dataframe)
def entity_create(filename, project):
"""
Create or update an entity
"""
entities = [
Entity.from_dict(entity_dict) for entity_dict in yaml_loader(filename)
]
feast_client = Client() # type: Client
feast_client.apply_entity(entities, project)
def test_ingest(
client: Client,
customer_entity: Entity,
driver_entity: Entity,
bq_featuretable: FeatureTable,
bq_dataset: pd.DataFrame,
bq_table_id: str,
):
gcp_project, _ = bq_table_id.split(":")
bq_table_id = bq_table_id.replace(":", ".")
# ApplyEntity
client.apply_entity(customer_entity)
client.apply_entity(driver_entity)
# ApplyFeatureTable
client.apply_feature_table(bq_featuretable)
client.ingest(bq_featuretable, bq_dataset, timeout=120)
from google.api_core.exceptions import NotFound
from google.cloud import bigquery
bq_client = bigquery.Client(project=gcp_project)
# Poll BQ for table until the table has been created
def try_get_table():
table_exist = False
table_resp = None
try:
table_resp = bq_client.get_table(bq_table_id)
if table_resp and table_resp.table_id == bq_table_id.split(
".")[-1]:
table_exist = True
except NotFound:
pass
return table_resp, table_exist
wait_retry_backoff(
retry_fn=try_get_table,
timeout_secs=30,
timeout_msg="Timed out trying to get bigquery table",
)
query_string = f"SELECT * FROM `{bq_table_id}`"
job = bq_client.query(query_string)
query_df = job.to_dataframe()
assert_frame_equal(query_df, bq_dataset)
bq_client.delete_table(bq_table_id, not_found_ok=True)
def test_get_list_basic(
client: Client,
customer_entity: Entity,
driver_entity: Entity,
basic_featuretable: FeatureTable,
):
# ApplyEntity
client.apply_entity(customer_entity)
client.apply_entity(driver_entity)
# GetEntity Check
assert client.get_entity(name="customer_id") == customer_entity
assert client.get_entity(name="driver_id") == driver_entity
# ListEntities Check
common_filtering_labels = {"common_key": "common_val"}
matchmaking_filtering_labels = {"team": "matchmaking"}
actual_common_entities = client.list_entities(
labels=common_filtering_labels)
actual_matchmaking_entities = client.list_entities(
labels=matchmaking_filtering_labels)
assert len(actual_common_entities) == 2
assert len(actual_matchmaking_entities) == 1
# ApplyFeatureTable
client.apply_feature_table(basic_featuretable)
# GetFeatureTable Check
actual_get_feature_table = client.get_feature_table(
name="basic_featuretable")
assert actual_get_feature_table == basic_featuretable
# ListFeatureTables Check
actual_list_feature_table = [
ft for ft in client.list_feature_tables()
if ft.name == "basic_featuretable"
][0]
assert actual_list_feature_table == basic_featuretable
def _create_ft(self, client: Client, features) -> None:
entity = Entity(
name="driver_car_id",
description="Car driver id",
value_type=ValueType.STRING,
labels={"team": "matchmaking"},
)
# Register Entity with Core
client.apply_entity(entity)
# Create Feature Tables
batch_source = FileSource(
file_format=ParquetFormat(),
file_url="file://feast/*",
event_timestamp_column="ts_col",
created_timestamp_column="timestamp",
date_partition_column="date_partition_col",
)
stream_source = KafkaSource(
bootstrap_servers="localhost:9094",
message_format=ProtoFormat("class.path"),
topic="test_topic",
event_timestamp_column="ts_col",
created_timestamp_column="timestamp",
)
ft1 = FeatureTable(
name=self.table_name,
features=features,
entities=["driver_car_id"],
labels={"team": "matchmaking"},
batch_source=batch_source,
stream_source=stream_source,
)
# Register Feature Table with Core
client.apply_feature_table(ft1)
