def run_offline_online_store_consistency_test(
fs: FeatureStore, fv: FeatureView
) -> None:
now = datetime.utcnow()
# Run materialize()
# use both tz-naive & tz-aware timestamps to test that they're both correctly handled
start_date = (now - timedelta(hours=5)).replace(tzinfo=utc)
end_date = now - timedelta(hours=2)
fs.materialize(feature_views=[fv.name], start_date=start_date, end_date=end_date)
# check result of materialize()
check_offline_and_online_features(
fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3
)
check_offline_and_online_features(
fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None
)
# check prior value for materialize_incremental()
check_offline_and_online_features(
fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4
)
# run materialize_incremental()
fs.materialize_incremental(feature_views=[fv.name], end_date=now)
# check result of materialize_incremental()
check_offline_and_online_features(
fs=fs, fv=fv, driver_id=3, event_timestamp=now, expected_value=5
)
def run_materialization_test(fs: FeatureStore, fv: FeatureView) -> None:
now = datetime.utcnow()
# Run materialize()
# use both tz-naive & tz-aware timestamps to test that they're both correctly handled
start_date = (now - timedelta(hours=5)).replace(tzinfo=utc)
end_date = now - timedelta(hours=2)
fs.materialize([fv.name], start_date, end_date)
# check result of materialize()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 1
}]).to_dict()
assert abs(response_dict[f"{fv.name}__value"][0] - 0.3) < 1e-6
# check prior value for materialize_incremental()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 3
}]).to_dict()
assert abs(response_dict[f"{fv.name}__value"][0] - 4) < 1e-6
# run materialize_incremental()
fs.materialize_incremental(
[fv.name],
now - timedelta(seconds=0),
)
# check result of materialize_incremental()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 3
}]).to_dict()
assert abs(response_dict[f"{fv.name}__value"][0] - 5) < 1e-6
def materialize_incremental_command(end_ts: str, views: List[str]):
"""
Run an incremental materialization job to ingest new data into the online store. Feast will read
all data from the previously ingested point to END_TS from the offline store and write it to the
online store. If you don't specify feature view names using --views, all registered Feature
Views will be incrementally materialized.
END_TS should be in ISO 8601 format, e.g. '2021-07-16T19:20:01'
"""
cli_check_repo(Path.cwd())
store = FeatureStore(repo_path=str(Path.cwd()))
store.materialize_incremental(
feature_views=None if not views else views,
end_date=datetime.fromisoformat(end_ts),
)
def test_bigquery_table_to_datastore_correctness(self):
# create dataset
now = datetime.utcnow()
ts = pd.Timestamp(now).round("ms")
data = {
"id": [1, 2, 1, 3, 3],
"value": [0.1, 0.2, 0.3, 4, 5],
"ts_1": [
ts - timedelta(seconds=4),
ts,
ts - timedelta(seconds=3),
ts - timedelta(seconds=4),
ts - timedelta(seconds=1),
],
"created_ts": [ts, ts, ts, ts, ts],
}
df = pd.DataFrame.from_dict(data)
# load dataset into BigQuery
job_config = bigquery.LoadJobConfig()
table_id = f"{self.gcp_project}.{self.bigquery_dataset}.table_correctness_{int(time.time())}"
job = self.client.load_table_from_dataframe(df,
table_id,
job_config=job_config)
job.result()
# create FeatureView
fv = FeatureView(
name="test_bq_table_correctness",
entities=["driver_id"],
features=[Feature("value", ValueType.FLOAT)],
ttl=timedelta(minutes=5),
input=BigQuerySource(
event_timestamp_column="ts",
table_ref=table_id,
created_timestamp_column="created_ts",
field_mapping={
"ts_1": "ts",
"id": "driver_id"
},
date_partition_column="",
),
)
config = RepoConfig(
metadata_store="./metadata.db",
project=f"test_bq_table_correctness_{int(time.time())}",
provider="gcp",
)
fs = FeatureStore(config=config)
fs.apply([fv])
# run materialize()
fs.materialize(
[fv.name],
now - timedelta(seconds=5),
now - timedelta(seconds=2),
)
# check result of materialize()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 1
}]).to_dict()
assert abs(response_dict[f"{fv.name}:value"][0] - 0.3) < 1e-6
# check prior value for materialize_incremental()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 3
}]).to_dict()
assert abs(response_dict[f"{fv.name}:value"][0] - 4) < 1e-6
# run materialize_incremental()
fs.materialize_incremental(
[fv.name],
now - timedelta(seconds=0),
)
# check result of materialize_incremental()
response_dict = fs.get_online_features([f"{fv.name}:value"],
[{
"driver_id": 3
}]).to_dict()
assert abs(response_dict[f"{fv.name}:value"][0] - 5) < 1e-6