def test_partial() -> None:
"""
Add another table to existing repo using partial apply API. Make sure both the table
applied via CLI apply and the new table are passing RW test.
"""
runner = CliRunner()
with runner.local_repo(get_example_repo("example_feature_repo_1.py"),
"bigquery") as store:
driver_locations_source = BigQuerySource(
table="feast-oss.public.drivers",
timestamp_field="event_timestamp",
created_timestamp_column="created_timestamp",
)
driver_locations_100 = FeatureView(
name="driver_locations_100",
entities=["driver"],
ttl=timedelta(days=1),
schema=[
Field(name="lat", dtype=Float32),
Field(name="lon", dtype=String),
Field(name="name", dtype=String),
],
online=True,
batch_source=driver_locations_source,
tags={},
)
store.apply([driver_locations_100])
basic_rw_test(store, view_name="driver_locations")
basic_rw_test(store, view_name="driver_locations_100")
def test_partial() -> None:
"""
Add another table to existing repo using partial apply API. Make sure both the table
applied via CLI apply and the new table are passing RW test.
"""
runner = CliRunner()
with runner.local_repo(get_example_repo("example_feature_repo_1.py"),
"bigquery") as store:
driver_locations_source = BigQuerySource(
table_ref="feast-oss.public.drivers",
event_timestamp_column="event_timestamp",
created_timestamp_column="created_timestamp",
)
driver_locations_100 = FeatureView(
name="driver_locations_100",
entities=["driver"],
ttl=Duration(seconds=86400 * 1),
features=[
Feature(name="lat", dtype=ValueType.FLOAT),
Feature(name="lon", dtype=ValueType.STRING),
Feature(name="name", dtype=ValueType.STRING),
],
online=True,
batch_source=driver_locations_source,
tags={},
)
store.apply([driver_locations_100])
basic_rw_test(store, view_name="driver_locations")
basic_rw_test(store, view_name="driver_locations_100")
def test_feature_service_read() -> None:
"""
Read feature values from the FeatureStore using a FeatureService.
"""
runner = CliRunner()
with runner.local_repo(get_example_repo("example_feature_repo_1.py"),
"bigquery") as store:
basic_rw_test(
store,
view_name="driver_locations",
feature_service_name="driver_locations_service",
)
def test_basic() -> None:
project_id = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
runner = CliRunner()
with tempfile.TemporaryDirectory(
) as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
repo_path = Path(repo_dir_name)
data_path = Path(data_dir_name)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(
dedent(f"""
project: {project_id}
registry: {data_path / "registry.db"}
provider: aws
online_store:
type: dynamodb
region: us-west-2
offline_store:
type: redshift
cluster_id: feast-integration-tests
region: us-west-2
user: admin
database: feast
s3_staging_location: s3://feast-integration-tests/redshift
iam_role: arn:aws:iam::402087665549:role/redshift_s3_access_role
"""))
repo_example = repo_path / "example.py"
repo_example.write_text(get_example_repo("example_feature_repo_1.py"))
result = runner.run(["apply"], cwd=repo_path)
assert result.returncode == 0
# Doing another apply should be a no op, and should not cause errors
result = runner.run(["apply"], cwd=repo_path)
assert result.returncode == 0
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_locations",
)
result = runner.run(["teardown"], cwd=repo_path)
assert result.returncode == 0
def test_basic() -> None:
project_id = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
runner = CliRunner()
with tempfile.TemporaryDirectory(
) as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
repo_path = Path(repo_dir_name)
data_path = Path(data_dir_name)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(
dedent(f"""
project: {project_id}
registry: {data_path / "registry.db"}
provider: local
offline_store:
type: bigquery
online_store:
type: redis
connection_string: localhost:6379,db=0
"""))
repo_example = repo_path / "example.py"
repo_example.write_text(get_example_repo("example_feature_repo_1.py"))
result = runner.run(["apply"], cwd=repo_path)
assert result.returncode == 0
# Doing another apply should be a no op, and should not cause errors
result = runner.run(["apply"], cwd=repo_path)
assert result.returncode == 0
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_locations",
)
result = runner.run(["teardown"], cwd=repo_path)
assert result.returncode == 0
def test_connection_error() -> None:
project_id = "".join(
random.choice(string.ascii_lowercase + string.digits)
for _ in range(10))
runner = CliRunner()
with tempfile.TemporaryDirectory(
) as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
repo_path = Path(repo_dir_name)
data_path = Path(data_dir_name)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(
dedent(f"""
project: {project_id}
registry: {data_path / "registry.db"}
provider: local
offline_store:
type: file
online_store:
type: redis
connection_string: localhost:6379,db=0=
"""))
repo_example = repo_path / "example.py"
repo_example.write_text(get_example_repo("example_feature_repo_2.py"))
result = runner.run(["apply"], cwd=repo_path)
assert result.returncode == 0
# Redis does not support names for its databases.
with pytest.raises(redis.exceptions.ResponseError):
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_hourly_stats",
)
def test_universal_cli(test_repo_config) -> None:
project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}"
runner = CliRunner()
with tempfile.TemporaryDirectory() as repo_dir_name:
feature_store_yaml = make_feature_store_yaml(project, test_repo_config,
repo_dir_name)
repo_path = Path(repo_dir_name)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(dedent(feature_store_yaml))
repo_example = repo_path / "example.py"
repo_example.write_text(get_example_repo("example_feature_repo_1.py"))
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# Store registry contents, to be compared later.
fs = FeatureStore(repo_path=str(repo_path))
registry_dict = fs.registry.to_dict(project=project)
# entity & feature view list commands should succeed
result = runner.run(["entities", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-views", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-services", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# entity & feature view describe commands should succeed when objects exist
result = runner.run(["entities", "describe", "driver"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-views", "describe", "driver_locations"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(
["feature-services", "describe", "driver_locations_service"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
assertpy.assert_that(fs.list_feature_views()).is_length(3)
# entity & feature view describe commands should fail when objects don't exist
result = runner.run(["entities", "describe", "foo"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-views", "describe", "foo"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-services", "describe", "foo"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
# Doing another apply should be a no op, and should not cause errors
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_locations",
)
# Confirm that registry contents have not changed.
assertpy.assert_that(registry_dict).is_equal_to(
fs.registry.to_dict(project=project))
result = runner.run(["teardown"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
def test_universal_cli(environment: Environment):
project = f"test_universal_cli_{str(uuid.uuid4()).replace('-', '')[:8]}"
runner = CliRunner()
with tempfile.TemporaryDirectory() as repo_dir_name:
try:
repo_path = Path(repo_dir_name)
feature_store_yaml = make_feature_store_yaml(
project, environment.test_repo_config, repo_path)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(dedent(feature_store_yaml))
repo_example = repo_path / "example.py"
repo_example.write_text(
get_example_repo("example_feature_repo_1.py"))
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# Store registry contents, to be compared later.
fs = FeatureStore(repo_path=str(repo_path))
registry_dict = fs.registry.to_dict(project=project)
# Save only the specs, not the metadata.
registry_specs = {
key: [fco["spec"] if "spec" in fco else fco for fco in value]
for key, value in registry_dict.items()
}
# entity & feature view list commands should succeed
result = runner.run(["entities", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-views", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-services", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["data-sources", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# entity & feature view describe commands should succeed when objects exist
result = runner.run(["entities", "describe", "driver"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(
["feature-views", "describe", "driver_locations"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(
["feature-services", "describe", "driver_locations_service"],
cwd=repo_path,
)
assertpy.assert_that(result.returncode).is_equal_to(0)
assertpy.assert_that(fs.list_feature_views()).is_length(4)
result = runner.run(
["data-sources", "describe", "customer_profile_source"],
cwd=repo_path,
)
assertpy.assert_that(result.returncode).is_equal_to(0)
assertpy.assert_that(fs.list_data_sources()).is_length(4)
# entity & feature view describe commands should fail when objects don't exist
result = runner.run(["entities", "describe", "foo"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-views", "describe", "foo"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-services", "describe", "foo"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["data-sources", "describe", "foo"],
cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
# Doing another apply should be a no op, and should not cause errors
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_locations",
)
# Confirm that registry contents have not changed.
registry_dict = fs.registry.to_dict(project=project)
assertpy.assert_that(registry_specs).is_equal_to({
key: [fco["spec"] if "spec" in fco else fco for fco in value]
for key, value in registry_dict.items()
})
result = runner.run(["teardown"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
finally:
runner.run(["teardown"], cwd=repo_path)
def test_workflow() -> None:
"""
Test running apply on a sample repo, and make sure the infra gets created.
"""
runner = CliRunner()
with tempfile.TemporaryDirectory() as repo_dir_name, tempfile.TemporaryDirectory() as data_dir_name:
# Construct an example repo in a temporary dir
repo_path = Path(repo_dir_name)
data_path = Path(data_dir_name)
repo_config = repo_path / "feature_store.yaml"
repo_config.write_text(
dedent(
f"""
project: foo
registry: {data_path / "registry.db"}
provider: local
online_store:
path: {data_path / "online_store.db"}
offline_store:
type: bigquery
"""
)
)
repo_example = repo_path / "example.py"
repo_example.write_text(get_example_repo("example_feature_repo_1.py"))
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# entity & feature view list commands should succeed
result = runner.run(["entities", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-views", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(["feature-services", "list"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
# entity & feature view describe commands should succeed when objects exist
result = runner.run(["entities", "describe", "driver"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(
["feature-views", "describe", "driver_locations"], cwd=repo_path
)
assertpy.assert_that(result.returncode).is_equal_to(0)
result = runner.run(
["feature-services", "describe", "driver_locations_service"], cwd=repo_path
)
assertpy.assert_that(result.returncode).is_equal_to(0)
# entity & feature view describe commands should fail when objects don't exist
result = runner.run(["entities", "describe", "foo"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-views", "describe", "foo"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
result = runner.run(["feature-services", "describe", "foo"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(1)
# Doing another apply should be a no op, and should not cause errors
result = runner.run(["apply"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)
basic_rw_test(
FeatureStore(repo_path=str(repo_path), config=None),
view_name="driver_locations",
)
result = runner.run(["teardown"], cwd=repo_path)
assertpy.assert_that(result.returncode).is_equal_to(0)