def create(name):
"""
Create a feature set
"""
feast_client = Client(
core_url=feast_config.get_config_property_or_fail("core_url")
) # type: Client
feast_client.apply(FeatureSet(name=name))
def test_auth_success_with_insecure_channel_on_core_url(
self, insecure_core_server_with_auth
):
client = Client(
core_url="localhost:50056",
core_enable_auth=True,
core_auth_token=_FAKE_JWT_TOKEN,
)
client.list_feature_sets()
def test_secure_channel_creation_with_secure_client(self, _mocked_obj):
client = Client(core_url="localhost:50053",
serving_url="localhost:50054",
serving_secure=True,
core_secure=True)
with mock.patch("grpc.secure_channel") as _grpc_mock, \
mock.patch("grpc.ssl_channel_credentials", MagicMock(return_value="test")) as _mocked_credentials:
client._connect_core()
_grpc_mock.assert_called_with(client.core_url,
_mocked_credentials.return_value)
def start_stream_to_online(feature_table: str, jar: str):
"""
Start stream to online sync job.
"""
import feast.pyspark.aws.jobs
client = Client()
table = client.get_feature_table(feature_table)
feast.pyspark.aws.jobs.start_stream_to_online(client, table,
[jar] if jar else [])
def secure_mock_client(self, mocker):
client = Client(core_url=CORE_URL,
serving_url=SERVING_URL,
core_secure=True,
serving_secure=True)
mocker.patch.object(client, "_connect_core")
mocker.patch.object(client, "_connect_serving")
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
def ingest(name, version, filename, file_type):
"""
Ingest feature data into a feature set
"""
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
feature_set = feast_client.get_feature_set(name=name, version=version)
feature_set.ingest_file(file_path=filename)
def mock_client_with_auth(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
"""
Sync offline store to online.
"""
import feast.pyspark.aws.jobs
client = Client()
table = client.get_feature_table(feature_table)
feast.pyspark.aws.jobs.sync_offline_to_online(client, table, start_time,
end_time)
def secure_mock_client(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
core_enable_ssl=True,
serving_enable_ssl=True,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
def entity_create(filename, project):
"""
Create or update an entity
"""
entities = [
Entity.from_dict(entity_dict) for entity_dict in yaml_loader(filename)
]
feast_client = Client() # type: Client
feast_client.apply_entity(entities, project)
def feature_table_create(filename):
"""
Create or update a feature table
"""
feature_tables = [
FeatureTable.from_dict(ft_dict) for ft_dict in yaml_loader(filename)
]
feast_client = Client() # type: Client
feast_client.apply_feature_table(feature_tables)
def feature_set_create(filename):
"""
Create or update a feature set
"""
feature_sets = [
FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)
]
feast_client = Client() # type: Client
feast_client.apply(feature_sets)
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
"""
Sync offline store data to online store
"""
from datetime import datetime
client = Client()
table = client.get_feature_table(feature_table)
client.start_offline_to_online_ingestion(
table, datetime.fromisoformat(start_time),
datetime.fromisoformat(end_time))
def feature_table_describe(name: str, project: str):
"""
Describe a feature table
"""
feast_client = Client() # type: Client
ft = feast_client.get_feature_table(name=name, project=project)
if not ft:
print(f'Feature table with name "{name}" could not be found')
return
print(yaml.dump(yaml.safe_load(str(ft)), default_flow_style=False, sort_keys=False))
def client(pytestconfig):
core_url = pytestconfig.getoption("core_url")
serving_url = pytestconfig.getoption("serving_url")
client = Client(
core_url=core_url,
serving_url=serving_url,
)
client.set_project(PROJECT_NAME)
return client
def client(core_url, serving_url, allow_dirty):
# Get client for core and serving
client = Client(core_url=core_url, serving_url=serving_url)
# Ensure Feast core is active, but empty
if not allow_dirty:
feature_sets = client.list_feature_sets()
if len(feature_sets) > 0:
raise Exception(
"Feast cannot have existing feature sets registered. Exiting tests."
)
return client
def project_list():
"""
List all projects
"""
feast_client = Client() # type: Client
table = []
for project in feast_client.list_projects():
table.append([project])
from tabulate import tabulate
print(tabulate(table, headers=["NAME"], tablefmt="plain"))
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
"""
Sync offline store data to online store
"""
from datetime import datetime
client = Client()
table = client.get_feature_table(feature_table)
client.start_offline_to_online_ingestion(
table,
datetime.strptime(start_time, DATETIME_ISO),
datetime.strptime(end_time, DATETIME_ISO),
)
def feature_set_describe(name: str, version: int):
"""
Describe a feature set
"""
feast_client = Client() # type: Client
fs = feast_client.get_feature_set(name=name, version=version)
if not fs:
print(
f'Feature set with name "{name}" and version "{version}" could not be found'
)
return
print(yaml.dump(yaml.safe_load(str(fs)), default_flow_style=False, sort_keys=False))
def feature_set_list():
"""
List all feature sets
"""
feast_client = Client() # type: Client
table = []
for fs in feast_client.list_feature_sets():
table.append([fs.name, fs.version])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
def feature_set_list():
"""
List all feature sets
"""
feast_client = Client() # type: Client
table = []
for fs in feast_client.list_feature_sets(project="*", name="*"):
table.append([fs.name, repr(fs)])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "REFERENCE"], tablefmt="plain"))
def feature_set_create(filename):
"""
Create or update a feature set
"""
feature_sets = [
FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)
]
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
feast_client.apply(feature_sets)
def list():
"""
List all feature sets
"""
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
table = []
for fs in feast_client.list_feature_sets():
table.append([fs.name, fs.version])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
def apply(filename):
"""
Apply a configuration to a resource by filename or stdin
"""
resources = [
ResourceFactory.get_resource(res_dict["kind"]).from_dict(res_dict)
for res_dict in loaders.yaml_loader(filename)
]
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
feast_client.apply(resources)
def test_get_list_alltypes(client: Client, alltypes_entity: Entity,
alltypes_featuretable: FeatureTable):
# ApplyEntity
client.apply_entity(alltypes_entity)
# GetEntity Check
assert client.get_entity(name="alltypes_id") == alltypes_entity
# ListEntities Check
alltypes_filtering_labels = {"cat": "alltypes"}
actual_alltypes_entities = client.list_entities(
labels=alltypes_filtering_labels)
assert len(actual_alltypes_entities) == 1
# ApplyFeatureTable
client.apply_feature_table(alltypes_featuretable)
# GetFeatureTable Check
actual_get_feature_table = client.get_feature_table(name="alltypes")
assert actual_get_feature_table == alltypes_featuretable
# ListFeatureTables Check
actual_list_feature_table = [
ft for ft in client.list_feature_tables() if ft.name == "alltypes"
][0]
assert actual_list_feature_table == alltypes_featuretable
def ingest_job_restart(job_id: str):
"""
Restart job for id.
Waits for the job to fully restart.
"""
# find ingestion job for id
feast_client = Client()
jobs = feast_client.list_ingest_jobs(job_id=job_id)
if len(jobs) < 1:
print(f"Ingestion Job with id {job_id} could not be found")
sys.exit(1)
job = jobs[0]
feast_client.restart_ingest_job(job)
def test_ingest_into_bq(
feast_client: Client,
customer_entity: Entity,
driver_entity: Entity,
bq_dataframe: pd.DataFrame,
bq_dataset: str,
pytestconfig,
):
bq_project = pytestconfig.getoption("bq_project")
bq_table_id = f"bq_staging_{datetime.now():%Y%m%d%H%M%s}"
ft = FeatureTable(
name="basic_featuretable",
entities=["driver_id", "customer_id"],
features=[
Feature(name="dev_feature_float", dtype=ValueType.FLOAT),
Feature(name="dev_feature_string", dtype=ValueType.STRING),
],
max_age=Duration(seconds=3600),
batch_source=BigQuerySource(
table_ref=f"{bq_project}:{bq_dataset}.{bq_table_id}",
event_timestamp_column="datetime",
created_timestamp_column="timestamp",
),
)
# ApplyEntity
feast_client.apply(customer_entity)
feast_client.apply(driver_entity)
# ApplyFeatureTable
feast_client.apply(ft)
feast_client.ingest(ft, bq_dataframe, timeout=120)
bq_client = bigquery.Client(project=bq_project)
# Poll BQ for table until the table has been created
def try_get_table():
try:
table = bq_client.get_table(
bigquery.TableReference(
bigquery.DatasetReference(bq_project, bq_dataset), bq_table_id
)
)
except NotFound:
return None, False
else:
return table, True
wait_retry_backoff(
retry_fn=try_get_table,
timeout_secs=30,
timeout_msg="Timed out trying to get bigquery table",
)
query_string = f"SELECT * FROM `{bq_project}.{bq_dataset}.{bq_table_id}`"
job = bq_client.query(query_string)
query_df = job.to_dataframe()
assert_frame_equal(query_df, bq_dataframe)
def feature_set_list():
"""
List all projects
"""
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
table = []
for project in feast_client.list_projects():
table.append([project])
from tabulate import tabulate
print(tabulate(table, headers=["NAME"], tablefmt="plain"))
def entity_describe(name: str, project: str):
"""
Describe an entity
"""
feast_client = Client() # type: Client
entity = feast_client.get_entity(name=name, project=project)
if not entity:
print(f'Entity with name "{name}" could not be found')
return
print(
yaml.dump(yaml.safe_load(str(entity)),
default_flow_style=False,
sort_keys=False))
def feature_table_list(project: str, labels: str):
"""
List all feature tables
"""
feast_client = Client() # type: Client
labels_dict = _get_labels_dict(labels)
table = []
for ft in feast_client.list_feature_tables(project=project, labels=labels_dict):
table.append([ft.name, ft.entities])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "ENTITIES"], tablefmt="plain"))
