def test_secure_channel_creation_with_secure_core_url(
self, _mocked_obj, secure_serving_server):
client = Client(
core_url="localhost:443",
serving_url=f"localhost:{secure_serving_server}",
)
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
_ = client._core_service
_grpc_mock.assert_called_with(
client.core_url, credentials=_mocked_credentials.return_value)
def secure_mock_client(self, mocker):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
core_secure=True,
serving_secure=True,
)
mocker.patch.object(client, "_connect_core")
mocker.patch.object(client, "_connect_serving")
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
def feature_table_describe(name: str, project: str):
"""
Describe a feature table
"""
feast_client = Client() # type: Client
ft = feast_client.get_feature_table(name=name, project=project)
if not ft:
print(f'Feature table with name "{name}" could not be found')
return
print(yaml.dump(yaml.safe_load(str(ft)), default_flow_style=False, sort_keys=False))
def test_secure_channel_creation_with_secure_serving_url(
self,
_mocked_obj,
):
client = Client(core_url="localhost:50051",
serving_url="localhost:443")
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
client._connect_serving()
_grpc_mock.assert_called_with(client.serving_url,
_mocked_credentials.return_value)
def secure_client(self, secure_core_server, secure_serving_server):
root_certificate_credentials = pkgutil.get_data(
__name__, _ROOT_CERTIFICATE_RESOURCE_PATH)
# this is needed to establish a secure connection using self-signed certificates, for the purpose of the test
ssl_channel_credentials = grpc.ssl_channel_credentials(
root_certificates=root_certificate_credentials)
with mock.patch("grpc.ssl_channel_credentials",
MagicMock(return_value=ssl_channel_credentials)):
yield Client(core_url="localhost:50053",
serving_url="localhost:50054",
core_secure=True,
serving_secure=True)
def secure_mock_client_with_auth(self):
client = Client(
core_url=CORE_URL,
serving_url=SERVING_URL,
core_enable_ssl=True,
serving_enable_ssl=True,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
client._core_url = CORE_URL
client._serving_url = SERVING_URL
return client
def client(pytestconfig):
core_url = pytestconfig.getoption("core_url")
serving_url = pytestconfig.getoption("serving_url")
client = Client(
core_url=core_url,
serving_url=serving_url,
)
client.set_project(PROJECT_NAME)
return client
def sync_offline_to_online(feature_table: str, start_time: str, end_time: str):
"""
Sync offline store data to online store
"""
from datetime import datetime
client = Client()
table = client.get_feature_table(feature_table)
client.start_offline_to_online_ingestion(
table,
datetime.strptime(start_time, DATETIME_ISO),
datetime.strptime(end_time, DATETIME_ISO),
)
def project_list():
"""
List all projects
"""
feast_client = Client() # type: Client
table = []
for project in feast_client.list_projects():
table.append([project])
from tabulate import tabulate
print(tabulate(table, headers=["NAME"], tablefmt="plain"))
def feature_set_list():
"""
List all feature sets
"""
feast_client = Client() # type: Client
table = []
for fs in feast_client.list_feature_sets():
table.append([fs.name, fs.version])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
def client(core_url, serving_url, allow_dirty):
# Get client for core and serving
client = Client(core_url=core_url, serving_url=serving_url)
# Ensure Feast core is active, but empty
if not allow_dirty:
feature_sets = client.list_feature_sets()
if len(feature_sets) > 0:
raise Exception(
"Feast cannot have existing feature sets registered. Exiting tests."
)
return client
def feature_set_describe(name: str, version: int):
"""
Describe a feature set
"""
feast_client = Client() # type: Client
fs = feast_client.get_feature_set(name=name, version=version)
if not fs:
print(
f'Feature set with name "{name}" and version "{version}" could not be found'
)
return
print(yaml.dump(yaml.safe_load(str(fs)), default_flow_style=False, sort_keys=False))
def feature_set_list():
"""
List all feature sets
"""
feast_client = Client() # type: Client
table = []
for fs in feast_client.list_feature_sets(project="*", name="*"):
table.append([fs.name, repr(fs)])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "REFERENCE"], tablefmt="plain"))
def feature_set_create(filename):
"""
Create or update a feature set
"""
feature_sets = [
FeatureSet.from_dict(fs_dict) for fs_dict in yaml_loader(filename)
]
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
feast_client.apply(feature_sets)
def test_secure_channel_creation_with_secure_client(self, _mocked_obj):
client = Client(
core_url="localhost:50051",
serving_url="localhost:50052",
serving_enable_ssl=True,
core_enable_ssl=True,
)
with mock.patch("grpc.secure_channel") as _grpc_mock, mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value="test")) as _mocked_credentials:
_ = client._serving_service
_grpc_mock.assert_called_with(
client.serving_url,
credentials=_mocked_credentials.return_value)
def feature_set_list():
"""
List all projects
"""
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
table = []
for project in feast_client.list_projects():
table.append([project])
from tabulate import tabulate
print(tabulate(table, headers=["NAME"], tablefmt="plain"))
def apply(filename):
"""
Apply a configuration to a resource by filename or stdin
"""
resources = [
ResourceFactory.get_resource(res_dict["kind"]).from_dict(res_dict)
for res_dict in loaders.yaml_loader(filename)
]
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
feast_client.apply(resources)
def ingest_job_restart(job_id: str):
"""
Restart job for id.
Waits for the job to fully restart.
"""
# find ingestion job for id
feast_client = Client()
jobs = feast_client.list_ingest_jobs(job_id=job_id)
if len(jobs) < 1:
print(f"Ingestion Job with id {job_id} could not be found")
sys.exit(1)
job = jobs[0]
feast_client.restart_ingest_job(job)
def list():
"""
List all feature sets
"""
feast_client = Client(core_url=feast_config.get_config_property_or_fail(
"core_url")) # type: Client
table = []
for fs in feast_client.list_feature_sets():
table.append([fs.name, fs.version])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "VERSION"], tablefmt="plain"))
def entity_describe(name: str, project: str):
"""
Describe an entity
"""
feast_client = Client() # type: Client
entity = feast_client.get_entity(name=name, project=project)
if not entity:
print(f'Entity with name "{name}" could not be found')
return
print(
yaml.dump(yaml.safe_load(str(entity)),
default_flow_style=False,
sort_keys=False))
def feature_table_list(project: str, labels: str):
"""
List all feature tables
"""
feast_client = Client() # type: Client
labels_dict = _get_labels_dict(labels)
table = []
for ft in feast_client.list_feature_tables(project=project, labels=labels_dict):
table.append([ft.name, ft.entities])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "ENTITIES"], tablefmt="plain"))
def entity_list(project: str, labels: str):
"""
List all entities
"""
feast_client = Client() # type: Client
labels_dict = _get_labels_dict(labels)
table = []
for entity in feast_client.list_entities(project=project, labels=labels_dict):
table.append([entity.name, entity.description, entity.value_type])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "DESCRIPTION", "TYPE"], tablefmt="plain"))
def secure_core_client_with_auth(self, secure_core_server_with_auth):
root_certificate_credentials = pkgutil.get_data(
__name__, _ROOT_CERTIFICATE_RESOURCE_PATH)
ssl_channel_credentials = grpc.ssl_channel_credentials(
root_certificates=root_certificate_credentials)
with mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value=ssl_channel_credentials),
):
yield Client(
core_url=f"localhost:{secure_core_server_with_auth}",
core_enable_ssl=True,
enable_auth=True,
auth_token=_FAKE_JWT_TOKEN,
)
def secure_client(self, secure_core_server, secure_serving_server):
root_certificate_credentials = pkgutil.get_data(
__name__, _ROOT_CERTIFICATE_RESOURCE_PATH)
ssl_channel_credentials = grpc.ssl_channel_credentials(
root_certificates=root_certificate_credentials)
with mock.patch(
"grpc.ssl_channel_credentials",
MagicMock(return_value=ssl_channel_credentials),
):
yield Client(
core_url=f"localhost:{secure_core_server}",
serving_url=f"localhost:{secure_serving_server}",
core_enable_ssl=True,
serving_enable_ssl=True,
)
def infra_teardown(pytestconfig, core_url, serving_url):
client = Client(core_url=core_url, serving_url=serving_url)
client.set_project(PROJECT_NAME)
marker = pytestconfig.getoption("-m")
yield marker
if marker == 'dataflow_runner':
ingest_jobs = client.list_ingest_jobs()
ingest_jobs = [client.list_ingest_jobs(job.id)[0].external_id for job in ingest_jobs if job.status == IngestionJobStatus.RUNNING]
cwd = os.getcwd()
with open(f"{cwd}/ingesting_jobs.txt", "w+") as output:
for job in ingest_jobs:
output.write('%s\n' % job)
else:
print('Cleaning up not required')
def ingest_job_stop(wait: bool, timeout: int, job_id: str):
"""
Stop ingestion job for id.
"""
# find ingestion job for id
feast_client = Client()
jobs = feast_client.list_ingest_jobs(job_id=job_id)
if len(jobs) < 1:
print(f"Ingestion Job with id {job_id} could not be found")
sys.exit(1)
job = jobs[0]
feast_client.stop_ingest_job(job)
# wait for ingestion job to stop
if wait:
job.wait(IngestionJobStatus.ABORTED, timeout=timeout)
def ingest_job_describe(job_id: str):
"""
Describe the ingestion job with the given id.
"""
# find ingestion job for id
feast_client = Client()
jobs = feast_client.list_ingest_jobs(job_id=job_id)
if len(jobs) < 1:
print(f"Ingestion Job with id {job_id} could not be found")
sys.exit(1)
job = jobs[0]
# pretty render ingestion job as yaml
print(
yaml.dump(yaml.safe_load(str(job)),
default_flow_style=False,
sort_keys=False))
def feature_set_list(project: str, name: str, labels: str):
"""
List all feature sets
"""
feast_client = Client() # type: Client
labels_dict = _get_labels_dict(labels)
table = []
for fs in feast_client.list_feature_sets(project=project,
name=name,
labels=labels_dict):
table.append([fs.name, repr(fs)])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "REFERENCE"], tablefmt="plain"))
def feature_list(project: str, entities: str, labels: str):
"""
List all features
"""
feast_client = Client() # type: Client
entities_list = _convert_entity_string_to_list(entities)
labels_dict: Dict[str, str] = _get_labels_dict(labels)
table = []
for feature_ref, feature in feast_client.list_features_by_ref(
project=project, entities=entities_list, labels=labels_dict
).items():
table.append([feature.name, feature.dtype, repr(feature_ref)])
from tabulate import tabulate
print(tabulate(table, headers=["NAME", "DTYPE", "REFERENCE"], tablefmt="plain"))
def version(client_only: bool, **kwargs):
"""
Displays version and connectivity information
"""
try:
feast_versions_dict = {
"sdk": {"version": str(pkg_resources.get_distribution("feast"))}
}
if not client_only:
feast_client = Client(**kwargs)
feast_versions_dict.update(feast_client.version())
print(json.dumps(feast_versions_dict))
except Exception as e:
_logger.error("Error initializing backend store")
_logger.exception(e)
sys.exit(1)
