def get_feature_set(self,
name: str,
project: str = None) -> Union[FeatureSet, None]:
"""
Retrieves a feature set.
Args:
project: Feast project that this feature set belongs to
name: Name of feature set
Returns:
Returns either the specified feature set, or raises an exception if
none is found
"""
if project is None:
if self.project is not None:
project = self.project
else:
raise ValueError("No project has been configured.")
try:
get_feature_set_response = self._core_service.GetFeatureSet(
GetFeatureSetRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
) # type: GetFeatureSetResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return FeatureSet.from_proto(get_feature_set_response.feature_set)
def _apply_feature_set(self, feature_set: FeatureSet):
self._connect_core()
feature_set._client = self
valid, message = feature_set.is_valid()
if not valid:
raise Exception(message)
try:
apply_fs_response = self._core_service_stub.ApplyFeatureSet(
ApplyFeatureSetRequest(feature_set=feature_set.to_proto()),
timeout=GRPC_CONNECTION_TIMEOUT_APPLY,
) # type: ApplyFeatureSetResponse
applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set)
if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED:
print(
f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}".'
)
feature_set._update_from_feature_set(applied_fs,
is_dirty=False)
return
if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE:
print(f"No change detected in feature set {feature_set.name}")
return
except grpc.RpcError as e:
print(
format_grpc_exception("ApplyFeatureSet", e.code(),
e.details()))
def get_feature_set(
self,
name: str,
version: int = None,
fail_if_missing: bool = False) -> Union[FeatureSet, None]:
"""
Retrieve a single feature set from Feast Core
:param name: (str) Name of feature set
:param version: (int) Version of feature set
:param fail_if_missing: (bool) Throws an exception if the feature set is not
found
:return: Returns a single feature set
"""
self._connect_core()
try:
get_feature_set_response = self._core_service_stub.GetFeatureSet(
GetFeatureSetRequest(
name=name.strip(),
version=str(version))) # type: GetFeatureSetResponse
feature_set = get_feature_set_response.feature_set
except grpc.RpcError as e:
print(format_grpc_exception("GetFeatureSet", e.code(),
e.details()))
else:
if feature_set is not None:
return FeatureSet.from_proto(feature_set)
if fail_if_missing:
raise Exception(
f'Could not find feature set with name "{name}" and '
f'version "{version}"')
def _apply_feature_set(self, feature_set: FeatureSet):
"""
Registers a single feature set with Feast
Args:
feature_set: Feature set that will be registered
"""
self._connect_core()
feature_set._client = self
feature_set.is_valid()
# Convert the feature set to a request and send to Feast Core
apply_fs_response = self._core_service_stub.ApplyFeatureSet(
ApplyFeatureSetRequest(feature_set=feature_set.to_proto()),
timeout=GRPC_CONNECTION_TIMEOUT_APPLY,
) # type: ApplyFeatureSetResponse
# Extract the returned feature set
applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set)
# If the feature set has changed, update the local copy
if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED:
print(
f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}"'
)
# If no change has been applied, do nothing
if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE:
print(f"No change detected or applied: {feature_set.name}")
# Deep copy from the returned feature set to the local feature set
feature_set._update_from_feature_set(applied_fs)
def list_feature_sets(self) -> List[FeatureSet]:
"""
Retrieve a list of feature sets from Feast Core
Returns:
List of feature sets
"""
self._connect_core()
try:
# Get latest feature sets from Feast Core
feature_set_protos = self._core_service_stub.ListFeatureSets(
ListFeatureSetsRequest()) # type: ListFeatureSetsResponse
except grpc.RpcError as e:
raise Exception(
format_grpc_exception("ListFeatureSets", e.code(),
e.details()))
# Extract feature sets and return
feature_sets = []
for feature_set_proto in feature_set_protos.feature_sets:
feature_set = FeatureSet.from_proto(feature_set_proto)
feature_set._client = self
feature_sets.append(feature_set)
return feature_sets
def test_list_ingest_jobs(self, mocked_client, mocker):
mocker.patch.object(
mocked_client,
"_core_service_stub",
return_value=Core.CoreServiceStub(grpc.insecure_channel("")),
)
feature_set_proto = FeatureSetProto(
spec=FeatureSetSpecProto(
project="test", name="driver", max_age=Duration(seconds=3600),
)
)
mocker.patch.object(
mocked_client._core_service_stub,
"ListIngestionJobs",
return_value=ListIngestionJobsResponse(
jobs=[
IngestJobProto(
id="kafka-to-redis",
external_id="job-2222",
status=IngestionJobStatus.RUNNING,
feature_sets=[feature_set_proto],
source=Source(
type=SourceType.KAFKA,
kafka_source_config=KafkaSourceConfig(
bootstrap_servers="localhost:9092", topic="topic"
),
),
store=Store(name="redis"),
)
]
),
)
# list ingestion jobs by target feature set reference
ingest_jobs = mocked_client.list_ingest_jobs(
feature_set_ref=FeatureSetRef.from_feature_set(
FeatureSet.from_proto(feature_set_proto)
)
)
assert len(ingest_jobs) >= 1
ingest_job = ingest_jobs[0]
assert (
ingest_job.status == IngestionJobStatus.RUNNING
and ingest_job.id == "kafka-to-redis"
and ingest_job.external_id == "job-2222"
and ingest_job.feature_sets[0].name == "driver"
and ingest_job.source.source_type == "Kafka"
)
def _apply_feature_set(self, feature_set: FeatureSet):
"""
Registers a single feature set with Feast
Args:
feature_set: Feature set that will be registered
"""
self._connect_core()
feature_set.is_valid()
feature_set_proto = feature_set.to_proto()
if len(feature_set_proto.spec.project) == 0:
if self.project is None:
raise ValueError(
f"No project found in feature set {feature_set.name}. "
f"Please set the project within the feature set or within "
f"your Feast Client.")
else:
feature_set_proto.spec.project = self.project
# Convert the feature set to a request and send to Feast Core
try:
apply_fs_response = self._core_service_stub.ApplyFeatureSet(
ApplyFeatureSetRequest(feature_set=feature_set_proto),
timeout=self._config.getint(
CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY),
) # type: ApplyFeatureSetResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# Extract the returned feature set
applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set)
# If the feature set has changed, update the local copy
if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED:
print(f'Feature set created: "{applied_fs.name}"')
if apply_fs_response.status == ApplyFeatureSetResponse.Status.UPDATED:
print(f'Feature set updated: "{applied_fs.name}"')
# If no change has been applied, do nothing
if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE:
print(f"No change detected or applied: {feature_set.name}")
# Deep copy from the returned feature set to the local feature set
feature_set._update_from_feature_set(applied_fs)
def list_feature_sets(self,
project: str = None,
name: str = None,
version: str = None) -> List[FeatureSet]:
"""
Retrieve a list of feature sets from Feast Core
Args:
project: Filter feature sets based on project name
name: Filter feature sets based on feature set name
version: Filter feature sets based on version numbf,
Returns:
List of feature sets
"""
self._connect_core()
if project is None:
if self.project is not None:
project = self.project
else:
project = "*"
if name is None:
name = "*"
if version is None:
version = "*"
filter = ListFeatureSetsRequest.Filter(project=project,
feature_set_name=name,
feature_set_version=version)
# Get latest feature sets from Feast Core
feature_set_protos = self._core_service_stub.ListFeatureSets(
ListFeatureSetsRequest(
filter=filter)) # type: ListFeatureSetsResponse
# Extract feature sets and return
feature_sets = []
for feature_set_proto in feature_set_protos.feature_sets:
feature_set = FeatureSet.from_proto(feature_set_proto)
feature_set._client = self
feature_sets.append(feature_set)
return feature_sets
def list_feature_sets(
self,
project: str = None,
name: str = None,
labels: Dict[str, str] = dict()) -> List[FeatureSet]:
"""
Retrieve a list of feature sets from Feast Core
Args:
project: Filter feature sets based on project name
name: Filter feature sets based on feature set name
Returns:
List of feature sets
"""
if project is None:
if self.project is not None:
project = self.project
else:
project = "*"
if name is None:
name = "*"
filter = ListFeatureSetsRequest.Filter(project=project,
feature_set_name=name,
labels=labels)
# Get latest feature sets from Feast Core
feature_set_protos = self._core_service.ListFeatureSets(
ListFeatureSetsRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListFeatureSetsResponse
# Extract feature sets and return
feature_sets = []
for feature_set_proto in feature_set_protos.feature_sets:
feature_set = FeatureSet.from_proto(feature_set_proto)
feature_set._client = self
feature_sets.append(feature_set)
return feature_sets
def _apply_feature_set(self, feature_set: FeatureSet):
"""
Registers a single feature set with Feast
Args:
feature_set: Feature set that will be registered
"""
self._connect_core()
feature_set._client = self
valid, message = feature_set.is_valid()
if not valid:
raise Exception(message)
try:
# Convert the feature set to a request and send to Feast Core
apply_fs_response = self._core_service_stub.ApplyFeatureSet(
ApplyFeatureSetRequest(feature_set=feature_set.to_proto()),
timeout=GRPC_CONNECTION_TIMEOUT_APPLY,
) # type: ApplyFeatureSetResponse
# Extract the returned feature set
applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set)
# If the feature set has changed, update the local copy
if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED:
print(
f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}".'
)
# Deep copy from the returned feature set to the local feature set
feature_set._update_from_feature_set(applied_fs,
is_dirty=False)
return
# If no change has been applied, do nothing
if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE:
print(f"No change detected in feature set {feature_set.name}")
return
except grpc.RpcError as e:
print(
format_grpc_exception("ApplyFeatureSet", e.code(),
e.details()))
def list_feature_sets(self) -> List[FeatureSet]:
"""
Retrieve a list of feature sets from Feast Core
Returns:
List of feature sets
"""
self._connect_core()
# Get latest feature sets from Feast Core
feature_set_protos = self._core_service_stub.ListFeatureSets(
ListFeatureSetsRequest()) # type: ListFeatureSetsResponse
# Extract feature sets and return
feature_sets = []
for feature_set_proto in feature_set_protos.feature_sets:
feature_set = FeatureSet.from_proto(feature_set_proto)
feature_set._client = self
feature_sets.append(feature_set)
return feature_sets
def get_feature_set(self,
name: str,
version: int = None) -> Union[FeatureSet, None]:
"""
Retrieve a single feature set from Feast Core
:param name: Name of feature set
:param version: Version of feature set (int)
:return: Returns a single feature set
"""
self._connect_core()
# TODO: Version should only be integer or unset. Core should handle 'latest'
if version is None:
version = "latest"
elif isinstance(version, int):
version = str(version)
else:
raise ValueError(f"Version {version} is not of type integer.")
try:
get_feature_set_response = self._core_service_stub.GetFeatureSets(
GetFeatureSetsRequest(filter=GetFeatureSetsRequest.Filter(
feature_set_name=name.strip(), feature_set_version=version)
)) # type: GetFeatureSetsResponse
except grpc.RpcError as e:
print(
format_grpc_exception("GetFeatureSets", e.code(), e.details()))
else:
num_feature_sets_found = len(
list(get_feature_set_response.feature_sets))
if num_feature_sets_found == 0:
return None
if num_feature_sets_found > 1:
raise Exception(
f'Found {num_feature_sets_found} feature sets with name "{name}"'
f' and version "{version}": {list(get_feature_set_response.feature_sets)}'
)
return FeatureSet.from_proto(
get_feature_set_response.feature_sets[0])
def get_feature_set(
self,
name: str,
version: int = None,
fail_if_missing: bool = False) -> Union[FeatureSet, None]:
"""
Retrieves a feature set. If no version is specified then the latest
version will be returned.
Args:
name: Name of feature set
version: Version of feature set
fail_if_missing: Raise an error if feature set is not found
Returns:
Returns either the specified feature set, or None if not found
"""
self._connect_core()
try:
name = name.strip()
if version is None:
version = 0
get_feature_set_response = self._core_service_stub.GetFeatureSet(
GetFeatureSetRequest(
name=name, version=version)) # type: GetFeatureSetResponse
feature_set = get_feature_set_response.feature_set
except grpc.RpcError as e:
print(format_grpc_exception("GetFeatureSet", e.code(),
e.details()))
else:
if feature_set is not None:
return FeatureSet.from_proto(feature_set)
if fail_if_missing:
raise Exception(
f'Could not find feature set with name "{name}" and '
f'version "{version}"')
def get_feature_set(self,
name: str,
version: int = None,
project: str = None) -> Union[FeatureSet, None]:
"""
Retrieves a feature set. If no version is specified then the latest
version will be returned.
Args:
project: Feast project that this feature set belongs to
name: Name of feature set
version: Version of feature set
Returns:
Returns either the specified feature set, or raises an exception if
none is found
"""
self._connect_core()
if project is None:
if self.project is not None:
project = self.project
else:
raise ValueError("No project has been configured.")
if version is None:
version = 0
try:
get_feature_set_response = self._core_service_stub.GetFeatureSet(
GetFeatureSetRequest(
project=project, name=name.strip(),
version=int(version))) # type: GetFeatureSetResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return FeatureSet.from_proto(get_feature_set_response.feature_set)
def get_feature_set(self,
name: str,
version: int = None) -> Union[FeatureSet, None]:
"""
Retrieves a feature set. If no version is specified then the latest
version will be returned.
Args:
name: Name of feature set
version: Version of feature set
Returns:
Returns either the specified feature set, or raises an exception if
none is found
"""
self._connect_core()
if version is None:
version = 0
get_feature_set_response = self._core_service_stub.GetFeatureSet(
GetFeatureSetRequest(
name=name.strip(),
version=int(version))) # type: GetFeatureSetResponse
return FeatureSet.from_proto(get_feature_set_response.feature_set)
def feature_sets(self) -> List[FeatureSet]:
"""
Getter for the IngestJob's feature sets
"""
# convert featureset protos to native objects
return [FeatureSet.from_proto(fs) for fs in self.proto.feature_sets]
