def get_feature_set(self, name: str, project: str = None) -> Union[FeatureSet, None]: """ Retrieves a feature set. Args: project: Feast project that this feature set belongs to name: Name of feature set Returns: Returns either the specified feature set, or raises an exception if none is found """ if project is None: if self.project is not None: project = self.project else: raise ValueError("No project has been configured.") try: get_feature_set_response = self._core_service.GetFeatureSet( GetFeatureSetRequest(project=project, name=name.strip()), metadata=self._get_grpc_metadata(), ) # type: GetFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) return FeatureSet.from_proto(get_feature_set_response.feature_set)
def _apply_feature_set(self, feature_set: FeatureSet): self._connect_core() feature_set._client = self valid, message = feature_set.is_valid() if not valid: raise Exception(message) try: apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set.to_proto()), timeout=GRPC_CONNECTION_TIMEOUT_APPLY, ) # type: ApplyFeatureSetResponse applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print( f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}".' ) feature_set._update_from_feature_set(applied_fs, is_dirty=False) return if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected in feature set {feature_set.name}") return except grpc.RpcError as e: print( format_grpc_exception("ApplyFeatureSet", e.code(), e.details()))
def get_feature_set( self, name: str, version: int = None, fail_if_missing: bool = False) -> Union[FeatureSet, None]: """ Retrieve a single feature set from Feast Core :param name: (str) Name of feature set :param version: (int) Version of feature set :param fail_if_missing: (bool) Throws an exception if the feature set is not found :return: Returns a single feature set """ self._connect_core() try: get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest( name=name.strip(), version=str(version))) # type: GetFeatureSetResponse feature_set = get_feature_set_response.feature_set except grpc.RpcError as e: print(format_grpc_exception("GetFeatureSet", e.code(), e.details())) else: if feature_set is not None: return FeatureSet.from_proto(feature_set) if fail_if_missing: raise Exception( f'Could not find feature set with name "{name}" and ' f'version "{version}"')
def _apply_feature_set(self, feature_set: FeatureSet): """ Registers a single feature set with Feast Args: feature_set: Feature set that will be registered """ self._connect_core() feature_set._client = self feature_set.is_valid() # Convert the feature set to a request and send to Feast Core apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set.to_proto()), timeout=GRPC_CONNECTION_TIMEOUT_APPLY, ) # type: ApplyFeatureSetResponse # Extract the returned feature set applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print( f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}"' ) # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected or applied: {feature_set.name}") # Deep copy from the returned feature set to the local feature set feature_set._update_from_feature_set(applied_fs)
def list_feature_sets(self) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Returns: List of feature sets """ self._connect_core() try: # Get latest feature sets from Feast Core feature_set_protos = self._core_service_stub.ListFeatureSets( ListFeatureSetsRequest()) # type: ListFeatureSetsResponse except grpc.RpcError as e: raise Exception( format_grpc_exception("ListFeatureSets", e.code(), e.details())) # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets
def test_list_ingest_jobs(self, mocked_client, mocker): mocker.patch.object( mocked_client, "_core_service_stub", return_value=Core.CoreServiceStub(grpc.insecure_channel("")), ) feature_set_proto = FeatureSetProto( spec=FeatureSetSpecProto( project="test", name="driver", max_age=Duration(seconds=3600), ) ) mocker.patch.object( mocked_client._core_service_stub, "ListIngestionJobs", return_value=ListIngestionJobsResponse( jobs=[ IngestJobProto( id="kafka-to-redis", external_id="job-2222", status=IngestionJobStatus.RUNNING, feature_sets=[feature_set_proto], source=Source( type=SourceType.KAFKA, kafka_source_config=KafkaSourceConfig( bootstrap_servers="localhost:9092", topic="topic" ), ), store=Store(name="redis"), ) ] ), ) # list ingestion jobs by target feature set reference ingest_jobs = mocked_client.list_ingest_jobs( feature_set_ref=FeatureSetRef.from_feature_set( FeatureSet.from_proto(feature_set_proto) ) ) assert len(ingest_jobs) >= 1 ingest_job = ingest_jobs[0] assert ( ingest_job.status == IngestionJobStatus.RUNNING and ingest_job.id == "kafka-to-redis" and ingest_job.external_id == "job-2222" and ingest_job.feature_sets[0].name == "driver" and ingest_job.source.source_type == "Kafka" )
def _apply_feature_set(self, feature_set: FeatureSet): """ Registers a single feature set with Feast Args: feature_set: Feature set that will be registered """ self._connect_core() feature_set.is_valid() feature_set_proto = feature_set.to_proto() if len(feature_set_proto.spec.project) == 0: if self.project is None: raise ValueError( f"No project found in feature set {feature_set.name}. " f"Please set the project within the feature set or within " f"your Feast Client.") else: feature_set_proto.spec.project = self.project # Convert the feature set to a request and send to Feast Core try: apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set_proto), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ApplyFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) # Extract the returned feature set applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print(f'Feature set created: "{applied_fs.name}"') if apply_fs_response.status == ApplyFeatureSetResponse.Status.UPDATED: print(f'Feature set updated: "{applied_fs.name}"') # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected or applied: {feature_set.name}") # Deep copy from the returned feature set to the local feature set feature_set._update_from_feature_set(applied_fs)
def list_feature_sets(self, project: str = None, name: str = None, version: str = None) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Args: project: Filter feature sets based on project name name: Filter feature sets based on feature set name version: Filter feature sets based on version numbf, Returns: List of feature sets """ self._connect_core() if project is None: if self.project is not None: project = self.project else: project = "*" if name is None: name = "*" if version is None: version = "*" filter = ListFeatureSetsRequest.Filter(project=project, feature_set_name=name, feature_set_version=version) # Get latest feature sets from Feast Core feature_set_protos = self._core_service_stub.ListFeatureSets( ListFeatureSetsRequest( filter=filter)) # type: ListFeatureSetsResponse # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets
def list_feature_sets( self, project: str = None, name: str = None, labels: Dict[str, str] = dict()) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Args: project: Filter feature sets based on project name name: Filter feature sets based on feature set name Returns: List of feature sets """ if project is None: if self.project is not None: project = self.project else: project = "*" if name is None: name = "*" filter = ListFeatureSetsRequest.Filter(project=project, feature_set_name=name, labels=labels) # Get latest feature sets from Feast Core feature_set_protos = self._core_service.ListFeatureSets( ListFeatureSetsRequest(filter=filter), metadata=self._get_grpc_metadata(), ) # type: ListFeatureSetsResponse # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets
def _apply_feature_set(self, feature_set: FeatureSet): """ Registers a single feature set with Feast Args: feature_set: Feature set that will be registered """ self._connect_core() feature_set._client = self valid, message = feature_set.is_valid() if not valid: raise Exception(message) try: # Convert the feature set to a request and send to Feast Core apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set.to_proto()), timeout=GRPC_CONNECTION_TIMEOUT_APPLY, ) # type: ApplyFeatureSetResponse # Extract the returned feature set applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print( f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}".' ) # Deep copy from the returned feature set to the local feature set feature_set._update_from_feature_set(applied_fs, is_dirty=False) return # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected in feature set {feature_set.name}") return except grpc.RpcError as e: print( format_grpc_exception("ApplyFeatureSet", e.code(), e.details()))
def list_feature_sets(self) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Returns: List of feature sets """ self._connect_core() # Get latest feature sets from Feast Core feature_set_protos = self._core_service_stub.ListFeatureSets( ListFeatureSetsRequest()) # type: ListFeatureSetsResponse # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets
def get_feature_set(self, name: str, version: int = None) -> Union[FeatureSet, None]: """ Retrieve a single feature set from Feast Core :param name: Name of feature set :param version: Version of feature set (int) :return: Returns a single feature set """ self._connect_core() # TODO: Version should only be integer or unset. Core should handle 'latest' if version is None: version = "latest" elif isinstance(version, int): version = str(version) else: raise ValueError(f"Version {version} is not of type integer.") try: get_feature_set_response = self._core_service_stub.GetFeatureSets( GetFeatureSetsRequest(filter=GetFeatureSetsRequest.Filter( feature_set_name=name.strip(), feature_set_version=version) )) # type: GetFeatureSetsResponse except grpc.RpcError as e: print( format_grpc_exception("GetFeatureSets", e.code(), e.details())) else: num_feature_sets_found = len( list(get_feature_set_response.feature_sets)) if num_feature_sets_found == 0: return None if num_feature_sets_found > 1: raise Exception( f'Found {num_feature_sets_found} feature sets with name "{name}"' f' and version "{version}": {list(get_feature_set_response.feature_sets)}' ) return FeatureSet.from_proto( get_feature_set_response.feature_sets[0])
def get_feature_set( self, name: str, version: int = None, fail_if_missing: bool = False) -> Union[FeatureSet, None]: """ Retrieves a feature set. If no version is specified then the latest version will be returned. Args: name: Name of feature set version: Version of feature set fail_if_missing: Raise an error if feature set is not found Returns: Returns either the specified feature set, or None if not found """ self._connect_core() try: name = name.strip() if version is None: version = 0 get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest( name=name, version=version)) # type: GetFeatureSetResponse feature_set = get_feature_set_response.feature_set except grpc.RpcError as e: print(format_grpc_exception("GetFeatureSet", e.code(), e.details())) else: if feature_set is not None: return FeatureSet.from_proto(feature_set) if fail_if_missing: raise Exception( f'Could not find feature set with name "{name}" and ' f'version "{version}"')
def get_feature_set(self, name: str, version: int = None, project: str = None) -> Union[FeatureSet, None]: """ Retrieves a feature set. If no version is specified then the latest version will be returned. Args: project: Feast project that this feature set belongs to name: Name of feature set version: Version of feature set Returns: Returns either the specified feature set, or raises an exception if none is found """ self._connect_core() if project is None: if self.project is not None: project = self.project else: raise ValueError("No project has been configured.") if version is None: version = 0 try: get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest( project=project, name=name.strip(), version=int(version))) # type: GetFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) return FeatureSet.from_proto(get_feature_set_response.feature_set)
def get_feature_set(self, name: str, version: int = None) -> Union[FeatureSet, None]: """ Retrieves a feature set. If no version is specified then the latest version will be returned. Args: name: Name of feature set version: Version of feature set Returns: Returns either the specified feature set, or raises an exception if none is found """ self._connect_core() if version is None: version = 0 get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest( name=name.strip(), version=int(version))) # type: GetFeatureSetResponse return FeatureSet.from_proto(get_feature_set_response.feature_set)
def feature_sets(self) -> List[FeatureSet]: """ Getter for the IngestJob's feature sets """ # convert featureset protos to native objects return [FeatureSet.from_proto(fs) for fs in self.proto.feature_sets]