class Client: """ Feast Client: Used for creating, managing, and retrieving features. """ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): """ The Feast Client should be initialized with at least one service url Args: core_url: Feast Core URL. Used to manage features serving_url: Feast Serving URL. Used to retrieve features project: Sets the active project. This field is optional. core_secure: Use client-side SSL/TLS for Core gRPC API serving_secure: Use client-side SSL/TLS for Serving gRPC API options: Configuration options to initialize client with **kwargs: Additional keyword arguments that will be used as configuration options along with "options" """ if options is None: options = dict() self._config = Config(options={**options, **kwargs}) self.__core_channel: grpc.Channel = None self.__serving_channel: grpc.Channel = None self._core_service_stub: CoreServiceStub = None self._serving_service_stub: ServingServiceStub = None @property def core_url(self) -> str: """ Retrieve Feast Core URL Returns: Feast Core URL string """ return self._config.get(CONFIG_CORE_URL_KEY) @core_url.setter def core_url(self, value: str): """ Set the Feast Core URL Args: value: Feast Core URL """ self._config.set(CONFIG_CORE_URL_KEY, value) @property def serving_url(self) -> str: """ Retrieve Serving Core URL Returns: Feast Serving URL string """ return self._config.get(CONFIG_SERVING_URL_KEY) @serving_url.setter def serving_url(self, value: str): """ Set the Feast Serving URL Args: value: Feast Serving URL """ self._config.set(CONFIG_SERVING_URL_KEY, value) @property def core_secure(self) -> bool: """ Retrieve Feast Core client-side SSL/TLS setting Returns: Whether client-side SSL/TLS is enabled """ return self._config.getboolean(CONFIG_CORE_SECURE_KEY) @core_secure.setter def core_secure(self, value: bool): """ Set the Feast Core client-side SSL/TLS setting Args: value: True to enable client-side SSL/TLS """ self._config.set(CONFIG_CORE_SECURE_KEY, value) @property def serving_secure(self) -> bool: """ Retrieve Feast Serving client-side SSL/TLS setting Returns: Whether client-side SSL/TLS is enabled """ return self._config.getboolean(CONFIG_SERVING_SECURE_KEY) @serving_secure.setter def serving_secure(self, value: bool): """ Set the Feast Serving client-side SSL/TLS setting Args: value: True to enable client-side SSL/TLS """ self._config.set(CONFIG_SERVING_SECURE_KEY, value) def version(self): """ Returns version information from Feast Core and Feast Serving """ result = {} if self.serving_url: self._connect_serving() serving_version = self._serving_service_stub.GetFeastServingInfo( GetFeastServingInfoRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ).version result["serving"] = { "url": self.serving_url, "version": serving_version } if self.core_url: self._connect_core() core_version = self._core_service_stub.GetFeastCoreVersion( GetFeastCoreVersionRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ).version result["core"] = {"url": self.core_url, "version": core_version} return result def _connect_core(self, skip_if_connected: bool = True): """ Connect to Core API Args: skip_if_connected: Do not attempt to connect if already connected """ if skip_if_connected and self._core_service_stub: return if not self.core_url: raise ValueError("Please set Feast Core URL.") if self.__core_channel is None: if self.core_secure or self.core_url.endswith(":443"): self.__core_channel = grpc.secure_channel( self.core_url, grpc.ssl_channel_credentials()) else: self.__core_channel = grpc.insecure_channel(self.core_url) try: grpc.channel_ready_future( self.__core_channel).result(timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY)) except grpc.FutureTimeoutError: raise ConnectionError( f"Connection timed out while attempting to connect to Feast " f"Core gRPC server {self.core_url} ") else: self._core_service_stub = CoreServiceStub(self.__core_channel) def _connect_serving(self, skip_if_connected=True): """ Connect to Serving API Args: skip_if_connected: Do not attempt to connect if already connected """ if skip_if_connected and self._serving_service_stub: return if not self.serving_url: raise ValueError("Please set Feast Serving URL.") if self.__serving_channel is None: if self.serving_secure or self.serving_url.endswith(":443"): self.__serving_channel = grpc.secure_channel( self.serving_url, grpc.ssl_channel_credentials()) else: self.__serving_channel = grpc.insecure_channel( self.serving_url) try: grpc.channel_ready_future( self.__serving_channel).result(timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY)) except grpc.FutureTimeoutError: raise ConnectionError( f"Connection timed out while attempting to connect to Feast " f"Serving gRPC server {self.serving_url} ") else: self._serving_service_stub = ServingServiceStub( self.__serving_channel) @property def project(self) -> Union[str, None]: """ Retrieve currently active project Returns: Project name """ return self._config.get(CONFIG_PROJECT_KEY) def set_project(self, project: str): """ Set currently active Feast project Args: project: Project to set as active """ self._config.set(CONFIG_PROJECT_KEY, project) def list_projects(self) -> List[str]: """ List all active Feast projects Returns: List of project names """ self._connect_core() response = self._core_service_stub.ListProjects( ListProjectsRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ListProjectsResponse return list(response.projects) def create_project(self, project: str): """ Creates a Feast project Args: project: Name of project """ self._connect_core() self._core_service_stub.CreateProject( CreateProjectRequest(name=project), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: CreateProjectResponse def archive_project(self, project): """ Archives a project. Project will still continue to function for ingestion and retrieval, but will be in a read-only state. It will also not be visible from the Core API for management purposes. Args: project: Name of project to archive """ self._connect_core() self._core_service_stub.ArchiveProject( ArchiveProjectRequest(name=project), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ArchiveProjectResponse if self._project == project: self._project = "" def apply(self, feature_sets: Union[List[FeatureSet], FeatureSet]): """ Idempotently registers feature set(s) with Feast Core. Either a single feature set or a list can be provided. Args: feature_sets: List of feature sets that will be registered """ if not isinstance(feature_sets, list): feature_sets = [feature_sets] for feature_set in feature_sets: if isinstance(feature_set, FeatureSet): self._apply_feature_set(feature_set) continue raise ValueError( f"Could not determine feature set type to apply {feature_set}") def _apply_feature_set(self, feature_set: FeatureSet): """ Registers a single feature set with Feast Args: feature_set: Feature set that will be registered """ self._connect_core() feature_set.is_valid() feature_set_proto = feature_set.to_proto() if len(feature_set_proto.spec.project) == 0: if self.project is None: raise ValueError( f"No project found in feature set {feature_set.name}. " f"Please set the project within the feature set or within " f"your Feast Client.") else: feature_set_proto.spec.project = self.project # Convert the feature set to a request and send to Feast Core try: apply_fs_response = self._core_service_stub.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set_proto), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: ApplyFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) # Extract the returned feature set applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print( f'Feature set updated/created: "{applied_fs.name}:{applied_fs.version}"' ) # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected or applied: {feature_set.name}") # Deep copy from the returned feature set to the local feature set feature_set._update_from_feature_set(applied_fs) def list_feature_sets(self, project: str = None, name: str = None, version: str = None) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Args: project: Filter feature sets based on project name name: Filter feature sets based on feature set name version: Filter feature sets based on version numbf, Returns: List of feature sets """ self._connect_core() if project is None: if self.project is not None: project = self.project else: project = "*" if name is None: name = "*" if version is None: version = "*" filter = ListFeatureSetsRequest.Filter(project=project, feature_set_name=name, feature_set_version=version) # Get latest feature sets from Feast Core feature_set_protos = self._core_service_stub.ListFeatureSets( ListFeatureSetsRequest( filter=filter)) # type: ListFeatureSetsResponse # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets def get_feature_set(self, name: str, version: int = None, project: str = None) -> Union[FeatureSet, None]: """ Retrieves a feature set. If no version is specified then the latest version will be returned. Args: project: Feast project that this feature set belongs to name: Name of feature set version: Version of feature set Returns: Returns either the specified feature set, or raises an exception if none is found """ self._connect_core() if project is None: if self.project is not None: project = self.project else: raise ValueError("No project has been configured.") if version is None: version = 0 try: get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest( project=project, name=name.strip(), version=int(version))) # type: GetFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) return FeatureSet.from_proto(get_feature_set_response.feature_set) def list_entities(self) -> Dict[str, Entity]: """ Returns a dictionary of entities across all feature sets Returns: Dictionary of entities, indexed by name """ entities_dict = OrderedDict() for fs in self.list_feature_sets(): for entity in fs.entities: entities_dict[entity.name] = entity return entities_dict def get_batch_features( self, feature_refs: List[str], entity_rows: Union[pd.DataFrame, str], default_project: str = None, ) -> RetrievalJob: """ Retrieves historical features from a Feast Serving deployment. Args: feature_refs (List[str]): List of feature references that will be returned for each entity. Each feature reference should have the following format "project/feature:version". entity_rows (Union[pd.DataFrame, str]): Pandas dataframe containing entities and a 'datetime' column. Each entity in a feature set must be present as a column in this dataframe. The datetime column must contain timestamps in datetime64 format. default_project: Default project where feature values will be found. Returns: feast.job.RetrievalJob: Returns a retrival job object that can be used to monitor retrieval progress asynchronously, and can be used to materialize the results. Examples: >>> from feast import Client >>> from datetime import datetime >>> >>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566") >>> feature_refs = ["my_project/bookings_7d:1", "booking_14d"] >>> entity_rows = pd.DataFrame( >>> { >>> "datetime": [pd.datetime.now() for _ in range(3)], >>> "customer": [1001, 1002, 1003], >>> } >>> ) >>> feature_retrieval_job = feast_client.get_batch_features( >>> feature_refs, entity_rows, default_project="my_project") >>> df = feature_retrieval_job.to_dataframe() >>> print(df) """ self._connect_serving() feature_references = _build_feature_references( feature_refs=feature_refs, default_project=default_project) # Retrieve serving information to determine store type and # staging location serving_info = self._serving_service_stub.GetFeastServingInfo( GetFeastServingInfoRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: GetFeastServingInfoResponse if serving_info.type != FeastServingType.FEAST_SERVING_TYPE_BATCH: raise Exception( f'You are connected to a store "{self._serving_url}" which ' f"does not support batch retrieval ") if isinstance(entity_rows, pd.DataFrame): # Pandas DataFrame detected # Remove timezone from datetime column if isinstance(entity_rows["datetime"].dtype, pd.core.dtypes.dtypes.DatetimeTZDtype): entity_rows["datetime"] = pd.DatetimeIndex( entity_rows["datetime"]).tz_localize(None) elif isinstance(entity_rows, str): # String based source if not entity_rows.endswith((".avro", "*")): raise Exception( f"Only .avro and wildcard paths are accepted as entity_rows" ) else: raise Exception(f"Only pandas.DataFrame and str types are allowed" f" as entity_rows, but got {type(entity_rows)}.") # Export and upload entity row DataFrame to staging location # provided by Feast staged_files = export_source_to_staging_location( entity_rows, serving_info.job_staging_location) # type: List[str] request = GetBatchFeaturesRequest( features=feature_references, dataset_source=DatasetSource(file_source=DatasetSource.FileSource( file_uris=staged_files, data_format=DataFormat.DATA_FORMAT_AVRO)), ) # Retrieve Feast Job object to manage life cycle of retrieval response = self._serving_service_stub.GetBatchFeatures(request) return RetrievalJob(response.job, self._serving_service_stub) def get_online_features( self, feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], default_project: Optional[str] = None, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references in the following format [project]/[feature_name]:[version]. Only the feature name is a required component in the reference. example: ["my_project/my_feature_1:3", "my_project3/my_feature_4:1",] entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature default_project: This project will be used if the project name is not provided in the feature reference Returns: Returns a list of maps where each item in the list contains the latest feature values for the provided entities """ self._connect_serving() return self._serving_service_stub.GetOnlineFeatures( GetOnlineFeaturesRequest( features=_build_feature_references( feature_refs=feature_refs, default_project=(default_project if not self.project else self.project), ), entity_rows=entity_rows, )) def list_ingest_jobs( self, job_id: str = None, feature_set_ref: FeatureSetRef = None, store_name: str = None, ): """ List the ingestion jobs currently registered in Feast, with optional filters. Provides detailed metadata about each ingestion job. Args: job_id: Select specific ingestion job with the given job_id feature_set_ref: Filter ingestion jobs by target feature set (via reference) store_name: Filter ingestion jobs by target feast store's name Returns: List of IngestJobs matching the given filters """ self._connect_core() # construct list request feature_set_ref = None list_filter = ListIngestionJobsRequest.Filter( id=job_id, feature_set_reference=feature_set_ref, store_name=store_name, ) request = ListIngestionJobsRequest(filter=list_filter) # make list request & unpack response response = self._core_service_stub.ListIngestionJobs(request) ingest_jobs = [ IngestJob(proto, self._core_service_stub) for proto in response.jobs ] return ingest_jobs def restart_ingest_job(self, job: IngestJob): """ Restart ingestion job currently registered in Feast. NOTE: Data might be lost during the restart for some job runners. Does not support stopping a job in a transitional (ie pending, suspending, aborting), terminal state (ie suspended or aborted) or unknown status Args: job: IngestJob to restart """ self._connect_core() request = RestartIngestionJobRequest(id=job.id) try: self._core_service_stub.RestartIngestionJob(request) except grpc.RpcError as e: raise grpc.RpcError(e.details()) def stop_ingest_job(self, job: IngestJob): """ Stop ingestion job currently resgistered in Feast Does nothing if the target job if already in a terminal state (ie suspended or aborted). Does not support stopping a job in a transitional (ie pending, suspending, aborting) or in a unknown status Args: job: IngestJob to restart """ self._connect_core() request = StopIngestionJobRequest(id=job.id) try: self._core_service_stub.StopIngestionJob(request) except grpc.RpcError as e: raise grpc.RpcError(e.details()) def ingest( self, feature_set: Union[str, FeatureSet], source: Union[pd.DataFrame, str], chunk_size: int = 10000, version: int = None, force_update: bool = False, max_workers: int = max(CPU_COUNT - 1, 1), disable_progress_bar: bool = False, timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT, ) -> None: """ Loads feature data into Feast for a specific feature set. Args: feature_set (typing.Union[str, feast.feature_set.FeatureSet]): Feature set object or the string name of the feature set (without a version). source (typing.Union[pd.DataFrame, str]): Either a file path or Pandas Dataframe to ingest into Feast Files that are currently supported: * parquet * csv * json chunk_size (int): Amount of rows to load and ingest at a time. version (int): Feature set version. force_update (bool): Automatically update feature set based on source data prior to ingesting. This will also register changes to Feast. max_workers (int): Number of worker processes to use to encode values. disable_progress_bar (bool): Disable printing of progress statistics. timeout (int): Timeout in seconds to wait for completion. Returns: None: None """ if isinstance(feature_set, FeatureSet): name = feature_set.name if version is None: version = feature_set.version elif isinstance(feature_set, str): name = feature_set else: raise Exception(f"Feature set name must be provided") # Read table and get row count dir_path, dest_path = _read_table_from_source(source, chunk_size, max_workers) pq_file = pq.ParquetFile(dest_path) row_count = pq_file.metadata.num_rows # Update the feature set based on PyArrow table of first row group if force_update: feature_set.infer_fields_from_pa( table=pq_file.read_row_group(0), discard_unused_fields=True, replace_existing_features=True, ) self.apply(feature_set) current_time = time.time() print("Waiting for feature set to be ready for ingestion...") while True: if timeout is not None and time.time() - current_time >= timeout: raise TimeoutError( "Timed out waiting for feature set to be ready") feature_set = self.get_feature_set(name, version) if (feature_set is not None and feature_set.status == FeatureSetStatus.STATUS_READY): break time.sleep(3) if timeout is not None: timeout = timeout - int(time.time() - current_time) try: # Kafka configs brokers = feature_set.get_kafka_source_brokers() topic = feature_set.get_kafka_source_topic() producer = get_producer(brokers, row_count, disable_progress_bar) # Loop optimization declarations produce = producer.produce flush = producer.flush # Transform and push data to Kafka if feature_set.source.source_type == "Kafka": for chunk in get_feature_row_chunks( file=dest_path, row_groups=list(range(pq_file.num_row_groups)), fs=feature_set, max_workers=max_workers, ): # Push FeatureRow one chunk at a time to kafka for serialized_row in chunk: produce(topic=topic, value=serialized_row) # Force a flush after each chunk flush(timeout=timeout) # Remove chunk from memory del chunk else: raise Exception( f"Could not determine source type for feature set " f'"{feature_set.name}" with source type ' f'"{feature_set.source.source_type}"') # Print ingestion statistics producer.print_results() finally: # Remove parquet file(s) that were created earlier print("Removing temporary file(s)...") shutil.rmtree(dir_path) return None
class Client: """ Feast Client: Used for creating, managing, and retrieving features. """ def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs): """ The Feast Client should be initialized with at least one service url Please see constants.py for configuration options. Commonly used options or arguments include: core_url: Feast Core URL. Used to manage features serving_url: Feast Serving URL. Used to retrieve features project: Sets the active project. This field is optional. core_secure: Use client-side SSL/TLS for Core gRPC API serving_secure: Use client-side SSL/TLS for Serving gRPC API core_enable_auth: Enable authentication and authorization core_auth_provider: Authentication provider – "google" or "oauth" if core_auth_provider is "oauth", the following fields are mandatory – oauth_grant_type, oauth_client_id, oauth_client_secret, oauth_audience, oauth_token_request_url Args: options: Configuration options to initialize client with **kwargs: Additional keyword arguments that will be used as configuration options along with "options" """ if options is None: options = dict() self._config = Config(options={**options, **kwargs}) self._core_service_stub: CoreServiceStub = None self._serving_service_stub: ServingServiceStub = None self._auth_metadata = None # Configure Auth Metadata Plugin if auth is enabled if self._config.getboolean(CONFIG_CORE_ENABLE_AUTH_KEY): self._auth_metadata = feast_auth.get_auth_metadata_plugin( self._config) @property def _core_service(self): """ Creates or returns the gRPC Feast Core Service Stub Returns: CoreServiceStub """ if not self._core_service_stub: channel = create_grpc_channel( url=self._config.get(CONFIG_CORE_URL_KEY), enable_ssl=self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY), enable_auth=self._config.getboolean( CONFIG_CORE_ENABLE_AUTH_KEY), ssl_server_cert_path=self._config.get( CONFIG_CORE_SERVER_SSL_CERT_KEY), auth_metadata_plugin=self._auth_metadata, timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) self._core_service_stub = CoreServiceStub(channel) return self._core_service_stub @property def _serving_service(self): """ Creates or returns the gRPC Feast Serving Service Stub Returns: ServingServiceStub """ if not self._serving_service_stub: channel = create_grpc_channel( url=self._config.get(CONFIG_SERVING_URL_KEY), enable_ssl=self._config.getboolean( CONFIG_SERVING_ENABLE_SSL_KEY), enable_auth=False, ssl_server_cert_path=self._config.get( CONFIG_SERVING_SERVER_SSL_CERT_KEY), auth_metadata_plugin=None, timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) self._serving_service_stub = ServingServiceStub(channel) return self._serving_service_stub @property def core_url(self) -> str: """ Retrieve Feast Core URL Returns: Feast Core URL string """ return self._config.get(CONFIG_CORE_URL_KEY) @core_url.setter def core_url(self, value: str): """ Set the Feast Core URL Args: value: Feast Core URL """ self._config.set(CONFIG_CORE_URL_KEY, value) @property def serving_url(self) -> str: """ Retrieve Serving Core URL Returns: Feast Serving URL string """ return self._config.get(CONFIG_SERVING_URL_KEY) @serving_url.setter def serving_url(self, value: str): """ Set the Feast Serving URL Args: value: Feast Serving URL """ self._config.set(CONFIG_SERVING_URL_KEY, value) @property def core_secure(self) -> bool: """ Retrieve Feast Core client-side SSL/TLS setting Returns: Whether client-side SSL/TLS is enabled """ return self._config.getboolean(CONFIG_CORE_ENABLE_SSL_KEY) @core_secure.setter def core_secure(self, value: bool): """ Set the Feast Core client-side SSL/TLS setting Args: value: True to enable client-side SSL/TLS """ self._config.set(CONFIG_CORE_ENABLE_SSL_KEY, value) @property def serving_secure(self) -> bool: """ Retrieve Feast Serving client-side SSL/TLS setting Returns: Whether client-side SSL/TLS is enabled """ return self._config.getboolean(CONFIG_SERVING_ENABLE_SSL_KEY) @serving_secure.setter def serving_secure(self, value: bool): """ Set the Feast Serving client-side SSL/TLS setting Args: value: True to enable client-side SSL/TLS """ self._config.set(CONFIG_SERVING_ENABLE_SSL_KEY, value) def version(self): """ Returns version information from Feast Core and Feast Serving """ import pkg_resources result = { "sdk": { "version": pkg_resources.get_distribution("feast").version }, "serving": "not configured", "core": "not configured", } if self.serving_url: serving_version = self._serving_service.GetFeastServingInfo( GetFeastServingInfoRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ).version result["serving"] = { "url": self.serving_url, "version": serving_version } if self.core_url: core_version = self._core_service.GetFeastCoreVersion( GetFeastCoreVersionRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), metadata=self._get_grpc_metadata(), ).version result["core"] = {"url": self.core_url, "version": core_version} return result @property def project(self) -> Union[str, None]: """ Retrieve currently active project Returns: Project name """ return self._config.get(CONFIG_PROJECT_KEY) def set_project(self, project: Optional[str] = None): """ Set currently active Feast project Args: project: Project to set as active. If unset, will reset to the default project. """ if project is None: project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] self._config.set(CONFIG_PROJECT_KEY, project) def list_projects(self) -> List[str]: """ List all active Feast projects Returns: List of project names """ response = self._core_service.ListProjects( ListProjectsRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), metadata=self._get_grpc_metadata(), ) # type: ListProjectsResponse return list(response.projects) def create_project(self, project: str): """ Creates a Feast project Args: project: Name of project """ self._core_service.CreateProject( CreateProjectRequest(name=project), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), metadata=self._get_grpc_metadata(), ) # type: CreateProjectResponse def archive_project(self, project): """ Archives a project. Project will still continue to function for ingestion and retrieval, but will be in a read-only state. It will also not be visible from the Core API for management purposes. Args: project: Name of project to archive """ try: self._core_service_stub.ArchiveProject( ArchiveProjectRequest(name=project), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), metadata=self._get_grpc_metadata(), ) # type: ArchiveProjectResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) # revert to the default project if self._project == project: self._project = FEAST_DEFAULT_OPTIONS[CONFIG_PROJECT_KEY] def apply(self, feature_sets: Union[List[FeatureSet], FeatureSet]): """ Idempotently registers feature set(s) with Feast Core. Either a single feature set or a list can be provided. Args: feature_sets: List of feature sets that will be registered """ if not isinstance(feature_sets, list): feature_sets = [feature_sets] for feature_set in feature_sets: if isinstance(feature_set, FeatureSet): self._apply_feature_set(feature_set) continue raise ValueError( f"Could not determine feature set type to apply {feature_set}") def _apply_feature_set(self, feature_set: FeatureSet): """ Registers a single feature set with Feast Args: feature_set: Feature set that will be registered """ feature_set.is_valid() feature_set_proto = feature_set.to_proto() if len(feature_set_proto.spec.project) == 0: if self.project is not None: feature_set_proto.spec.project = self.project # Convert the feature set to a request and send to Feast Core try: apply_fs_response = self._core_service.ApplyFeatureSet( ApplyFeatureSetRequest(feature_set=feature_set_proto), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), metadata=self._get_grpc_metadata(), ) # type: ApplyFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) # Extract the returned feature set applied_fs = FeatureSet.from_proto(apply_fs_response.feature_set) # If the feature set has changed, update the local copy if apply_fs_response.status == ApplyFeatureSetResponse.Status.CREATED: print(f'Feature set created: "{applied_fs.name}"') if apply_fs_response.status == ApplyFeatureSetResponse.Status.UPDATED: print(f'Feature set updated: "{applied_fs.name}"') # If no change has been applied, do nothing if apply_fs_response.status == ApplyFeatureSetResponse.Status.NO_CHANGE: print(f"No change detected or applied: {feature_set.name}") # Deep copy from the returned feature set to the local feature set feature_set._update_from_feature_set(applied_fs) def list_feature_sets( self, project: str = None, name: str = None, labels: Dict[str, str] = dict()) -> List[FeatureSet]: """ Retrieve a list of feature sets from Feast Core Args: project: Filter feature sets based on project name name: Filter feature sets based on feature set name Returns: List of feature sets """ if project is None: if self.project is not None: project = self.project else: project = "*" if name is None: name = "*" filter = ListFeatureSetsRequest.Filter(project=project, feature_set_name=name, labels=labels) # Get latest feature sets from Feast Core feature_set_protos = self._core_service.ListFeatureSets( ListFeatureSetsRequest(filter=filter), metadata=self._get_grpc_metadata(), ) # type: ListFeatureSetsResponse # Extract feature sets and return feature_sets = [] for feature_set_proto in feature_set_protos.feature_sets: feature_set = FeatureSet.from_proto(feature_set_proto) feature_set._client = self feature_sets.append(feature_set) return feature_sets def get_feature_set(self, name: str, project: str = None) -> Union[FeatureSet, None]: """ Retrieves a feature set. Args: project: Feast project that this feature set belongs to name: Name of feature set Returns: Returns either the specified feature set, or raises an exception if none is found """ if project is None: if self.project is not None: project = self.project else: raise ValueError("No project has been configured.") try: get_feature_set_response = self._core_service_stub.GetFeatureSet( GetFeatureSetRequest(project=project, name=name.strip()), metadata=self._get_grpc_metadata(), ) # type: GetFeatureSetResponse except grpc.RpcError as e: raise grpc.RpcError(e.details()) return FeatureSet.from_proto(get_feature_set_response.feature_set) def list_features_by_ref( self, project: str = None, entities: List[str] = list(), labels: Dict[str, str] = dict(), ) -> Dict[FeatureRef, Feature]: """ Returns a list of features based on filters provided. Args: project: Feast project that these features belongs to entities: Feast entity that these features are associated with labels: Feast labels that these features are associated with Returns: Dictionary of <feature references: features> Examples: >>> from feast import Client >>> >>> feast_client = Client(core_url="localhost:6565") >>> features = list_features_by_ref(project="test_project", entities=["driver_id"], labels={"key1":"val1","key2":"val2"}) >>> print(features) """ if project is None: if self.project is not None: project = self.project else: project = "default" filter = ListFeaturesRequest.Filter(project=project, entities=entities, labels=labels) feature_protos = self._core_service.ListFeatures( ListFeaturesRequest(filter=filter)) # type: ListFeaturesResponse features_dict = {} for ref_str, feature_proto in feature_protos.features.items(): feature_ref = FeatureRef.from_str(ref_str, ignore_project=True) feature = Feature.from_proto(feature_proto) features_dict[feature_ref] = feature return features_dict def list_entities(self) -> Dict[str, Entity]: """ Returns a dictionary of entities across all feature sets Returns: Dictionary of entities, indexed by name """ entities_dict = OrderedDict() for fs in self.list_feature_sets(): for entity in fs.entities: entities_dict[entity.name] = entity return entities_dict def get_batch_features( self, feature_refs: List[str], entity_rows: Union[pd.DataFrame, str], compute_statistics: bool = False, project: str = None, ) -> RetrievalJob: """ Retrieves historical features from a Feast Serving deployment. Args: feature_refs: List of feature references that will be returned for each entity. Each feature reference should have the following format: "feature_set:feature" where "feature_set" & "feature" refer to the feature and feature set names respectively. Only the feature name is required. entity_rows (Union[pd.DataFrame, str]): Pandas dataframe containing entities and a 'datetime' column. Each entity in a feature set must be present as a column in this dataframe. The datetime column must contain timestamps in datetime64 format. compute_statistics (bool): Indicates whether Feast should compute statistics over the retrieved dataset. project: Specifies the project which contain the FeatureSets which the requested features belong to. Returns: feast.job.RetrievalJob: Returns a retrival job object that can be used to monitor retrieval progress asynchronously, and can be used to materialize the results. Examples: >>> from feast import Client >>> from datetime import datetime >>> >>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566") >>> feature_refs = ["my_project/bookings_7d", "booking_14d"] >>> entity_rows = pd.DataFrame( >>> { >>> "datetime": [pd.datetime.now() for _ in range(3)], >>> "customer": [1001, 1002, 1003], >>> } >>> ) >>> feature_retrieval_job = feast_client.get_batch_features( >>> feature_refs, entity_rows, project="my_project") >>> df = feature_retrieval_job.to_dataframe() >>> print(df) """ # Retrieve serving information to determine store type and # staging location serving_info = self._serving_service.GetFeastServingInfo( GetFeastServingInfoRequest(), timeout=self._config.getint( CONFIG_GRPC_CONNECTION_TIMEOUT_DEFAULT_KEY), ) # type: GetFeastServingInfoResponse if serving_info.type != FeastServingType.FEAST_SERVING_TYPE_BATCH: raise Exception( f'You are connected to a store "{self.serving_url}" which ' f"does not support batch retrieval ") if isinstance(entity_rows, pd.DataFrame): # Pandas DataFrame detected # Remove timezone from datetime column if isinstance(entity_rows["datetime"].dtype, pd.core.dtypes.dtypes.DatetimeTZDtype): entity_rows["datetime"] = pd.DatetimeIndex( entity_rows["datetime"]).tz_localize(None) elif isinstance(entity_rows, str): # String based source if not entity_rows.endswith((".avro", "*")): raise Exception( "Only .avro and wildcard paths are accepted as entity_rows" ) else: raise Exception(f"Only pandas.DataFrame and str types are allowed" f" as entity_rows, but got {type(entity_rows)}.") # Export and upload entity row DataFrame to staging location # provided by Feast staged_files = export_source_to_staging_location( entity_rows, serving_info.job_staging_location) # type: List[str] request = GetBatchFeaturesRequest( features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, ), dataset_source=DatasetSource(file_source=DatasetSource.FileSource( file_uris=staged_files, data_format=DataFormat.DATA_FORMAT_AVRO)), compute_statistics=compute_statistics, ) # Retrieve Feast Job object to manage life cycle of retrieval try: response = self._serving_service.GetBatchFeatures(request) except grpc.RpcError as e: raise grpc.RpcError(e.details()) return RetrievalJob(response.job, self._serving_service) def get_online_features( self, feature_refs: List[str], entity_rows: List[GetOnlineFeaturesRequest.EntityRow], project: Optional[str] = None, omit_entities: bool = False, ) -> GetOnlineFeaturesResponse: """ Retrieves the latest online feature data from Feast Serving Args: feature_refs: List of feature references that will be returned for each entity. Each feature reference should have the following format: "feature_set:feature" where "feature_set" & "feature" refer to the feature and feature set names respectively. Only the feature name is required. entity_rows: List of GetFeaturesRequest.EntityRow where each row contains entities. Timestamp should not be set for online retrieval. All entity types within a feature project: Specifies the project which contain the FeatureSets which the requested features belong to. omit_entities: If true will omit entity values in the returned feature data. Returns: GetOnlineFeaturesResponse containing the feature data in records. Each EntityRow provided will yield one record, which contains data fields with data value and field status metadata (if included). """ try: response = self._serving_service.GetOnlineFeatures( GetOnlineFeaturesRequest( omit_entities_in_response=omit_entities, features=_build_feature_references( feature_ref_strs=feature_refs, project=project if project is not None else self.project, ), entity_rows=entity_rows, )) entity_refs = { key for entity_row in entity_rows for key in entity_row.fields.keys() } # strip the project part the string feature references returned from serving strip = ( lambda ref: repr(FeatureRef.from_str(ref, ignore_project=True)) if ref not in entity_refs else ref) strip_field_values = [] for field_value in response.field_values: keys, fields, statuses = ( field_value.fields.keys(), field_value.fields, field_value.statuses, ) fields_and_statuses = [(strip(key), fields[key], statuses[key]) for key in keys] keys, fields, statuses = zip(*fields_and_statuses) strip_field_values.append( GetOnlineFeaturesResponse.FieldValues( fields=dict(zip(keys, fields)), statuses=dict(zip(keys, statuses)), )) del response.field_values[:] response.field_values.extend(strip_field_values) except grpc.RpcError as e: raise grpc.RpcError(e.details()) return response def list_ingest_jobs( self, job_id: str = None, feature_set_ref: FeatureSetRef = None, store_name: str = None, ): """ List the ingestion jobs currently registered in Feast, with optional filters. Provides detailed metadata about each ingestion job. Args: job_id: Select specific ingestion job with the given job_id feature_set_ref: Filter ingestion jobs by target feature set (via reference) store_name: Filter ingestion jobs by target feast store's name Returns: List of IngestJobs matching the given filters """ # construct list request feature_set_ref = None list_filter = ListIngestionJobsRequest.Filter( id=job_id, feature_set_reference=feature_set_ref, store_name=store_name, ) request = ListIngestionJobsRequest(filter=list_filter) # make list request & unpack response response = self._core_service_stub.ListIngestionJobs(request) ingest_jobs = [ IngestJob(proto, self._core_service_stub) for proto in response.jobs ] return ingest_jobs def restart_ingest_job(self, job: IngestJob): """ Restart ingestion job currently registered in Feast. NOTE: Data might be lost during the restart for some job runners. Does not support stopping a job in a transitional (ie pending, suspending, aborting), terminal state (ie suspended or aborted) or unknown status Args: job: IngestJob to restart """ request = RestartIngestionJobRequest(id=job.id) try: self._core_service_stub.RestartIngestionJob(request) except grpc.RpcError as e: raise grpc.RpcError(e.details()) def stop_ingest_job(self, job: IngestJob): """ Stop ingestion job currently resgistered in Feast Does nothing if the target job if already in a terminal state (ie suspended or aborted). Does not support stopping a job in a transitional (ie pending, suspending, aborting) or in a unknown status Args: job: IngestJob to restart """ request = StopIngestionJobRequest(id=job.id) try: self._core_service_stub.StopIngestionJob(request) except grpc.RpcError as e: raise grpc.RpcError(e.details()) def ingest( self, feature_set: Union[str, FeatureSet], source: Union[pd.DataFrame, str], chunk_size: int = 10000, max_workers: int = max(CPU_COUNT - 1, 1), disable_progress_bar: bool = False, timeout: int = KAFKA_CHUNK_PRODUCTION_TIMEOUT, ) -> str: """ Loads feature data into Feast for a specific feature set. Args: feature_set (typing.Union[str, feast.feature_set.FeatureSet]): Feature set object or the string name of the feature set source (typing.Union[pd.DataFrame, str]): Either a file path or Pandas Dataframe to ingest into Feast Files that are currently supported: * parquet * csv * json chunk_size (int): Amount of rows to load and ingest at a time. max_workers (int): Number of worker processes to use to encode values. disable_progress_bar (bool): Disable printing of progress statistics. timeout (int): Timeout in seconds to wait for completion. Returns: str: ingestion id for this dataset """ if isinstance(feature_set, FeatureSet): name = feature_set.name elif isinstance(feature_set, str): name = feature_set else: raise Exception("Feature set name must be provided") # Read table and get row count dir_path, dest_path = _read_table_from_source(source, chunk_size, max_workers) pq_file = pq.ParquetFile(dest_path) row_count = pq_file.metadata.num_rows current_time = time.time() print("Waiting for feature set to be ready for ingestion...") while True: if timeout is not None and time.time() - current_time >= timeout: raise TimeoutError( "Timed out waiting for feature set to be ready") feature_set = self.get_feature_set(name) if (feature_set is not None and feature_set.status == FeatureSetStatus.STATUS_READY): break time.sleep(3) if timeout is not None: timeout = timeout - int(time.time() - current_time) try: # Kafka configs brokers = feature_set.get_kafka_source_brokers() topic = feature_set.get_kafka_source_topic() producer = get_producer(brokers, row_count, disable_progress_bar) # Loop optimization declarations produce = producer.produce flush = producer.flush ingestion_id = _generate_ingestion_id(feature_set) # Transform and push data to Kafka if feature_set.source.source_type == "Kafka": for chunk in get_feature_row_chunks( file=dest_path, row_groups=list(range(pq_file.num_row_groups)), fs=feature_set, ingestion_id=ingestion_id, max_workers=max_workers, ): # Push FeatureRow one chunk at a time to kafka for serialized_row in chunk: produce(topic=topic, value=serialized_row) # Force a flush after each chunk flush(timeout=timeout) # Remove chunk from memory del chunk else: raise Exception( f"Could not determine source type for feature set " f'"{feature_set.name}" with source type ' f'"{feature_set.source.source_type}"') # Print ingestion statistics producer.print_results() finally: # Remove parquet file(s) that were created earlier print("Removing temporary file(s)...") shutil.rmtree(dir_path) return ingestion_id def get_statistics( self, feature_set_id: str, store: str, features: List[str] = [], ingestion_ids: Optional[List[str]] = None, start_date: Optional[datetime.datetime] = None, end_date: Optional[datetime.datetime] = None, force_refresh: bool = False, default_project: Optional[str] = None, ) -> statistics_pb2.DatasetFeatureStatisticsList: """ Retrieves the feature featureStatistics computed over the data in the batch stores. Args: feature_set_id: Fully qualified feature set id in the format project/feature_set to retrieve batch featureStatistics for. If project is not provided, the default ("default") will be used. store: Name of the store to retrieve feature featureStatistics over. This store must be a historical store. features: Optional list of feature names to filter from the results. ingestion_ids: Optional list of dataset Ids by which to filter data before retrieving featureStatistics. Cannot be used with start_date and end_date. If multiple dataset ids are provided, unaggregatable featureStatistics will be dropped. start_date: Optional start date over which to filter statistical data. Data from this date will be included. Cannot be used with dataset_ids. If the provided period spans multiple days, unaggregatable featureStatistics will be dropped. end_date: Optional end date over which to filter statistical data. Data from this data will not be included. Cannot be used with dataset_ids. If the provided period spans multiple days, unaggregatable featureStatistics will be dropped. force_refresh: Setting this flag to true will force a recalculation of featureStatistics and overwrite results currently in the cache, if any. default_project: Manual override for default project. Returns: Returns a tensorflow DatasetFeatureStatisticsList containing TFDV featureStatistics. """ if ingestion_ids is not None and (start_date is not None or end_date is not None): raise ValueError( "Only one of dataset_id or [start_date, end_date] can be provided." ) if default_project != "" and "/" not in feature_set_id: feature_set_id = f"{default_project}/{feature_set_id}" request = GetFeatureStatisticsRequest( feature_set_id=feature_set_id, features=features, store=store, force_refresh=force_refresh, ) if ingestion_ids is not None: request.ingestion_ids.extend(ingestion_ids) else: if start_date is not None: request.start_date.CopyFrom( Timestamp(seconds=int(start_date.timestamp()))) if end_date is not None: request.end_date.CopyFrom( Timestamp(seconds=int(end_date.timestamp()))) return self._core_service_stub.GetFeatureStatistics( request).dataset_feature_statistics_list def _get_grpc_metadata(self): """ Returns a metadata tuple to attach to gRPC requests. This is primarily used when authentication is enabled but SSL/TLS is disabled. Returns: Tuple of metadata to attach to each gRPC call """ if self._config.getboolean( CONFIG_CORE_ENABLE_AUTH_KEY) and self._auth_metadata: return self._auth_metadata.get_signed_meta() return ()