def test_exists(self):
"""
Test type casting of strings to other types
"""
fd, path = mkstemp()
config = Config(path=path)
config.set("my_val_exist", 1)
assert config.exists("my_val_exist") is True
assert config.exists("my_val_not_exist") is False
def _source_to_argument(source: DataSource, config: Config):
common_properties = {
"field_mapping": dict(source.field_mapping),
"event_timestamp_column": source.event_timestamp_column,
"created_timestamp_column": source.created_timestamp_column,
"date_partition_column": source.date_partition_column,
}
properties = {**common_properties}
if isinstance(source, FileSource):
properties["path"] = source.file_options.file_url
properties["format"] = dict(
json_class=source.file_options.file_format.__class__.__name__)
return {"file": properties}
if isinstance(source, BigQuerySource):
project, dataset_and_table = source.bigquery_options.table_ref.split(
":")
dataset, table = dataset_and_table.split(".")
properties["project"] = project
properties["dataset"] = dataset
properties["table"] = table
if config.exists(
opt.SPARK_BQ_MATERIALIZATION_PROJECT) and config.exists(
opt.SPARK_BQ_MATERIALIZATION_DATASET):
properties["materialization"] = dict(
project=config.get(opt.SPARK_BQ_MATERIALIZATION_PROJECT),
dataset=config.get(opt.SPARK_BQ_MATERIALIZATION_DATASET),
)
return {"bq": properties}
if isinstance(source, KafkaSource):
properties[
"bootstrap_servers"] = source.kafka_options.bootstrap_servers
properties["topic"] = source.kafka_options.topic
properties["format"] = {
**source.kafka_options.message_format.__dict__,
"json_class":
source.kafka_options.message_format.__class__.__name__,
}
return {"kafka": properties}
raise NotImplementedError(f"Unsupported Datasource: {type(source)}")
def __init__(self, config: Config):
"""
Initializes an OAuthMetadataPlugin, used to sign gRPC requests
Args:
config: Feast Configuration object
"""
super(OAuthMetadataPlugin, self).__init__()
self._static_token = None
self._token = None
# If provided, set a static token
if config.exists(CONFIG_CORE_ENABLE_AUTH_TOKEN_KEY):
self._static_token = config.get(CONFIG_CORE_ENABLE_AUTH_TOKEN_KEY)
self._refresh_token(config)
elif (config.exists(CONFIG_OAUTH_GRANT_TYPE_KEY)
and config.exists(CONFIG_OAUTH_CLIENT_ID_KEY)
and config.exists(CONFIG_OAUTH_CLIENT_SECRET_KEY)
and config.exists(CONFIG_OAUTH_AUDIENCE_KEY)
and config.exists(CONFIG_OAUTH_TOKEN_REQUEST_URL_KEY)):
self._refresh_token(config)
else:
raise RuntimeError(
" Please ensure that the "
"necessary parameters are passed to the client - "
"oauth_grant_type, oauth_client_id, oauth_client_secret, "
"oauth_audience, oauth_token_request_url.")
def __init__(self, config: Config):
"""
Initializes a GoogleOpenIDAuthMetadataPlugin, used to sign gRPC requests
Args:
config: Feast Configuration object
"""
super(GoogleOpenIDAuthMetadataPlugin, self).__init__()
self._static_token = None
self._token = None
# If provided, set a static token
if config.exists(opt.AUTH_TOKEN):
self._static_token = config.get(opt.AUTH_TOKEN)
self._request = RequestWithTimeout(timeout=5)
self._refresh_token()
def __init__(self, config: Config):
"""
Initializes a GoogleOpenIDAuthMetadataPlugin, used to sign gRPC requests
Args:
config: Feast Configuration object
"""
super(GoogleOpenIDAuthMetadataPlugin, self).__init__()
from google.auth.transport import requests
self._static_token = None
self._token = None
# If provided, set a static token
if config.exists(CONFIG_CORE_ENABLE_AUTH_TOKEN_KEY):
self._static_token = config.get(CONFIG_CORE_ENABLE_AUTH_TOKEN_KEY)
self._request = requests.Request()
self._refresh_token()
class Client:
"""
Feast Client: Used for creating, managing, and retrieving features.
"""
def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs):
"""
The Feast Client should be initialized with at least one service url
Please see constants.py for configuration options. Commonly used options
or arguments include:
core_url: Feast Core URL. Used to manage features
serving_url: Feast Serving URL. Used to retrieve features
project: Sets the active project. This field is optional.
core_secure: Use client-side SSL/TLS for Core gRPC API
serving_secure: Use client-side SSL/TLS for Serving gRPC API
enable_auth: Enable authentication and authorization
auth_provider: Authentication provider – "google" or "oauth"
if auth_provider is "oauth", the following fields are mandatory –
oauth_grant_type, oauth_client_id, oauth_client_secret, oauth_audience, oauth_token_request_url
Args:
options: Configuration options to initialize client with
**kwargs: Additional keyword arguments that will be used as
configuration options along with "options"
"""
if options is None:
options = dict()
self._config = Config(options={**options, **kwargs})
self._core_service_stub: Optional[CoreServiceStub] = None
self._serving_service_stub: Optional[ServingServiceStub] = None
self._job_service_stub: Optional[JobServiceStub] = None
self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None
# Configure Auth Metadata Plugin if auth is enabled
if self._config.getboolean(opt.ENABLE_AUTH):
self._auth_metadata = feast_auth.get_auth_metadata_plugin(
self._config)
@property
def _core_service(self):
"""
Creates or returns the gRPC Feast Core Service Stub
Returns: CoreServiceStub
"""
if not self._core_service_stub:
channel = create_grpc_channel(
url=self._config.get(opt.CORE_URL),
enable_ssl=self._config.getboolean(opt.CORE_ENABLE_SSL),
enable_auth=self._config.getboolean(opt.ENABLE_AUTH),
ssl_server_cert_path=self._config.get(
opt.CORE_SERVER_SSL_CERT),
auth_metadata_plugin=self._auth_metadata,
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
)
self._core_service_stub = CoreServiceStub(channel)
return self._core_service_stub
@property
def _serving_service(self):
"""
Creates or returns the gRPC Feast Serving Service Stub. If both `opentracing`
and `grpcio-opentracing` are installed, an opentracing interceptor will be
instantiated based on the global tracer.
Returns: ServingServiceStub
"""
if not self._serving_service_stub:
channel = create_grpc_channel(
url=self._config.get(opt.SERVING_URL),
enable_ssl=self._config.getboolean(opt.SERVING_ENABLE_SSL),
enable_auth=self._config.getboolean(opt.ENABLE_AUTH),
ssl_server_cert_path=self._config.get(
opt.SERVING_SERVER_SSL_CERT),
auth_metadata_plugin=self._auth_metadata,
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
)
try:
import opentracing
from grpc_opentracing import open_tracing_client_interceptor
from grpc_opentracing.grpcext import intercept_channel
interceptor = open_tracing_client_interceptor(
opentracing.global_tracer())
channel = intercept_channel(channel, interceptor)
except ImportError:
pass
self._serving_service_stub = ServingServiceStub(channel)
return self._serving_service_stub
@property
def _use_job_service(self) -> bool:
return self._config.exists(opt.JOB_SERVICE_URL)
@property
def _job_service(self):
"""
Creates or returns the gRPC Feast Job Service Stub
Returns: JobServiceStub
"""
# Don't try to initialize job service stub if the job service is disabled
if not self._use_job_service:
return None
if not self._job_service_stub:
channel = create_grpc_channel(
url=self._config.get(opt.JOB_SERVICE_URL),
enable_ssl=self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL),
enable_auth=self._config.getboolean(opt.ENABLE_AUTH),
ssl_server_cert_path=self._config.get(
opt.JOB_SERVICE_SERVER_SSL_CERT),
auth_metadata_plugin=self._auth_metadata,
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
)
self._job_service_service_stub = JobServiceStub(channel)
return self._job_service_service_stub
def _extra_grpc_params(self) -> Dict[str, Any]:
return dict(
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
)
@property
def core_url(self) -> str:
"""
Retrieve Feast Core URL
Returns:
Feast Core URL string
"""
return self._config.get(opt.CORE_URL)
@core_url.setter
def core_url(self, value: str):
"""
Set the Feast Core URL
Args:
value: Feast Core URL
"""
self._config.set(opt.CORE_URL, value)
@property
def serving_url(self) -> str:
"""
Retrieve Feast Serving URL
Returns:
Feast Serving URL string
"""
return self._config.get(opt.SERVING_URL)
@serving_url.setter
def serving_url(self, value: str):
"""
Set the Feast Serving URL
Args:
value: Feast Serving URL
"""
self._config.set(opt.SERVING_URL, value)
@property
def job_service_url(self) -> str:
"""
Retrieve Feast Job Service URL
Returns:
Feast Job Service URL string
"""
return self._config.get(opt.JOB_SERVICE_URL)
@job_service_url.setter
def job_service_url(self, value: str):
"""
Set the Feast Job Service URL
Args:
value: Feast Job Service URL
"""
self._config.set(opt.JOB_SERVICE_URL, value)
@property
def core_secure(self) -> bool:
"""
Retrieve Feast Core client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.CORE_ENABLE_SSL)
@core_secure.setter
def core_secure(self, value: bool):
"""
Set the Feast Core client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.CORE_ENABLE_SSL, value)
@property
def serving_secure(self) -> bool:
"""
Retrieve Feast Serving client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.SERVING_ENABLE_SSL)
@serving_secure.setter
def serving_secure(self, value: bool):
"""
Set the Feast Serving client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.SERVING_ENABLE_SSL, value)
@property
def job_service_secure(self) -> bool:
"""
Retrieve Feast Job Service client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL)
@job_service_secure.setter
def job_service_secure(self, value: bool):
"""
Set the Feast Job Service client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.JOB_SERVICE_ENABLE_SSL, value)
def version(self):
"""
Returns version information from Feast Core and Feast Serving
"""
import pkg_resources
result = {
"sdk": {
"version": pkg_resources.get_distribution("feast").version
},
"serving": "not configured",
"core": "not configured",
}
if self.serving_url:
serving_version = self._serving_service.GetFeastServingInfo(
GetFeastServingInfoRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
).version
result["serving"] = {
"url": self.serving_url,
"version": serving_version
}
if self.core_url:
core_version = self._core_service.GetFeastCoreVersion(
GetFeastCoreVersionRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
).version
result["core"] = {"url": self.core_url, "version": core_version}
return result
@property
def project(self) -> str:
"""
Retrieve currently active project
Returns:
Project name
"""
if not self._config.get(opt.PROJECT):
raise ValueError("No project has been configured.")
return self._config.get(opt.PROJECT)
def set_project(self, project: Optional[str] = None):
"""
Set currently active Feast project
Args:
project: Project to set as active. If unset, will reset to the default project.
"""
if project is None:
project = opt().PROJECT
self._config.set(opt.PROJECT, project)
def list_projects(self) -> List[str]:
"""
List all active Feast projects
Returns:
List of project names
"""
response = self._core_service.ListProjects(
ListProjectsRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ListProjectsResponse
return list(response.projects)
def create_project(self, project: str):
"""
Creates a Feast project
Args:
project: Name of project
"""
self._core_service.CreateProject(
CreateProjectRequest(name=project),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: CreateProjectResponse
def archive_project(self, project):
"""
Archives a project. Project will still continue to function for
ingestion and retrieval, but will be in a read-only state. It will
also not be visible from the Core API for management purposes.
Args:
project: Name of project to archive
"""
try:
self._core_service_stub.ArchiveProject(
ArchiveProjectRequest(name=project),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ArchiveProjectResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# revert to the default project
if self._project == project:
self._project = opt().PROJECT
def apply(
self,
objects: Union[List[Union[Entity, FeatureTable]], Entity,
FeatureTable],
project: str = None,
):
"""
Idempotently registers entities and feature tables with Feast Core. Either a single
entity or feature table or a list can be provided.
Args:
objects: List of entities and/or feature tables that will be registered
Examples:
>>> from feast import Client
>>> from feast.entity import Entity
>>> from feast.value_type import ValueType
>>>
>>> feast_client = Client(core_url="localhost:6565")
>>> entity = Entity(
>>> name="driver_entity",
>>> description="Driver entity for car rides",
>>> value_type=ValueType.STRING,
>>> labels={
>>> "key": "val"
>>> }
>>> )
>>> feast_client.apply(entity)
"""
if project is None:
project = self.project
if not isinstance(objects, list):
objects = [objects]
for obj in objects:
if isinstance(obj, Entity):
self._apply_entity(project, obj) # type: ignore
elif isinstance(obj, FeatureTable):
self._apply_feature_table(project, obj) # type: ignore
else:
raise ValueError(
f"Could not determine object type to apply {obj} with type {type(obj)}. Type must be Entity or FeatureTable."
)
def apply_entity(self,
entities: Union[List[Entity], Entity],
project: str = None):
"""
Deprecated. Please see apply().
"""
warnings.warn(
"The method apply_entity() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_entity().",
DeprecationWarning,
)
if project is None:
project = self.project
if not isinstance(entities, list):
entities = [entities]
for entity in entities:
if isinstance(entity, Entity):
self._apply_entity(project, entity) # type: ignore
continue
raise ValueError(
f"Could not determine entity type to apply {entity}")
def _apply_entity(self, project: str, entity: Entity):
"""
Registers a single entity with Feast
Args:
entity: Entity that will be registered
"""
entity.is_valid()
entity_proto = entity.to_spec_proto()
# Convert the entity to a request and send to Feast Core
try:
apply_entity_response = self._core_service.ApplyEntity(
ApplyEntityRequest(project=project,
spec=entity_proto), # type: ignore
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ApplyEntityResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# Extract the returned entity
applied_entity = Entity.from_proto(apply_entity_response.entity)
# Deep copy from the returned entity to the local entity
entity._update_from_entity(applied_entity)
def list_entities(self,
project: str = None,
labels: Dict[str, str] = dict()) -> List[Entity]:
"""
Retrieve a list of entities from Feast Core
Args:
project: Filter entities based on project name
labels: User-defined labels that these entities are associated with
Returns:
List of entities
"""
if project is None:
project = self.project
filter = ListEntitiesRequest.Filter(project=project, labels=labels)
# Get latest entities from Feast Core
entity_protos = self._core_service.ListEntities(
ListEntitiesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListEntitiesResponse
# Extract entities and return
entities = []
for entity_proto in entity_protos.entities:
entity = Entity.from_proto(entity_proto)
entity._client = self
entities.append(entity)
return entities
def get_entity(self, name: str, project: str = None) -> Entity:
"""
Retrieves an entity.
Args:
project: Feast project that this entity belongs to
name: Name of entity
Returns:
Returns either the specified entity, or raises an exception if
none is found
"""
if project is None:
project = self.project
try:
get_entity_response = self._core_service.GetEntity(
GetEntityRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
) # type: GetEntityResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
entity = Entity.from_proto(get_entity_response.entity)
return entity
def apply_feature_table(
self,
feature_tables: Union[List[FeatureTable], FeatureTable],
project: str = None,
):
"""
Deprecated. Please see apply().
"""
warnings.warn(
"The method apply_feature_table() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_feature_table().",
DeprecationWarning,
)
if project is None:
project = self.project
if not isinstance(feature_tables, list):
feature_tables = [feature_tables]
for feature_table in feature_tables:
if isinstance(feature_table, FeatureTable):
self._apply_feature_table(project,
feature_table) # type: ignore
continue
raise ValueError(
f"Could not determine feature table type to apply {feature_table}"
)
def _apply_feature_table(self, project: str, feature_table: FeatureTable):
"""
Registers a single feature table with Feast
Args:
feature_table: Feature table that will be registered
"""
feature_table.is_valid()
feature_table_proto = feature_table.to_spec_proto()
# Convert the feature table to a request and send to Feast Core
try:
apply_feature_table_response = self._core_service.ApplyFeatureTable(
ApplyFeatureTableRequest(
project=project,
table_spec=feature_table_proto), # type: ignore
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ApplyFeatureTableResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# Extract the returned feature table
applied_feature_table = FeatureTable.from_proto(
apply_feature_table_response.table)
# Deep copy from the returned feature table to the local entity
feature_table._update_from_feature_table(applied_feature_table)
def list_feature_tables(
self,
project: str = None,
labels: Dict[str, str] = dict()
) -> List[FeatureTable]:
"""
Retrieve a list of feature tables from Feast Core
Args:
project: Filter feature tables based on project name
Returns:
List of feature tables
"""
if project is None:
project = self.project
filter = ListFeatureTablesRequest.Filter(project=project,
labels=labels)
# Get latest feature tables from Feast Core
feature_table_protos = self._core_service.ListFeatureTables(
ListFeatureTablesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListFeatureTablesResponse
# Extract feature tables and return
feature_tables = []
for feature_table_proto in feature_table_protos.tables:
feature_table = FeatureTable.from_proto(feature_table_proto)
feature_table._client = self
feature_tables.append(feature_table)
return feature_tables
def get_feature_table(self,
name: str,
project: str = None) -> FeatureTable:
"""
Retrieves a feature table.
Args:
project: Feast project that this feature table belongs to
name: Name of feature table
Returns:
Returns either the specified feature table, or raises an exception if
none is found
"""
if project is None:
project = self.project
try:
get_feature_table_response = self._core_service.GetFeatureTable(
GetFeatureTableRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
) # type: GetFeatureTableResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return FeatureTable.from_proto(get_feature_table_response.table)
def delete_feature_table(self, name: str, project: str = None) -> None:
"""
Deletes a feature table.
Args:
project: Feast project that this feature table belongs to
name: Name of feature table
"""
if project is None:
project = self.project
try:
self._core_service.DeleteFeatureTable(
DeleteFeatureTableRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
def list_features_by_ref(
self,
project: str = None,
entities: List[str] = list(),
labels: Dict[str, str] = dict(),
) -> Dict[FeatureRef, Feature]:
"""
Retrieve a dictionary of feature reference to feature from Feast Core based on filters provided.
Args:
project: Feast project that these features belongs to
entities: Feast entity that these features are associated with
labels: Feast labels that these features are associated with
Returns:
Dictionary of <feature references: features>
Examples:
>>> from feast import Client
>>>
>>> feast_client = Client(core_url="localhost:6565")
>>> features = feast_client.list_features(project="test_project", entities=["driver_id"], labels={"key1":"val1","key2":"val2"})
>>> print(features)
"""
if project is None:
project = self.project
filter = ListFeaturesRequest.Filter(project=project,
entities=entities,
labels=labels)
feature_protos = self._core_service.ListFeatures(
ListFeaturesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListFeaturesResponse
# Extract features and return
features_dict = {}
for ref_str, feature_proto in feature_protos.features.items():
feature_ref = FeatureRef.from_str(ref_str)
feature = Feature.from_proto(feature_proto)
features_dict[feature_ref] = feature
return features_dict
def ingest(
self,
feature_table: Union[str, FeatureTable],
source: Union[pd.DataFrame, str],
project: str = None,
chunk_size: int = 10000,
max_workers: int = max(CPU_COUNT - 1, 1),
timeout: int = int(opt().BATCH_INGESTION_PRODUCTION_TIMEOUT),
) -> None:
"""
Batch load feature data into a FeatureTable.
Args:
feature_table (typing.Union[str, feast.feature_table.FeatureTable]):
FeatureTable object or the string name of the feature table
source (typing.Union[pd.DataFrame, str]):
Either a file path or Pandas Dataframe to ingest into Feast
Files that are currently supported:
* parquet
* csv
* json
project: Feast project to locate FeatureTable
chunk_size (int):
Amount of rows to load and ingest at a time.
max_workers (int):
Number of worker processes to use to encode values.
timeout (int):
Timeout in seconds to wait for completion.
Examples:
>>> from feast import Client
>>>
>>> client = Client(core_url="localhost:6565")
>>> ft_df = pd.DataFrame(
>>> {
>>> "datetime": [pd.datetime.now()],
>>> "driver": [1001],
>>> "rating": [4.3],
>>> }
>>> )
>>> client.set_project("project1")
>>>
>>> driver_ft = client.get_feature_table("driver")
>>> client.ingest(driver_ft, ft_df)
"""
if project is None:
project = self.project
if isinstance(feature_table, str):
name = feature_table
if isinstance(feature_table, FeatureTable):
name = feature_table.name
fetched_feature_table: Optional[FeatureTable] = self.get_feature_table(
name, project)
if fetched_feature_table is not None:
feature_table = fetched_feature_table
else:
raise Exception(f"FeatureTable, {name} cannot be found.")
# Check 1) Only parquet file format for FeatureTable batch source is supported
if (feature_table.batch_source
and issubclass(type(feature_table.batch_source), FileSource)
and isinstance(
type(feature_table.batch_source.file_options.file_format),
ParquetFormat)):
raise Exception(
f"No suitable batch source found for FeatureTable, {name}."
f"Only BATCH_FILE source with parquet format is supported for batch ingestion."
)
pyarrow_table, column_names = _read_table_from_source(source)
# Check 2) Check if FeatureTable batch source field mappings can be found in provided source table
_check_field_mappings(
column_names,
name,
feature_table.batch_source.event_timestamp_column,
feature_table.batch_source.field_mapping,
)
dir_path = None
with_partitions = False
if (issubclass(type(feature_table.batch_source), FileSource)
and feature_table.batch_source.date_partition_column):
with_partitions = True
dest_path = _write_partitioned_table_from_source(
column_names,
pyarrow_table,
feature_table.batch_source.date_partition_column,
feature_table.batch_source.event_timestamp_column,
)
else:
dir_path, dest_path = _write_non_partitioned_table_from_source(
column_names,
pyarrow_table,
chunk_size,
max_workers,
)
try:
if issubclass(type(feature_table.batch_source), FileSource):
file_url = feature_table.batch_source.file_options.file_url.rstrip(
"*")
_upload_to_file_source(file_url, with_partitions, dest_path,
self._config)
if issubclass(type(feature_table.batch_source), BigQuerySource):
bq_table_ref = feature_table.batch_source.bigquery_options.table_ref
feature_table_timestamp_column = (
feature_table.batch_source.event_timestamp_column)
_upload_to_bq_source(bq_table_ref,
feature_table_timestamp_column, dest_path)
finally:
# Remove parquet file(s) that were created earlier
print("Removing temporary file(s)...")
if dir_path:
shutil.rmtree(dir_path)
print(
"Data has been successfully ingested into FeatureTable batch source."
)
def _get_grpc_metadata(self):
"""
Returns a metadata tuple to attach to gRPC requests. This is primarily
used when authentication is enabled but SSL/TLS is disabled.
Returns: Tuple of metadata to attach to each gRPC call
"""
if self._config.getboolean(opt.ENABLE_AUTH) and self._auth_metadata:
return self._auth_metadata.get_signed_meta()
return ()
def get_online_features(
self,
feature_refs: List[str],
entity_rows: List[Dict[str, Any]],
project: Optional[str] = None,
) -> OnlineResponse:
"""
Retrieves the latest online feature data from Feast Serving.
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
Only the feature name is required.
entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair.
project: Optionally specify the the project override. If specified, uses given project for retrieval.
Overrides the projects specified in Feature References if also are specified.
Returns:
GetOnlineFeaturesResponse containing the feature data in records.
Each EntityRow provided will yield one record, which contains
data fields with data value and field status metadata (if included).
Examples:
>>> from feast import Client
>>>
>>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566")
>>> feature_refs = ["sales:daily_transactions"]
>>> entity_rows = [{"customer_id": 0},{"customer_id": 1}]
>>>
>>> online_response = feast_client.get_online_features(
>>> feature_refs, entity_rows, project="my_project")
>>> online_response_dict = online_response.to_dict()
>>> print(online_response_dict)
{'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]}
"""
try:
response = self._serving_service.GetOnlineFeaturesV2(
GetOnlineFeaturesRequestV2(
features=_build_feature_references(
feature_ref_strs=feature_refs),
entity_rows=_infer_online_entity_rows(entity_rows),
project=project if project is not None else self.project,
),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
response = OnlineResponse(response)
return response
def get_historical_features(
self,
feature_refs: List[str],
entity_source: Union[pd.DataFrame, FileSource, BigQuerySource],
output_location: Optional[str] = None,
) -> RetrievalJob:
"""
Launch a historical feature retrieval job.
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
entity_source (Union[pd.DataFrame, FileSource, BigQuerySource]): Source for the entity rows.
If entity_source is a Panda DataFrame, the dataframe will be staged
to become accessible by spark workers.
If one of feature tables' source is in BigQuery - entities will be upload to BQ.
Otherwise to remote file storage (derived from configured staging location).
It is also assumed that the column event_timestamp is present
in the dataframe, and is of type datetime without timezone information.
The user needs to make sure that the source (or staging location, if entity_source is
a Panda DataFrame) is accessible from the Spark cluster that will be used for the
retrieval job.
destination_path: Specifies the path in a bucket to write the exported feature data files
Returns:
Returns a retrieval job object that can be used to monitor retrieval
progress asynchronously, and can be used to materialize the
results.
Examples:
>>> from feast import Client
>>> from feast.data_format import ParquetFormat
>>> from datetime import datetime
>>> feast_client = Client(core_url="localhost:6565")
>>> feature_refs = ["bookings:bookings_7d", "bookings:booking_14d"]
>>> entity_source = FileSource("event_timestamp", ParquetFormat(), "gs://some-bucket/customer")
>>> feature_retrieval_job = feast_client.get_historical_features(
>>> feature_refs, entity_source)
>>> output_file_uri = feature_retrieval_job.get_output_file_uri()
"gs://some-bucket/output/
"""
feature_tables = self._get_feature_tables_from_feature_refs(
feature_refs, self.project)
assert all(
ft.batch_source.created_timestamp_column
for ft in feature_tables), (
"All BatchSources attached to retrieved FeatureTables "
"must have specified `created_timestamp_column` to be used in "
"historical dataset generation.")
if output_location is None:
output_location = os.path.join(
self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_LOCATION),
str(uuid.uuid4()),
)
output_format = self._config.get(opt.HISTORICAL_FEATURE_OUTPUT_FORMAT)
feature_sources = [
feature_table.batch_source for feature_table in feature_tables
]
if isinstance(entity_source, pd.DataFrame):
if any(
isinstance(source, BigQuerySource)
for source in feature_sources):
first_bq_source = [
source for source in feature_sources
if isinstance(source, BigQuerySource)
][0]
source_ref = table_reference_from_string(
first_bq_source.bigquery_options.table_ref)
entity_source = stage_entities_to_bq(entity_source,
source_ref.project,
source_ref.dataset_id)
else:
entity_source = stage_entities_to_fs(
entity_source,
staging_location=self._config.get(
opt.SPARK_STAGING_LOCATION),
config=self._config,
)
if self._use_job_service:
response = self._job_service.GetHistoricalFeatures(
GetHistoricalFeaturesRequest(
feature_refs=feature_refs,
entity_source=entity_source.to_proto(),
project=self.project,
output_format=output_format,
output_location=output_location,
),
**self._extra_grpc_params(),
)
return RemoteRetrievalJob(
self._job_service,
self._extra_grpc_params,
response.id,
output_file_uri=response.output_file_uri,
)
else:
return start_historical_feature_retrieval_job(
client=self,
project=self.project,
entity_source=entity_source,
feature_tables=feature_tables,
output_format=output_format,
output_path=output_location,
)
def get_historical_features_df(
self,
feature_refs: List[str],
entity_source: Union[FileSource, BigQuerySource],
):
"""
Launch a historical feature retrieval job.
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
entity_source (Union[FileSource, BigQuerySource]): Source for the entity rows.
The user needs to make sure that the source is accessible from the Spark cluster
that will be used for the retrieval job.
Returns:
Returns the historical feature retrieval result in the form of Spark dataframe.
Examples:
>>> from feast import Client
>>> from feast.data_format import ParquetFormat
>>> from datetime import datetime
>>> from pyspark.sql import SparkSession
>>> spark = SparkSession.builder.getOrCreate()
>>> feast_client = Client(core_url="localhost:6565")
>>> feature_refs = ["bookings:bookings_7d", "bookings:booking_14d"]
>>> entity_source = FileSource("event_timestamp", ParquetFormat, "gs://some-bucket/customer")
>>> df = feast_client.get_historical_features(
>>> feature_refs, entity_source)
"""
feature_tables = self._get_feature_tables_from_feature_refs(
feature_refs, self.project)
return start_historical_feature_retrieval_spark_session(
client=self,
project=self.project,
entity_source=entity_source,
feature_tables=feature_tables,
)
def _get_feature_tables_from_feature_refs(self, feature_refs: List[str],
project: Optional[str]):
feature_refs_grouped_by_table = [
(feature_table_name, list(grouped_feature_refs))
for feature_table_name, grouped_feature_refs in groupby(
feature_refs, lambda x: x.split(":")[0])
]
feature_tables = []
for feature_table_name, grouped_feature_refs in feature_refs_grouped_by_table:
feature_table = self.get_feature_table(feature_table_name, project)
feature_names = [f.split(":")[-1] for f in grouped_feature_refs]
feature_table.features = [
f for f in feature_table.features if f.name in feature_names
]
feature_tables.append(feature_table)
return feature_tables
def start_offline_to_online_ingestion(
self,
feature_table: FeatureTable,
start: datetime,
end: datetime,
) -> SparkJob:
"""
Launch Ingestion Job from Batch Source to Online Store for given featureTable
:param feature_table: FeatureTable which will be ingested
:param start: lower datetime boundary
:param end: upper datetime boundary
:return: Spark Job Proxy object
"""
if not self._use_job_service:
return start_offline_to_online_ingestion(
client=self,
project=self.project,
feature_table=feature_table,
start=start,
end=end,
)
else:
request = StartOfflineToOnlineIngestionJobRequest(
project=self.project,
table_name=feature_table.name,
)
request.start_date.FromDatetime(start)
request.end_date.FromDatetime(end)
response = self._job_service.StartOfflineToOnlineIngestionJob(
request)
return RemoteBatchIngestionJob(
self._job_service,
self._extra_grpc_params,
response.id,
)
def start_stream_to_online_ingestion(
self,
feature_table: FeatureTable,
extra_jars: Optional[List[str]] = None,
project: str = None,
) -> SparkJob:
if not self._use_job_service:
return start_stream_to_online_ingestion(
client=self,
project=project or self.project,
feature_table=feature_table,
extra_jars=extra_jars or [],
)
else:
request = StartStreamToOnlineIngestionJobRequest(
project=self.project,
table_name=feature_table.name,
)
response = self._job_service.StartStreamToOnlineIngestionJob(
request)
return RemoteStreamIngestionJob(self._job_service,
self._extra_grpc_params,
response.id)
def list_jobs(self, include_terminated: bool) -> List[SparkJob]:
if not self._use_job_service:
return list_jobs(include_terminated, self)
else:
request = ListJobsRequest(include_terminated=include_terminated)
response = self._job_service.ListJobs(request)
return [
get_remote_job_from_proto(self._job_service,
self._extra_grpc_params, job)
for job in response.jobs
]
def get_job_by_id(self, job_id: str) -> SparkJob:
if not self._use_job_service:
return get_job_by_id(job_id, self)
else:
request = GetJobRequest(job_id=job_id)
response = self._job_service.GetJob(request)
return get_remote_job_from_proto(self._job_service,
self._extra_grpc_params,
response.job)
def stage_dataframe(
self,
df: pd.DataFrame,
event_timestamp_column: str,
) -> FileSource:
return stage_dataframe(df, event_timestamp_column, self._config)
class Client:
"""
Feast Client: Used for creating, managing, and retrieving features.
"""
def __init__(self, options: Optional[Dict[str, str]] = None, **kwargs):
"""
The Feast Client should be initialized with at least one service url
Please see constants.py for configuration options. Commonly used options
or arguments include:
core_url: Feast Core URL. Used to manage features
serving_url: Feast Serving URL. Used to retrieve features
project: Sets the active project. This field is optional.
core_secure: Use client-side SSL/TLS for Core gRPC API
serving_secure: Use client-side SSL/TLS for Serving gRPC API
enable_auth: Enable authentication and authorization
auth_provider: Authentication provider – "google" or "oauth"
if auth_provider is "oauth", the following fields are mandatory –
oauth_grant_type, oauth_client_id, oauth_client_secret, oauth_audience, oauth_token_request_url
Args:
options: Configuration options to initialize client with
**kwargs: Additional keyword arguments that will be used as
configuration options along with "options"
"""
if options is None:
options = dict()
self._config = Config(options={**options, **kwargs})
self._core_service_stub: Optional[CoreServiceStub] = None
self._serving_service_stub: Optional[ServingServiceStub] = None
self._auth_metadata: Optional[grpc.AuthMetadataPlugin] = None
self._registry_impl: Optional[Registry] = None
# Configure Auth Metadata Plugin if auth is enabled
if self._config.getboolean(opt.ENABLE_AUTH):
self._auth_metadata = feast_auth.get_auth_metadata_plugin(
self._config)
self._configure_telemetry()
@property
def config(self) -> Config:
return self._config
@property
def _core_service(self):
"""
Creates or returns the gRPC Feast Core Service Stub
Returns: CoreServiceStub
"""
if not self._core_service_stub:
channel = create_grpc_channel(
url=self._config.get(opt.CORE_URL),
enable_ssl=self._config.getboolean(opt.CORE_ENABLE_SSL),
enable_auth=self._config.getboolean(opt.ENABLE_AUTH),
ssl_server_cert_path=self._config.get(
opt.CORE_SERVER_SSL_CERT),
auth_metadata_plugin=self._auth_metadata,
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
)
self._core_service_stub = CoreServiceStub(channel)
return self._core_service_stub
@property
def _use_object_store_registry(self) -> bool:
return self._config.exists(opt.REGISTRY_PATH)
@property
def _registry(self):
if self._registry_impl is None:
self._registry_impl = Registry(self._config.get(opt.REGISTRY_PATH))
return self._registry_impl
@property
def _serving_service(self):
"""
Creates or returns the gRPC Feast Serving Service Stub. If both `opentracing`
and `grpcio-opentracing` are installed, an opentracing interceptor will be
instantiated based on the global tracer.
Returns: ServingServiceStub
"""
if not self._serving_service_stub:
channel = create_grpc_channel(
url=self._config.get(opt.SERVING_URL),
enable_ssl=self._config.getboolean(opt.SERVING_ENABLE_SSL),
enable_auth=self._config.getboolean(opt.ENABLE_AUTH),
ssl_server_cert_path=self._config.get(
opt.SERVING_SERVER_SSL_CERT),
auth_metadata_plugin=self._auth_metadata,
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
)
try:
import opentracing
from grpc_opentracing import open_tracing_client_interceptor
from grpc_opentracing.grpcext import intercept_channel
interceptor = open_tracing_client_interceptor(
opentracing.global_tracer())
channel = intercept_channel(channel, interceptor)
except ImportError:
pass
self._serving_service_stub = ServingServiceStub(channel)
return self._serving_service_stub
def _extra_grpc_params(self) -> Dict[str, Any]:
return dict(
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
)
@property
def core_url(self) -> str:
"""
Retrieve Feast Core URL
Returns:
Feast Core URL string
"""
return self._config.get(opt.CORE_URL)
@core_url.setter
def core_url(self, value: str):
"""
Set the Feast Core URL
Args:
value: Feast Core URL
"""
self._config.set(opt.CORE_URL, value)
@property
def serving_url(self) -> str:
"""
Retrieve Feast Serving URL
Returns:
Feast Serving URL string
"""
return self._config.get(opt.SERVING_URL)
@serving_url.setter
def serving_url(self, value: str):
"""
Set the Feast Serving URL
Args:
value: Feast Serving URL
"""
self._config.set(opt.SERVING_URL, value)
@property
def job_service_url(self) -> str:
"""
Retrieve Feast Job Service URL
Returns:
Feast Job Service URL string
"""
return self._config.get(opt.JOB_SERVICE_URL)
@job_service_url.setter
def job_service_url(self, value: str):
"""
Set the Feast Job Service URL
Args:
value: Feast Job Service URL
"""
self._config.set(opt.JOB_SERVICE_URL, value)
@property
def core_secure(self) -> bool:
"""
Retrieve Feast Core client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.CORE_ENABLE_SSL)
@core_secure.setter
def core_secure(self, value: bool):
"""
Set the Feast Core client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.CORE_ENABLE_SSL, value)
@property
def serving_secure(self) -> bool:
"""
Retrieve Feast Serving client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.SERVING_ENABLE_SSL)
@serving_secure.setter
def serving_secure(self, value: bool):
"""
Set the Feast Serving client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.SERVING_ENABLE_SSL, value)
@property
def job_service_secure(self) -> bool:
"""
Retrieve Feast Job Service client-side SSL/TLS setting
Returns:
Whether client-side SSL/TLS is enabled
"""
return self._config.getboolean(opt.JOB_SERVICE_ENABLE_SSL)
@job_service_secure.setter
def job_service_secure(self, value: bool):
"""
Set the Feast Job Service client-side SSL/TLS setting
Args:
value: True to enable client-side SSL/TLS
"""
self._config.set(opt.JOB_SERVICE_ENABLE_SSL, value)
def version(self, sdk_only=False):
"""
Returns version information from Feast Core and Feast Serving
"""
import pkg_resources
try:
sdk_version = pkg_resources.get_distribution("feast").version
except pkg_resources.DistributionNotFound:
sdk_version = "local build"
if sdk_only:
return sdk_version
result = {
"sdk": {
"version": sdk_version
},
"serving": "not configured",
"core": "not configured",
}
if self.serving_url:
serving_version = self._serving_service.GetFeastServingInfo(
GetFeastServingInfoRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
).version
result["serving"] = {
"url": self.serving_url,
"version": serving_version
}
if not self._use_object_store_registry and self.core_url:
core_version = self._core_service.GetFeastCoreVersion(
GetFeastCoreVersionRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
).version
result["core"] = {"url": self.core_url, "version": core_version}
return result
def _configure_telemetry(self):
telemetry_filepath = join(expanduser("~"), ".feast", "telemetry")
self._telemetry_enabled = (
self._config.get(opt.TELEMETRY, "True") == "True"
) # written this way to turn the env var string into a boolean
if self._telemetry_enabled:
self._telemetry_counter = {"get_online_features": 0}
if os.path.exists(telemetry_filepath):
with open(telemetry_filepath, "r") as f:
self._telemetry_id = f.read()
else:
self._telemetry_id = str(uuid.uuid4())
print(
"Feast is an open source project that collects anonymized usage statistics. To opt out or learn more see https://docs.feast.dev/v/master/advanced/telemetry"
)
with open(telemetry_filepath, "w") as f:
f.write(self._telemetry_id)
else:
if os.path.exists(telemetry_filepath):
os.remove(telemetry_filepath)
@property
def project(self) -> str:
"""
Retrieve currently active project
Returns:
Project name
"""
if not self._config.get(opt.PROJECT):
raise ValueError("No project has been configured.")
return self._config.get(opt.PROJECT)
def set_project(self, project: Optional[str] = None):
"""
Set currently active Feast project
Args:
project: Project to set as active. If unset, will reset to the default project.
"""
if project is None:
project = opt().PROJECT
self._config.set(opt.PROJECT, project)
def list_projects(self) -> List[str]:
"""
List all active Feast projects
Returns:
List of project names
"""
if self._use_object_store_registry:
raise NotImplementedError(
"Projects are not implemented for object store registry.")
else:
response = self._core_service.ListProjects(
ListProjectsRequest(),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ListProjectsResponse
return list(response.projects)
def create_project(self, project: str):
"""
Creates a Feast project
Args:
project: Name of project
"""
if self._use_object_store_registry:
raise NotImplementedError(
"Projects are not implemented for object store registry.")
else:
self._core_service.CreateProject(
CreateProjectRequest(name=project),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: CreateProjectResponse
def archive_project(self, project):
"""
Archives a project. Project will still continue to function for
ingestion and retrieval, but will be in a read-only state. It will
also not be visible from the Core API for management purposes.
Args:
project: Name of project to archive
"""
if self._use_object_store_registry:
raise NotImplementedError(
"Projects are not implemented for object store registry.")
else:
try:
self._core_service.ArchiveProject(
ArchiveProjectRequest(name=project),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ArchiveProjectResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# revert to the default project
if self._project == project:
self._project = opt().PROJECT
def apply(
self,
objects: Union[List[Union[Entity, FeatureTable]], Entity,
FeatureTable],
project: str = None,
):
"""
Idempotently registers entities and feature tables with Feast Core. Either a single
entity or feature table or a list can be provided.
Args:
objects: List of entities and/or feature tables that will be registered
Examples:
>>> from feast import Client
>>> from feast.entity import Entity
>>> from feast.value_type import ValueType
>>>
>>> feast_client = Client(core_url="localhost:6565")
>>> entity = Entity(
>>> name="driver_entity",
>>> description="Driver entity for car rides",
>>> value_type=ValueType.STRING,
>>> labels={
>>> "key": "val"
>>> }
>>> )
>>> feast_client.apply(entity)
"""
if self._telemetry_enabled:
log_usage(
"apply",
self._telemetry_id,
datetime.utcnow(),
self.version(sdk_only=True),
)
if project is None:
project = self.project
if not isinstance(objects, list):
objects = [objects]
for obj in objects:
if isinstance(obj, Entity):
self._apply_entity(project, obj) # type: ignore
elif isinstance(obj, FeatureTable):
self._apply_feature_table(project, obj) # type: ignore
else:
raise ValueError(
f"Could not determine object type to apply {obj} with type {type(obj)}. Type must be Entity or FeatureTable."
)
def apply_entity(self,
entities: Union[List[Entity], Entity],
project: str = None):
"""
Deprecated. Please see apply().
"""
warnings.warn(
"The method apply_entity() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_entity().",
DeprecationWarning,
)
if project is None:
project = self.project
if not isinstance(entities, list):
entities = [entities]
for entity in entities:
if isinstance(entity, Entity):
self._apply_entity(project, entity) # type: ignore
continue
raise ValueError(
f"Could not determine entity type to apply {entity}")
def _apply_entity(self, project: str, entity: Entity):
"""
Registers a single entity with Feast
Args:
entity: Entity that will be registered
"""
if self._use_object_store_registry:
return self._registry.apply_entity(entity, project)
else:
entity.is_valid()
entity_proto = entity.to_spec_proto()
# Convert the entity to a request and send to Feast Core
try:
apply_entity_response = self._core_service.ApplyEntity(
ApplyEntityRequest(project=project,
spec=entity_proto), # type: ignore
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ApplyEntityResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# Extract the returned entity
applied_entity = Entity.from_proto(apply_entity_response.entity)
# Deep copy from the returned entity to the local entity
entity._update_from_entity(applied_entity)
def list_entities(self,
project: str = None,
labels: Dict[str, str] = dict()) -> List[Entity]:
"""
Retrieve a list of entities from Feast Core
Args:
project: Filter entities based on project name
labels: User-defined labels that these entities are associated with
Returns:
List of entities
"""
if project is None:
project = self.project
if self._use_object_store_registry:
return self._registry.list_entities(project)
else:
filter = ListEntitiesRequest.Filter(project=project, labels=labels)
# Get latest entities from Feast Core
entity_protos = self._core_service.ListEntities(
ListEntitiesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListEntitiesResponse
# Extract entities and return
entities = []
for entity_proto in entity_protos.entities:
entity = Entity.from_proto(entity_proto)
entity._client = self
entities.append(entity)
return entities
def get_entity(self, name: str, project: str = None) -> Entity:
"""
Retrieves an entity.
Args:
project: Feast project that this entity belongs to
name: Name of entity
Returns:
Returns either the specified entity, or raises an exception if
none is found
"""
if self._telemetry_enabled:
log_usage(
"get_entity",
self._telemetry_id,
datetime.utcnow(),
self.version(sdk_only=True),
)
if project is None:
project = self.project
if self._use_object_store_registry:
return self._registry.get_entity(name, project)
else:
try:
get_entity_response = self._core_service.GetEntity(
GetEntityRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
) # type: GetEntityResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
entity = Entity.from_proto(get_entity_response.entity)
return entity
def apply_feature_table(
self,
feature_tables: Union[List[FeatureTable], FeatureTable],
project: str = None,
):
"""
Deprecated. Please see apply().
"""
warnings.warn(
"The method apply_feature_table() is being deprecated. Please use apply() instead. Feast 0.10 and onwards will not support apply_feature_table().",
DeprecationWarning,
)
if project is None:
project = self.project
if not isinstance(feature_tables, list):
feature_tables = [feature_tables]
for feature_table in feature_tables:
if isinstance(feature_table, FeatureTable):
self._apply_feature_table(project,
feature_table) # type: ignore
continue
raise ValueError(
f"Could not determine feature table type to apply {feature_table}"
)
def _apply_feature_table(self, project: str, feature_table: FeatureTable):
"""
Registers a single feature table with Feast
Args:
feature_table: Feature table that will be registered
"""
if self._use_object_store_registry:
return self._registry.apply_feature_table(feature_table, project)
else:
feature_table.is_valid()
feature_table_proto = feature_table.to_spec_proto()
# Convert the feature table to a request and send to Feast Core
try:
apply_feature_table_response = self._core_service.ApplyFeatureTable(
ApplyFeatureTableRequest(
project=project,
table_spec=feature_table_proto), # type: ignore
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
) # type: ApplyFeatureTableResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
# Extract the returned feature table
applied_feature_table = FeatureTable.from_proto(
apply_feature_table_response.table)
# Deep copy from the returned feature table to the local entity
feature_table._update_from_feature_table(applied_feature_table)
def list_feature_tables(
self,
project: str = None,
labels: Dict[str, str] = dict()
) -> List[FeatureTable]:
"""
Retrieve a list of feature tables from Feast Core
Args:
project: Filter feature tables based on project name
Returns:
List of feature tables
"""
if project is None:
project = self.project
if self._use_object_store_registry:
return self._registry.list_feature_tables(project)
else:
filter = ListFeatureTablesRequest.Filter(project=project,
labels=labels)
# Get latest feature tables from Feast Core
feature_table_protos = self._core_service.ListFeatureTables(
ListFeatureTablesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListFeatureTablesResponse
# Extract feature tables and return
feature_tables = []
for feature_table_proto in feature_table_protos.tables:
feature_table = FeatureTable.from_proto(feature_table_proto)
feature_table._client = self
feature_tables.append(feature_table)
return feature_tables
def get_feature_table(self,
name: str,
project: str = None) -> FeatureTable:
"""
Retrieves a feature table.
Args:
project: Feast project that this feature table belongs to
name: Name of feature table
Returns:
Returns either the specified feature table, or raises an exception if
none is found
"""
if self._telemetry_enabled:
log_usage(
"get_feature_table",
self._telemetry_id,
datetime.utcnow(),
self.version(sdk_only=True),
)
if project is None:
project = self.project
if self._use_object_store_registry:
return self._registry.get_feature_table(name, project)
else:
try:
get_feature_table_response = self._core_service.GetFeatureTable(
GetFeatureTableRequest(project=project, name=name.strip()),
metadata=self._get_grpc_metadata(),
) # type: GetFeatureTableResponse
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
return FeatureTable.from_proto(get_feature_table_response.table)
def delete_feature_table(self, name: str, project: str = None) -> None:
"""
Deletes a feature table.
Args:
project: Feast project that this feature table belongs to
name: Name of feature table
"""
if project is None:
project = self.project
if self._use_object_store_registry:
return self._registry.delete_feature_table(name, project)
else:
try:
self._core_service.DeleteFeatureTable(
DeleteFeatureTableRequest(project=project,
name=name.strip()),
metadata=self._get_grpc_metadata(),
)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
def list_features_by_ref(
self,
project: str = None,
entities: List[str] = list(),
labels: Dict[str, str] = dict(),
) -> Dict[FeatureRef, Feature]:
"""
Retrieve a dictionary of feature reference to feature from Feast Core based on filters provided.
Args:
project: Feast project that these features belongs to
entities: Feast entity that these features are associated with
labels: Feast labels that these features are associated with
Returns:
Dictionary of <feature references: features>
Examples:
>>> from feast import Client
>>>
>>> feast_client = Client(core_url="localhost:6565")
>>> features = feast_client.list_features(project="test_project", entities=["driver_id"], labels={"key1":"val1","key2":"val2"})
>>> print(features)
"""
if self._use_object_store_registry:
raise NotImplementedError(
"This function is not implemented for object store registry.")
else:
if project is None:
project = self.project
filter = ListFeaturesRequest.Filter(project=project,
entities=entities,
labels=labels)
feature_protos = self._core_service.ListFeatures(
ListFeaturesRequest(filter=filter),
metadata=self._get_grpc_metadata(),
) # type: ListFeaturesResponse
# Extract features and return
features_dict = {}
for ref_str, feature_proto in feature_protos.features.items():
feature_ref = FeatureRef.from_str(ref_str)
feature = Feature.from_proto(feature_proto)
features_dict[feature_ref] = feature
return features_dict
def ingest(
self,
feature_table: Union[str, FeatureTable],
source: Union[pd.DataFrame, str],
project: str = None,
chunk_size: int = 10000,
max_workers: int = max(CPU_COUNT - 1, 1),
timeout: int = int(opt().BATCH_INGESTION_PRODUCTION_TIMEOUT),
) -> None:
"""
Batch load feature data into a FeatureTable.
Args:
feature_table (typing.Union[str, feast.feature_table.FeatureTable]):
FeatureTable object or the string name of the feature table
source (typing.Union[pd.DataFrame, str]):
Either a file path or Pandas Dataframe to ingest into Feast
Files that are currently supported:
* parquet
* csv
* json
project: Feast project to locate FeatureTable
chunk_size (int):
Amount of rows to load and ingest at a time.
max_workers (int):
Number of worker processes to use to encode values.
timeout (int):
Timeout in seconds to wait for completion.
Examples:
>>> from feast import Client
>>>
>>> client = Client(core_url="localhost:6565")
>>> ft_df = pd.DataFrame(
>>> {
>>> "datetime": [pd.datetime.now()],
>>> "driver": [1001],
>>> "rating": [4.3],
>>> }
>>> )
>>> client.set_project("project1")
>>>
>>> driver_ft = client.get_feature_table("driver")
>>> client.ingest(driver_ft, ft_df)
"""
if self._telemetry_enabled:
log_usage(
"ingest",
self._telemetry_id,
datetime.utcnow(),
self.version(sdk_only=True),
)
if project is None:
project = self.project
if isinstance(feature_table, str):
name = feature_table
if isinstance(feature_table, FeatureTable):
name = feature_table.name
fetched_feature_table: Optional[FeatureTable] = self.get_feature_table(
name, project)
if fetched_feature_table is not None:
feature_table = fetched_feature_table
else:
raise Exception(f"FeatureTable, {name} cannot be found.")
# Check 1) Only parquet file format for FeatureTable batch source is supported
if (feature_table.batch_source
and issubclass(type(feature_table.batch_source), FileSource)
and isinstance(
type(feature_table.batch_source.file_options.file_format),
ParquetFormat)):
raise Exception(
f"No suitable batch source found for FeatureTable, {name}."
f"Only BATCH_FILE source with parquet format is supported for batch ingestion."
)
pyarrow_table, column_names = _read_table_from_source(source)
# Check 2) Check if FeatureTable batch source field mappings can be found in provided source table
_check_field_mappings(
column_names,
name,
feature_table.batch_source.event_timestamp_column,
feature_table.batch_source.field_mapping,
)
dir_path = None
with_partitions = False
if (issubclass(type(feature_table.batch_source), FileSource)
and feature_table.batch_source.date_partition_column):
with_partitions = True
dest_path = _write_partitioned_table_from_source(
column_names,
pyarrow_table,
feature_table.batch_source.date_partition_column,
feature_table.batch_source.event_timestamp_column,
)
else:
dir_path, dest_path = _write_non_partitioned_table_from_source(
column_names,
pyarrow_table,
chunk_size,
max_workers,
)
try:
if issubclass(type(feature_table.batch_source), FileSource):
file_url = feature_table.batch_source.file_options.file_url.rstrip(
"*")
_upload_to_file_source(file_url, with_partitions, dest_path,
self._config)
if issubclass(type(feature_table.batch_source), BigQuerySource):
bq_table_ref = feature_table.batch_source.bigquery_options.table_ref
feature_table_timestamp_column = (
feature_table.batch_source.event_timestamp_column)
_upload_to_bq_source(bq_table_ref,
feature_table_timestamp_column, dest_path)
finally:
# Remove parquet file(s) that were created earlier
print("Removing temporary file(s)...")
if dir_path:
shutil.rmtree(dir_path)
print(
"Data has been successfully ingested into FeatureTable batch source."
)
def _get_grpc_metadata(self):
"""
Returns a metadata tuple to attach to gRPC requests. This is primarily
used when authentication is enabled but SSL/TLS is disabled.
Returns: Tuple of metadata to attach to each gRPC call
"""
if self._config.getboolean(opt.ENABLE_AUTH) and self._auth_metadata:
return self._auth_metadata.get_signed_meta()
return ()
def get_online_features(
self,
feature_refs: List[str],
entity_rows: List[Dict[str, Any]],
project: Optional[str] = None,
) -> OnlineResponse:
"""
Retrieves the latest online feature data from Feast Serving.
Args:
feature_refs: List of feature references that will be returned for each entity.
Each feature reference should have the following format:
"feature_table:feature" where "feature_table" & "feature" refer to
the feature and feature table names respectively.
Only the feature name is required.
entity_rows: A list of dictionaries where each key-value is an entity-name, entity-value pair.
project: Optionally specify the the project override. If specified, uses given project for retrieval.
Overrides the projects specified in Feature References if also are specified.
Returns:
GetOnlineFeaturesResponse containing the feature data in records.
Each EntityRow provided will yield one record, which contains
data fields with data value and field status metadata (if included).
Examples:
>>> from feast import Client
>>>
>>> feast_client = Client(core_url="localhost:6565", serving_url="localhost:6566")
>>> feature_refs = ["sales:daily_transactions"]
>>> entity_rows = [{"customer_id": 0},{"customer_id": 1}]
>>>
>>> online_response = feast_client.get_online_features(
>>> feature_refs, entity_rows, project="my_project")
>>> online_response_dict = online_response.to_dict()
>>> print(online_response_dict)
{'sales:daily_transactions': [1.1,1.2], 'sales:customer_id': [0,1]}
"""
if self._telemetry_enabled:
if self._telemetry_counter["get_online_features"] % 1000 == 0:
log_usage(
"get_online_features",
self._telemetry_id,
datetime.utcnow(),
self.version(sdk_only=True),
)
self._telemetry_counter["get_online_features"] += 1
try:
response = self._serving_service.GetOnlineFeaturesV2(
GetOnlineFeaturesRequestV2(
features=_build_feature_references(
feature_ref_strs=feature_refs),
entity_rows=_infer_online_entity_rows(entity_rows),
project=project if project is not None else self.project,
),
timeout=self._config.getint(opt.GRPC_CONNECTION_TIMEOUT),
metadata=self._get_grpc_metadata(),
)
except grpc.RpcError as e:
raise grpc.RpcError(e.details())
response = OnlineResponse(response)
return response
