def _get_docker_image_version() -> str:
"""Returns a version for the feature server Docker image.
If the feast.constants.DOCKER_IMAGE_TAG_ENV_NAME environment variable is set,
we return that (mostly used for integration tests, but can be used for local testing too).
For public Feast releases this equals to the Feast SDK version modified by replacing "." with "_".
For example, Feast SDK version "0.14.1" would correspond to Docker image version "0_14_1".
During development (when Feast is installed in editable mode) this equals to the Feast SDK version
modified by removing the "dev..." suffix and replacing "." with "_". For example, Feast SDK version
"0.14.1.dev41+g1cbfa225.d20211103" would correspond to Docker image version "0_14_1". This way,
Feast SDK will use an already existing Docker image built during the previous public release.
"""
tag = os.environ.get(DOCKER_IMAGE_TAG_ENV_NAME)
if tag is not None:
return tag
else:
version = get_version()
if "dev" in version:
version = version[: version.find("dev") - 1].replace(".", "_")
_logger.warning(
"You are trying to use AWS Lambda feature server while Feast is in a development mode. "
f"Feast will use a docker image version {version} derived from Feast SDK "
f"version {get_version()}. If you want to update the Feast SDK version, make "
"sure to first fetch all new release tags from Github and then reinstall the library:\n"
"> git fetch --all --tags\n"
"> pip install -e sdk/python"
)
else:
version = version.replace(".", "_")
return version
def log(self, function_name: str):
self.check_env_and_configure()
if self._telemetry_enabled and self.telemetry_id:
if function_name == "get_online_features":
if self._telemetry_counter["get_online_features"] % 10000 != 0:
self._telemetry_counter["get_online_features"] += 1
return
json = {
"function_name": function_name,
"telemetry_id": self.telemetry_id,
"timestamp": datetime.utcnow().isoformat(),
"version": get_version(),
"os": sys.platform,
"is_test": self._is_test,
}
try:
requests.post(TELEMETRY_ENDPOINT, json=json)
except Exception as e:
if self._is_test:
raise e
else:
pass
return
def log(self, function_name: str):
self.check_env_and_configure()
if self._usage_enabled and self.usage_id:
if function_name == "get_online_features":
self._usage_counter["get_online_features"] += 1
if self._usage_counter["get_online_features"] % 10000 != 2:
return
self._usage_counter[
"get_online_features"] = 2 # avoid overflow
json = {
"function_name": function_name,
"usage_id": self.usage_id,
"timestamp": datetime.utcnow().isoformat(),
"version": get_version(),
"os": sys.platform,
"is_test": self._is_test,
}
try:
requests.post(USAGE_ENDPOINT, json=json)
except Exception as e:
if self._is_test:
raise e
else:
pass
return
def log_function(self, function_name: str):
self.check_env_and_configure()
if self._usage_enabled and self.usage_id:
if (function_name == "get_online_features"
and not self.should_log_for_get_online_features_event(
function_name)):
return
json = {
"function_name": function_name,
"usage_id": self.usage_id,
"timestamp": datetime.utcnow().isoformat(),
"version": get_version(),
"os": sys.platform,
"is_test": self._is_test,
}
self._send_usage_request(json)
def log_event(self, event: UsageEvent):
self.check_env_and_configure()
if self._usage_enabled and self.usage_id:
event_name = str(event)
if (event == UsageEvent.GET_ONLINE_FEATURES_WITH_ODFV
and not self.should_log_for_get_online_features_event(
event_name)):
return
json = {
"event_name": event_name,
"usage_id": self.usage_id,
"timestamp": datetime.utcnow().isoformat(),
"version": get_version(),
"os": sys.platform,
"is_test": self._is_test,
}
self._send_usage_request(json)
def log_exception(self, error_type: str, traceback: List[Tuple[str, int, str]]):
self.check_env_and_configure()
if self._telemetry_enabled and self.telemetry_id:
json = {
"error_type": error_type,
"traceback": traceback,
"telemetry_id": self.telemetry_id,
"version": get_version(),
"os": sys.platform,
"is_test": self._is_test,
}
try:
requests.post(TELEMETRY_ENDPOINT, json=json)
except Exception as e:
if self._is_test:
raise e
else:
pass
return
def version(self) -> str:
"""Returns the version of the current Feast SDK/CLI"""
return get_version()
USAGE_ENDPOINT = "https://usage.feast.dev"
_logger = logging.getLogger(__name__)
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
_is_enabled = os.getenv(FEAST_USAGE,
default=DEFAULT_FEAST_USAGE_VALUE) == "True"
_constant_attributes = {
"session_id":
str(uuid.uuid4()),
"installation_id":
None,
"version":
get_version(),
"python_version":
platform.python_version(),
"platform":
platform.platform(),
"env_signature":
hashlib.md5(",".join(
sorted([k for k in os.environ.keys()
if not k.startswith("FEAST")])).encode()).hexdigest(),
}
@dataclasses.dataclass
class FnCall:
fn_name: str
id: str
def _get_version_for_aws():
"""Returns Feast version with certain characters replaced.
This allows the version to be included in names for AWS resources.
"""
return get_version().replace(".", "_").replace("+", "_")
def _deploy_feature_server(self, project: str, image_uri: str):
_logger.info("Deploying feature server...")
if not self.repo_config.repo_path:
raise RepoConfigPathDoesNotExist()
with open(self.repo_config.repo_path / "feature_store.yaml",
"rb") as f:
config_bytes = f.read()
config_base64 = base64.b64encode(config_bytes).decode()
resource_name = _get_lambda_name(project)
lambda_client = boto3.client("lambda")
api_gateway_client = boto3.client("apigatewayv2")
function = aws_utils.get_lambda_function(lambda_client, resource_name)
if function is None:
# If the Lambda function does not exist, create it.
_logger.info(" Creating AWS Lambda...")
assert isinstance(self.repo_config.feature_server,
AwsLambdaFeatureServerConfig)
lambda_client.create_function(
FunctionName=resource_name,
Role=self.repo_config.feature_server.execution_role_name,
Code={"ImageUri": image_uri},
PackageType="Image",
MemorySize=1769,
Environment={
"Variables": {
FEATURE_STORE_YAML_ENV_NAME: config_base64,
FEAST_USAGE: "False",
}
},
Tags={
"feast-owned": "True",
"project": project,
"feast-sdk-version": get_version(),
},
)
function = aws_utils.get_lambda_function(lambda_client,
resource_name)
if not function:
raise AwsLambdaDoesNotExist(resource_name)
else:
# If the feature_store.yaml has changed, need to update the environment variable.
env = function.get("Environment", {}).get("Variables", {})
if env.get(FEATURE_STORE_YAML_ENV_NAME) != config_base64:
# Note, that this does not update Lambda gracefully (e.g. no rolling deployment).
# It's expected that feature_store.yaml is not regularly updated while the lambda
# is serving production traffic. However, the update in registry (e.g. modifying
# feature views, feature services, and other definitions does not update lambda).
_logger.info(" Updating AWS Lambda...")
lambda_client.update_function_configuration(
FunctionName=resource_name,
Environment={
"Variables": {
FEATURE_STORE_YAML_ENV_NAME: config_base64
}
},
)
api = aws_utils.get_first_api_gateway(api_gateway_client,
resource_name)
if not api:
# If the API Gateway doesn't exist, create it
_logger.info(" Creating AWS API Gateway...")
api = api_gateway_client.create_api(
Name=resource_name,
ProtocolType="HTTP",
Target=function["FunctionArn"],
RouteKey="POST /get-online-features",
Tags={
"feast-owned": "True",
"project": project,
"feast-sdk-version": get_version(),
},
)
if not api:
raise AwsAPIGatewayDoesNotExist(resource_name)
# Make sure to give AWS Lambda a permission to be invoked by the newly created API Gateway
api_id = api["ApiId"]
region = lambda_client.meta.region_name
account_id = aws_utils.get_account_id()
lambda_client.add_permission(
FunctionName=function["FunctionArn"],
StatementId=str(uuid.uuid4()),
Action="lambda:InvokeFunction",
Principal="apigateway.amazonaws.com",
SourceArn=
f"arn:aws:execute-api:{region}:{account_id}:{api_id}/*/*/get-online-features",
)
import requests
from feast.constants import FEAST_USAGE
from feast.version import get_version
USAGE_ENDPOINT = "https://usage.feast.dev"
_logger = logging.getLogger(__name__)
_executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
_is_enabled = os.getenv(FEAST_USAGE, default="True") == "True"
_constant_attributes = {
"session_id": str(uuid.uuid4()),
"installation_id": None,
"version": get_version(),
"python_version": platform.python_version(),
"platform": platform.platform(),
"env_signature": hashlib.md5(
",".join(
sorted([k for k in os.environ.keys() if not k.startswith("FEAST")])
).encode()
).hexdigest(),
}
@dataclasses.dataclass
class FnCall:
fn_name: str
id: str
def update_infra(
self,
project: str,
tables_to_delete: Sequence[Union[FeatureTable, FeatureView]],
tables_to_keep: Sequence[Union[FeatureTable, FeatureView]],
entities_to_delete: Sequence[Entity],
entities_to_keep: Sequence[Entity],
partial: bool,
):
self.online_store.update(
config=self.repo_config,
tables_to_delete=tables_to_delete,
tables_to_keep=tables_to_keep,
entities_to_keep=entities_to_keep,
entities_to_delete=entities_to_delete,
partial=partial,
)
if self.repo_config.feature_server and self.repo_config.feature_server.enabled:
if not enable_aws_lambda_feature_server(self.repo_config):
raise ExperimentalFeatureNotEnabled(FLAG_AWS_LAMBDA_FEATURE_SERVER_NAME)
# Since the AWS Lambda feature server will attempt to load the registry, we
# only allow the registry to be in S3.
registry_path = (
self.repo_config.registry
if isinstance(self.repo_config.registry, str)
else self.repo_config.registry.path
)
registry_store_class = get_registry_store_class_from_scheme(registry_path)
if registry_store_class != S3RegistryStore:
raise IncompatibleRegistryStoreClass(
registry_store_class.__name__, S3RegistryStore.__name__
)
image_uri = self._upload_docker_image(project)
_logger.info("Deploying feature server...")
if not self.repo_config.repo_path:
raise RepoConfigPathDoesNotExist()
with open(self.repo_config.repo_path / "feature_store.yaml", "rb") as f:
config_bytes = f.read()
config_base64 = base64.b64encode(config_bytes).decode()
resource_name = self._get_lambda_name(project)
lambda_client = boto3.client("lambda")
api_gateway_client = boto3.client("apigatewayv2")
function = aws_utils.get_lambda_function(lambda_client, resource_name)
if function is None:
# If the Lambda function does not exist, create it.
_logger.info(" Creating AWS Lambda...")
lambda_client.create_function(
FunctionName=resource_name,
Role=self.repo_config.feature_server.execution_role_name,
Code={"ImageUri": image_uri},
PackageType="Image",
MemorySize=1769,
Environment={
"Variables": {
FEATURE_STORE_YAML_ENV_NAME: config_base64,
FEAST_USAGE: "False",
}
},
Tags={
"feast-owned": "True",
"project": project,
"feast-sdk-version": get_version(),
},
)
function = aws_utils.get_lambda_function(lambda_client, resource_name)
if not function:
raise AwsLambdaDoesNotExist(resource_name)
else:
# If the feature_store.yaml has changed, need to update the environment variable.
env = function.get("Environment", {}).get("Variables", {})
if env.get(FEATURE_STORE_YAML_ENV_NAME) != config_base64:
# Note, that this does not update Lambda gracefully (e.g. no rolling deployment).
# It's expected that feature_store.yaml is not regularly updated while the lambda
# is serving production traffic. However, the update in registry (e.g. modifying
# feature views, feature services, and other definitions does not update lambda).
_logger.info(" Updating AWS Lambda...")
lambda_client.update_function_configuration(
FunctionName=resource_name,
Environment={
"Variables": {FEATURE_STORE_YAML_ENV_NAME: config_base64}
},
)
api = aws_utils.get_first_api_gateway(api_gateway_client, resource_name)
if not api:
# If the API Gateway doesn't exist, create it
_logger.info(" Creating AWS API Gateway...")
api = api_gateway_client.create_api(
Name=resource_name,
ProtocolType="HTTP",
Target=function["FunctionArn"],
RouteKey="POST /get-online-features",
Tags={
"feast-owned": "True",
"project": project,
"feast-sdk-version": get_version(),
},
)
if not api:
raise AwsAPIGatewayDoesNotExist(resource_name)
# Make sure to give AWS Lambda a permission to be invoked by the newly created API Gateway
api_id = api["ApiId"]
region = lambda_client.meta.region_name
account_id = aws_utils.get_account_id()
lambda_client.add_permission(
FunctionName=function["FunctionArn"],
StatementId=str(uuid.uuid4()),
Action="lambda:InvokeFunction",
Principal="apigateway.amazonaws.com",
SourceArn=f"arn:aws:execute-api:{region}:{account_id}:{api_id}/*/*/get-online-features",
)
