def get_registry_proto(self):
file_obj = TemporaryFile()
registry_proto = RegistryProto()
try:
from botocore.exceptions import ClientError
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("aws", str(e))
try:
bucket = self.s3_client.Bucket(self._bucket)
self.s3_client.meta.client.head_bucket(Bucket=bucket.name)
except ClientError as e:
# If a client error is thrown, then check that it was a 404 error.
# If it was a 404 error, then the bucket does not exist.
error_code = int(e.response["Error"]["Code"])
if error_code == 404:
raise S3RegistryBucketNotExist(self._bucket)
else:
raise S3RegistryBucketForbiddenAccess(self._bucket) from e
try:
obj = bucket.Object(self._key)
obj.download_fileobj(file_obj)
file_obj.seek(0)
registry_proto.ParseFromString(file_obj.read())
return registry_proto
except ClientError as e:
raise FileNotFoundError(
f"Error while trying to locate Registry at path {self._uri.geturl()}"
) from e
def _upload_docker_image(self, ecr_client, repository_uri: str,
docker_image_version: str) -> str:
"""
Pulls the AWS Lambda docker image from Dockerhub and uploads it to AWS ECR.
Returns:
The URI of the uploaded docker image.
"""
try:
import docker
from docker.errors import APIError
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("docker", str(e))
try:
docker_client = docker.from_env()
except APIError:
from feast.errors import DockerDaemonNotRunning
raise DockerDaemonNotRunning()
dockerhub_image = f"{AWS_LAMBDA_FEATURE_SERVER_IMAGE}:{docker_image_version}"
_logger.info(
f"Pulling remote image {Style.BRIGHT + Fore.GREEN}{dockerhub_image}{Style.RESET_ALL}"
)
for line in docker_client.api.pull(dockerhub_image,
stream=True,
decode=True):
_logger.debug(f" {line}")
auth_token = ecr_client.get_authorization_token(
)["authorizationData"][0]["authorizationToken"]
username, password = base64.b64decode(auth_token).decode(
"utf-8").split(":")
ecr_address = repository_uri.split("/")[0]
_logger.info(
f"Logging in Docker client to {Style.BRIGHT + Fore.GREEN}{ecr_address}{Style.RESET_ALL}"
)
login_status = docker_client.login(username=username,
password=password,
registry=ecr_address)
_logger.debug(f" {login_status}")
image = docker_client.images.get(dockerhub_image)
image_remote_name = f"{repository_uri}:{docker_image_version}"
_logger.info(
f"Pushing local image to remote {Style.BRIGHT + Fore.GREEN}{image_remote_name}{Style.RESET_ALL}"
)
image.tag(image_remote_name)
for line in docker_client.api.push(repository_uri,
tag=docker_image_version,
stream=True,
decode=True):
_logger.debug(f" {line}")
return image_remote_name
def __init__(self, uri: str):
try:
from google.cloud import storage
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("gcp", str(e))
self.gcs_client = storage.Client()
self._uri = urlparse(uri)
self._bucket = self._uri.hostname
self._blob = self._uri.path.lstrip("/")
def __init__(self, registry_config: RegistryConfig, repo_path: Path):
uri = registry_config.path
try:
from google.cloud import storage
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("gcp", str(e))
self.gcs_client = storage.Client()
self._uri = urlparse(uri)
self._bucket = self._uri.hostname
self._blob = self._uri.path.lstrip("/")
def __init__(self, uri: str):
try:
import boto3
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("aws", str(e))
self._uri = urlparse(uri)
self._bucket = self._uri.hostname
self._key = self._uri.path.lstrip("/")
self.s3_client = boto3.resource(
"s3", endpoint_url=os.environ.get("FEAST_S3_ENDPOINT_URL"))
from feast.infra.online_stores.online_store import OnlineStore
from feast.protos.feast.core.DynamoDBTable_pb2 import (
DynamoDBTable as DynamoDBTableProto, )
from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto
from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto
from feast.protos.feast.types.Value_pb2 import Value as ValueProto
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.usage import log_exceptions_and_usage, tracing_span
try:
import boto3
from botocore.exceptions import ClientError
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("aws", str(e))
logger = logging.getLogger(__name__)
class DynamoDBOnlineStoreConfig(FeastConfigBaseModel):
"""Online store config for DynamoDB store"""
type: Literal["dynamodb"] = "dynamodb"
"""Online store type selector"""
region: StrictStr
""" AWS Region Name """
class DynamoDBOnlineStore(OnlineStore):
from feast.infra.utils.snowflake_utils import (
execute_snowflake_statement,
get_snowflake_conn,
write_pandas,
)
from feast.registry import Registry
from feast.repo_config import FeastConfigBaseModel, RepoConfig
from feast.saved_dataset import SavedDatasetStorage
from feast.usage import log_exceptions_and_usage
try:
from snowflake.connector import SnowflakeConnection
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("snowflake", str(e))
class SnowflakeOfflineStoreConfig(FeastConfigBaseModel):
"""Offline store config for Snowflake"""
type: Literal["snowflake.offline"] = "snowflake.offline"
""" Offline store type selector"""
config_path: Optional[str] = (Path(os.environ["HOME"]) /
".snowsql/config").__str__()
""" Snowflake config path -- absolute path required (Cant use ~)"""
account: Optional[str] = None
""" Snowflake deployment identifier -- drop .snowflakecomputing.com"""
def _upload_docker_image(self, project: str) -> str:
"""
Pulls the AWS Lambda docker image from Dockerhub and uploads it to AWS ECR.
Args:
project: Feast project name
Returns:
The URI of the uploaded docker image.
"""
import base64
try:
import boto3
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("aws", str(e))
try:
import docker
from docker.errors import APIError
except ImportError as e:
from feast.errors import FeastExtrasDependencyImportError
raise FeastExtrasDependencyImportError("docker", str(e))
try:
docker_client = docker.from_env()
except APIError:
from feast.errors import DockerDaemonNotRunning
raise DockerDaemonNotRunning()
_logger.info(
f"Pulling remote image {Style.BRIGHT + Fore.GREEN}{AWS_LAMBDA_FEATURE_SERVER_IMAGE}{Style.RESET_ALL}:"
)
docker_client.images.pull(AWS_LAMBDA_FEATURE_SERVER_IMAGE)
version = self._get_version_for_aws()
repository_name = f"feast-python-server-{project}-{version}"
ecr_client = boto3.client("ecr")
try:
_logger.info(
f"Creating remote ECR repository {Style.BRIGHT + Fore.GREEN}{repository_name}{Style.RESET_ALL}:"
)
response = ecr_client.create_repository(repositoryName=repository_name)
repository_uri = response["repository"]["repositoryUri"]
except ecr_client.exceptions.RepositoryAlreadyExistsException:
response = ecr_client.describe_repositories(
repositoryNames=[repository_name]
)
repository_uri = response["repositories"][0]["repositoryUri"]
auth_token = ecr_client.get_authorization_token()["authorizationData"][0][
"authorizationToken"
]
username, password = base64.b64decode(auth_token).decode("utf-8").split(":")
ecr_address = repository_uri.split("/")[0]
docker_client.login(username=username, password=password, registry=ecr_address)
image = docker_client.images.get(AWS_LAMBDA_FEATURE_SERVER_IMAGE)
image_remote_name = f"{repository_uri}:{version}"
_logger.info(
f"Pushing local image to remote {Style.BRIGHT + Fore.GREEN}{image_remote_name}{Style.RESET_ALL}:"
)
image.tag(image_remote_name)
docker_client.api.push(repository_uri, tag=version)
return image_remote_name
