def configure_metadata_service(existing_env):
empty_profile = False
if not existing_env:
empty_profile = True
env = {}
# Set Metadata Service as default.
env["METAFLOW_DEFAULT_METADATA"] = "service"
# Set URL for the Metadata Service.
env["METAFLOW_SERVICE_URL"] = click.prompt(
cyan("[METAFLOW_SERVICE_URL]") + " URL for Metaflow Service.",
default=existing_env.get("METAFLOW_SERVICE_URL"),
show_default=True,
)
# Set internal URL for the Metadata Service.
env["METAFLOW_SERVICE_INTERNAL_URL"] = click.prompt(
cyan("[METAFLOW_SERVICE_INTERNAL_URL]") + yellow(" (optional)") +
" URL for Metaflow Service " + "(Accessible only within VPC).",
default=existing_env.get("METAFLOW_SERVICE_INTERNAL_URL",
env["METAFLOW_SERVICE_URL"]),
show_default=True,
)
# Set Auth Key for the Metadata Service.
env["METAFLOW_SERVICE_AUTH_KEY"] = click.prompt(
cyan("[METAFLOW_SERVICE_AUTH_KEY]") + yellow(" (optional)") +
" Auth Key for Metaflow Service.",
default=existing_env.get("METAFLOW_SERVICE_AUTH_KEY", ""),
show_default=True,
)
return env
def configure_eks(existing_env):
empty_profile = False
if not existing_env:
empty_profile = True
env = {}
# Set K8S Namespace
env["METAFLOW_KUBERNETES_NAMESPACE"] = click.prompt(
cyan("[METAFLOW_KUBERNETES_NAMESPACE]") + yellow(" (optional)") +
" Kubernetes Namespace ",
default="default",
show_default=True,
)
# Set K8S SA
env["METAFLOW_KUBERNETES_SERVICE_ACCOUNT"] = click.prompt(
cyan("[METAFLOW_KUBERNETES_SERVICE_ACCOUNT]") + yellow(" (optional)") +
" Kubernetes Service Account ",
default="default",
show_default=True,
)
# Set default Docker repository for K8S jobs.
env["METAFLOW_KUBERNETES_CONTAINER_REGISTRY"] = click.prompt(
cyan("[METAFLOW_KUBERNETES_CONTAINER_REGISTRY]") +
yellow(" (optional)") + " Default Docker image repository for K8S " +
"jobs. If nothing is specified, " + "dockerhub (hub.docker.com/) is " +
"used as default.",
default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_REGISTRY", ""),
show_default=True,
)
# Set default Docker image for K8S jobs.
env["METAFLOW_KUBERNETES_CONTAINER_IMAGE"] = click.prompt(
cyan("[METAFLOW_KUBERNETES_CONTAINER_IMAGE]") + yellow(" (optional)") +
" Default Docker image for K8S jobs. " +
"If nothing is specified, an appropriate " +
"python image is used as default.",
default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_IMAGE", ""),
show_default=True,
)
return env
def configure_s3_datastore(existing_env):
env = {}
# Set Amazon S3 as default datastore.
env["METAFLOW_DEFAULT_DATASTORE"] = "s3"
# Set Amazon S3 folder for datastore.
env["METAFLOW_DATASTORE_SYSROOT_S3"] = click.prompt(
cyan("[METAFLOW_DATASTORE_SYSROOT_S3]") +
" Amazon S3 folder for Metaflow artifact storage " +
"(s3://<bucket>/<prefix>).",
default=existing_env.get("METAFLOW_DATASTORE_SYSROOT_S3"),
show_default=True,
)
# Set Amazon S3 folder for datatools.
env["METAFLOW_DATATOOLS_SYSROOT_S3"] = click.prompt(
cyan("[METAFLOW_DATATOOLS_SYSROOT_S3]") + yellow(" (optional)") +
" Amazon S3 folder for Metaflow datatools " +
"(s3://<bucket>/<prefix>).",
default=existing_env.get(
"METAFLOW_DATATOOLS_SYSROOT_S3",
os.path.join(env["METAFLOW_DATASTORE_SYSROOT_S3"], "data"),
),
show_default=True,
)
return env
def sandbox(profile):
overwrite_config(profile)
# Prompt for user input.
encoded_str = click.prompt(
"Following instructions from "
"https://metaflow.org/sandbox, "
"please paste the encoded magic string",
type=str,
)
# Decode the bytes to env_dict.
try:
import base64, zlib
from metaflow.util import to_bytes
env_dict = json.loads(
to_unicode(zlib.decompress(base64.b64decode(
to_bytes(encoded_str)))))
except:
# TODO: Add the URL for contact us page in the error?
raise click.BadArgumentUsage("Could not decode the sandbox "
"configuration. Please contact us.")
# Persist to a file.
persist_env(env_dict, profile)
def configure_aws_batch(existing_env):
empty_profile = False
if not existing_env:
empty_profile = True
env = {}
# Set AWS Batch Job Queue.
env["METAFLOW_BATCH_JOB_QUEUE"] = click.prompt(
cyan("[METAFLOW_BATCH_JOB_QUEUE]") + " AWS Batch Job Queue.",
default=existing_env.get("METAFLOW_BATCH_JOB_QUEUE"),
show_default=True,
)
# Set IAM role for AWS Batch jobs to assume.
env["METAFLOW_ECS_S3_ACCESS_IAM_ROLE"] = click.prompt(
cyan("[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]") +
" IAM role for AWS Batch jobs to access AWS " +
"resources (Amazon S3 etc.).",
default=existing_env.get("METAFLOW_ECS_S3_ACCESS_IAM_ROLE"),
show_default=True,
)
# Set default Docker repository for AWS Batch jobs.
env["METAFLOW_BATCH_CONTAINER_REGISTRY"] = click.prompt(
cyan("[METAFLOW_BATCH_CONTAINER_REGISTRY]") + yellow(" (optional)") +
" Default Docker image repository for AWS " +
"Batch jobs. If nothing is specified, " +
"dockerhub (hub.docker.com/) is " + "used as default.",
default=existing_env.get("METAFLOW_BATCH_CONTAINER_REGISTRY", ""),
show_default=True,
)
# Set default Docker image for AWS Batch jobs.
env["METAFLOW_BATCH_CONTAINER_IMAGE"] = click.prompt(
cyan("[METAFLOW_BATCH_CONTAINER_IMAGE]") + yellow(" (optional)") +
" Default Docker image for AWS Batch jobs. " +
"If nothing is specified, an appropriate " +
"python image is used as default.",
default=existing_env.get("METAFLOW_BATCH_CONTAINER_IMAGE", ""),
show_default=True,
)
# Configure AWS Step Functions for scheduling.
if click.confirm(
"\nMetaflow can " + yellow("schedule your flows on AWS Step "
"Functions") +
" and trigger them at a specific cadence using "
"Amazon EventBridge.\nTo support flows involving "
"foreach steps, you would need access to AWS "
"DynamoDB.\nWould you like to configure AWS Step "
"Functions for scheduling?",
default=empty_profile or "METAFLOW_SFN_IAM_ROLE" in existing_env,
abort=False,
):
# Configure IAM role for AWS Step Functions.
env["METAFLOW_SFN_IAM_ROLE"] = click.prompt(
cyan("[METAFLOW_SFN_IAM_ROLE]") +
" IAM role for AWS Step Functions to " +
"access AWS resources (AWS Batch, " + "AWS DynamoDB).",
default=existing_env.get("METAFLOW_SFN_IAM_ROLE"),
show_default=True,
)
# Configure IAM role for AWS Events Bridge.
env["METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"] = click.prompt(
cyan("[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]") +
" IAM role for Amazon EventBridge to " +
"access AWS Step Functions.",
default=existing_env.get("METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"),
show_default=True,
)
# Configure AWS DynamoDB Table for AWS Step Functions.
env["METAFLOW_SFN_DYNAMO_DB_TABLE"] = click.prompt(
cyan("[METAFLOW_SFN_DYNAMO_DB_TABLE]") +
" AWS DynamoDB table name for tracking " +
"AWS Step Functions execution metadata.",
default=existing_env.get("METAFLOW_SFN_DYNAMO_DB_TABLE"),
show_default=True,
)
return env
