def configure_metadata_service(existing_env): empty_profile = False if not existing_env: empty_profile = True env = {} # Set Metadata Service as default. env["METAFLOW_DEFAULT_METADATA"] = "service" # Set URL for the Metadata Service. env["METAFLOW_SERVICE_URL"] = click.prompt( cyan("[METAFLOW_SERVICE_URL]") + " URL for Metaflow Service.", default=existing_env.get("METAFLOW_SERVICE_URL"), show_default=True, ) # Set internal URL for the Metadata Service. env["METAFLOW_SERVICE_INTERNAL_URL"] = click.prompt( cyan("[METAFLOW_SERVICE_INTERNAL_URL]") + yellow(" (optional)") + " URL for Metaflow Service " + "(Accessible only within VPC).", default=existing_env.get("METAFLOW_SERVICE_INTERNAL_URL", env["METAFLOW_SERVICE_URL"]), show_default=True, ) # Set Auth Key for the Metadata Service. env["METAFLOW_SERVICE_AUTH_KEY"] = click.prompt( cyan("[METAFLOW_SERVICE_AUTH_KEY]") + yellow(" (optional)") + " Auth Key for Metaflow Service.", default=existing_env.get("METAFLOW_SERVICE_AUTH_KEY", ""), show_default=True, ) return env
def configure_eks(existing_env): empty_profile = False if not existing_env: empty_profile = True env = {} # Set K8S Namespace env["METAFLOW_KUBERNETES_NAMESPACE"] = click.prompt( cyan("[METAFLOW_KUBERNETES_NAMESPACE]") + yellow(" (optional)") + " Kubernetes Namespace ", default="default", show_default=True, ) # Set K8S SA env["METAFLOW_KUBERNETES_SERVICE_ACCOUNT"] = click.prompt( cyan("[METAFLOW_KUBERNETES_SERVICE_ACCOUNT]") + yellow(" (optional)") + " Kubernetes Service Account ", default="default", show_default=True, ) # Set default Docker repository for K8S jobs. env["METAFLOW_KUBERNETES_CONTAINER_REGISTRY"] = click.prompt( cyan("[METAFLOW_KUBERNETES_CONTAINER_REGISTRY]") + yellow(" (optional)") + " Default Docker image repository for K8S " + "jobs. If nothing is specified, " + "dockerhub (hub.docker.com/) is " + "used as default.", default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_REGISTRY", ""), show_default=True, ) # Set default Docker image for K8S jobs. env["METAFLOW_KUBERNETES_CONTAINER_IMAGE"] = click.prompt( cyan("[METAFLOW_KUBERNETES_CONTAINER_IMAGE]") + yellow(" (optional)") + " Default Docker image for K8S jobs. " + "If nothing is specified, an appropriate " + "python image is used as default.", default=existing_env.get("METAFLOW_KUBERNETES_CONTAINER_IMAGE", ""), show_default=True, ) return env
def configure_s3_datastore(existing_env): env = {} # Set Amazon S3 as default datastore. env["METAFLOW_DEFAULT_DATASTORE"] = "s3" # Set Amazon S3 folder for datastore. env["METAFLOW_DATASTORE_SYSROOT_S3"] = click.prompt( cyan("[METAFLOW_DATASTORE_SYSROOT_S3]") + " Amazon S3 folder for Metaflow artifact storage " + "(s3://<bucket>/<prefix>).", default=existing_env.get("METAFLOW_DATASTORE_SYSROOT_S3"), show_default=True, ) # Set Amazon S3 folder for datatools. env["METAFLOW_DATATOOLS_SYSROOT_S3"] = click.prompt( cyan("[METAFLOW_DATATOOLS_SYSROOT_S3]") + yellow(" (optional)") + " Amazon S3 folder for Metaflow datatools " + "(s3://<bucket>/<prefix>).", default=existing_env.get( "METAFLOW_DATATOOLS_SYSROOT_S3", os.path.join(env["METAFLOW_DATASTORE_SYSROOT_S3"], "data"), ), show_default=True, ) return env
def sandbox(profile): overwrite_config(profile) # Prompt for user input. encoded_str = click.prompt( "Following instructions from " "https://metaflow.org/sandbox, " "please paste the encoded magic string", type=str, ) # Decode the bytes to env_dict. try: import base64, zlib from metaflow.util import to_bytes env_dict = json.loads( to_unicode(zlib.decompress(base64.b64decode( to_bytes(encoded_str))))) except: # TODO: Add the URL for contact us page in the error? raise click.BadArgumentUsage("Could not decode the sandbox " "configuration. Please contact us.") # Persist to a file. persist_env(env_dict, profile)
def configure_aws_batch(existing_env): empty_profile = False if not existing_env: empty_profile = True env = {} # Set AWS Batch Job Queue. env["METAFLOW_BATCH_JOB_QUEUE"] = click.prompt( cyan("[METAFLOW_BATCH_JOB_QUEUE]") + " AWS Batch Job Queue.", default=existing_env.get("METAFLOW_BATCH_JOB_QUEUE"), show_default=True, ) # Set IAM role for AWS Batch jobs to assume. env["METAFLOW_ECS_S3_ACCESS_IAM_ROLE"] = click.prompt( cyan("[METAFLOW_ECS_S3_ACCESS_IAM_ROLE]") + " IAM role for AWS Batch jobs to access AWS " + "resources (Amazon S3 etc.).", default=existing_env.get("METAFLOW_ECS_S3_ACCESS_IAM_ROLE"), show_default=True, ) # Set default Docker repository for AWS Batch jobs. env["METAFLOW_BATCH_CONTAINER_REGISTRY"] = click.prompt( cyan("[METAFLOW_BATCH_CONTAINER_REGISTRY]") + yellow(" (optional)") + " Default Docker image repository for AWS " + "Batch jobs. If nothing is specified, " + "dockerhub (hub.docker.com/) is " + "used as default.", default=existing_env.get("METAFLOW_BATCH_CONTAINER_REGISTRY", ""), show_default=True, ) # Set default Docker image for AWS Batch jobs. env["METAFLOW_BATCH_CONTAINER_IMAGE"] = click.prompt( cyan("[METAFLOW_BATCH_CONTAINER_IMAGE]") + yellow(" (optional)") + " Default Docker image for AWS Batch jobs. " + "If nothing is specified, an appropriate " + "python image is used as default.", default=existing_env.get("METAFLOW_BATCH_CONTAINER_IMAGE", ""), show_default=True, ) # Configure AWS Step Functions for scheduling. if click.confirm( "\nMetaflow can " + yellow("schedule your flows on AWS Step " "Functions") + " and trigger them at a specific cadence using " "Amazon EventBridge.\nTo support flows involving " "foreach steps, you would need access to AWS " "DynamoDB.\nWould you like to configure AWS Step " "Functions for scheduling?", default=empty_profile or "METAFLOW_SFN_IAM_ROLE" in existing_env, abort=False, ): # Configure IAM role for AWS Step Functions. env["METAFLOW_SFN_IAM_ROLE"] = click.prompt( cyan("[METAFLOW_SFN_IAM_ROLE]") + " IAM role for AWS Step Functions to " + "access AWS resources (AWS Batch, " + "AWS DynamoDB).", default=existing_env.get("METAFLOW_SFN_IAM_ROLE"), show_default=True, ) # Configure IAM role for AWS Events Bridge. env["METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"] = click.prompt( cyan("[METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE]") + " IAM role for Amazon EventBridge to " + "access AWS Step Functions.", default=existing_env.get("METAFLOW_EVENTS_SFN_ACCESS_IAM_ROLE"), show_default=True, ) # Configure AWS DynamoDB Table for AWS Step Functions. env["METAFLOW_SFN_DYNAMO_DB_TABLE"] = click.prompt( cyan("[METAFLOW_SFN_DYNAMO_DB_TABLE]") + " AWS DynamoDB table name for tracking " + "AWS Step Functions execution metadata.", default=existing_env.get("METAFLOW_SFN_DYNAMO_DB_TABLE"), show_default=True, ) return env