Default duration of the boost factor is 40 minutes and default value is 1.5 """ import logging from collections import namedtuple from datetime import datetime from time import time as get_time from kazoo.client import KazooClient from kazoo.exceptions import NoNodeError from paasta_tools.clusterman import get_clusterman_metrics from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import ZookeeperPool clusterman_metrics, __ = get_clusterman_metrics() DEFAULT_BOOST_FACTOR = 1.5 DEFAULT_BOOST_DURATION = 40 MIN_BOOST_FACTOR = 1.0 MAX_BOOST_FACTOR = 3.0 MAX_BOOST_DURATION = 240 BoostValues = namedtuple( 'BoostValues', [ 'end_time', 'boost_factor', 'expected_load', ],
from paasta_tools.utils import list_services from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import NoConfigurationForServiceError from paasta_tools.utils import NoDeploymentsAvailable from paasta_tools.utils import NoDockerImageError from paasta_tools.utils import PaastaColors from paasta_tools.utils import PaastaNotConfiguredError from paasta_tools.utils import SystemPaastaConfig DEFAULT_AWS_REGION = "us-west-2" DEFAULT_SPARK_WORK_DIR = "/spark_driver" DEFAULT_SPARK_DOCKER_IMAGE_PREFIX = "paasta-spark-run" DEFAULT_SPARK_DOCKER_REGISTRY = "docker-dev.yelpcorp.com" SENSITIVE_ENV = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] clusterman_metrics, CLUSTERMAN_YAML_FILE_PATH = get_clusterman_metrics() CLUSTER_MANAGER_MESOS = "mesos" CLUSTER_MANAGER_K8S = "kubernetes" CLUSTER_MANAGERS = {CLUSTER_MANAGER_MESOS, CLUSTER_MANAGER_K8S} # Reference: https://spark.apache.org/docs/latest/configuration.html#application-properties DEFAULT_DRIVER_CORES_BY_SPARK = 1 DEFAULT_DRIVER_MEMORY_BY_SPARK = "1g" # Extra room for memory overhead and for any other running inside container DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2 POD_TEMPLATE_DIR = "/nail/tmp" POD_TEMPLATE_PATH = "/nail/tmp/spark-pt-{file_uuid}.yaml" DEFAULT_RUNTIME_TIMEOUT = "12h" POD_TEMPLATE = """ apiVersion: v1