Default duration of the boost factor is 40 minutes and default value is 1.5
"""
import logging
from collections import namedtuple
from datetime import datetime
from time import time as get_time
from kazoo.client import KazooClient
from kazoo.exceptions import NoNodeError
from paasta_tools.clusterman import get_clusterman_metrics
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import ZookeeperPool
clusterman_metrics, __ = get_clusterman_metrics()
DEFAULT_BOOST_FACTOR = 1.5
DEFAULT_BOOST_DURATION = 40
MIN_BOOST_FACTOR = 1.0
MAX_BOOST_FACTOR = 3.0
MAX_BOOST_DURATION = 240
BoostValues = namedtuple(
'BoostValues',
[
'end_time',
'boost_factor',
'expected_load',
],
from paasta_tools.utils import list_services
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoConfigurationForServiceError
from paasta_tools.utils import NoDeploymentsAvailable
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import PaastaColors
from paasta_tools.utils import PaastaNotConfiguredError
from paasta_tools.utils import SystemPaastaConfig
DEFAULT_AWS_REGION = "us-west-2"
DEFAULT_SPARK_WORK_DIR = "/spark_driver"
DEFAULT_SPARK_DOCKER_IMAGE_PREFIX = "paasta-spark-run"
DEFAULT_SPARK_DOCKER_REGISTRY = "docker-dev.yelpcorp.com"
SENSITIVE_ENV = ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
clusterman_metrics, CLUSTERMAN_YAML_FILE_PATH = get_clusterman_metrics()
CLUSTER_MANAGER_MESOS = "mesos"
CLUSTER_MANAGER_K8S = "kubernetes"
CLUSTER_MANAGERS = {CLUSTER_MANAGER_MESOS, CLUSTER_MANAGER_K8S}
# Reference: https://spark.apache.org/docs/latest/configuration.html#application-properties
DEFAULT_DRIVER_CORES_BY_SPARK = 1
DEFAULT_DRIVER_MEMORY_BY_SPARK = "1g"
# Extra room for memory overhead and for any other running inside container
DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2
POD_TEMPLATE_DIR = "/nail/tmp"
POD_TEMPLATE_PATH = "/nail/tmp/spark-pt-{file_uuid}.yaml"
DEFAULT_RUNTIME_TIMEOUT = "12h"
POD_TEMPLATE = """
apiVersion: v1
