from themis.constants import *
# config file location
CONFIG_FILE_LOCATION = os.path.join(os.getcwd(), 'themis.config.json')
RESOURCES_FILE_LOCATION = os.path.join(os.getcwd(), 'themis.resources.json')
# configuration sections
SECTION_GLOBAL = 'general'
SECTION_EMR = 'emr'
SECTION_KINESIS = 'kinesis'
# set this to override config for testing
TEST_CONFIG = None
# logger
LOG = common.get_logger(__name__)
# maps config keys to their descriptions
ALL_DESCRIPTIONS = {}
# configuration change listeners
CONFIG_LISTENERS = set()
# seconds to cache the config for
CONFIG_CACHE_DURATION = 10
last_config_load_time = 0
class ConfigObject(JsonObject):
def get(self, *keys, **kwargs):
default = kwargs.get('default')
Dates to match against are formatted as "%a %Y-%m-%d %H:%M:%S". \
Example config: { "(Mon|Tue|Wed|Thu|Fri).01:.:.*": 1}'}""".replace(' ', '')},
{KEY: KEY_PREFERRED_UPSCALE_INSTANCE_MARKET, VAL: MARKET_SPOT,
DESC: """Whether to prefer increasing the pool of SPOT instances or \
ON_DEMAND instances (if both exist in the cluster)""".replace(' ', '')}
]
}
# load list of clusters
CLUSTER_LIST = common.load_json_file(CLUSTERS_FILE_LOCATION, [])
# set this to override config for testing
TEST_CONFIG = None
# logger
LOG = common.get_logger(__name__)
def init_clusters_file():
cfg = []
# Don't run this in parallel for now. There seems to be an issue with
# the AWS CLI or API if we run multiple "aws ..." commands in parallel
run_parallel = False
def init_cluster_config(c):
if c['Status']['State'][0:10] != 'TERMINATED':
out1 = run('aws emr describe-cluster --cluster-id=%s' % c['Id'], retries=1)
out1 = json.loads(out1)
cluster_details = out1['Cluster']
cluster = {
'id': c['Id'],
import pandas
import time
import pytz
from datetime import datetime, timedelta
from themis.util import aws_common
from themis.util.common import get_logger
# logger
LOG = get_logger(__name__)
class DefaultExtractor(object):
def get_x(self, value, index=None, values=None):
return index
def get_y(self, value, index=None, values=None):
return value
class CloudwatchExtractor(object):
def get_x(self, value, index=None, values=None):
return value['Timestamp']
def get_y(self, value, index=None, values=None):
for k, v in value.iteritems():
if k not in ['Timestamp', 'Unit']:
return v
return None
class SpotPriceHistoryExtractor(object):
