def loop():
while True:
try:
tick()
except Exception, e:
LOG.warning("Exception in main loop: %s" % (traceback.format_exc(e)))
time.sleep(int(config.get_value(KEY_LOOP_INTERVAL_SECS)))
def get_ssh_keys():
keys = re.split(r'\s*,\s*', config.get_value(KEY_SSH_KEYS))
for i in range(0, len(keys)):
key = keys[i]
if key[0] == '$':
var_name = key[1:]
key_file = KEY_FILE_NAME_PATTERN % var_name
if not os.path.isfile(key_file):
key_value = os.environ.get(var_name)
if key_value:
marker_begin = '-----BEGIN RSA PRIVATE KEY-----'
marker_end = '-----END RSA PRIVATE KEY-----'
key_value = key_value.replace(marker_begin, '')
key_value = key_value.replace(marker_end, '')
key_value = key_value.replace(' ', '\n')
if marker_begin not in key_value:
key_value = ('%s\n' % marker_begin) + key_value
if marker_end not in key_value:
key_value += ('\n%s' % marker_end)
key_value = key_value.replace('\n\n', '\n')
save_file(key_file, key_value)
run('chmod 600 %s' % key_file)
else:
LOG.warning(
'Unable to read SSH key from environment variable: %s'
% var_name)
keys[i] = key_file
return keys
def get_cluster_nodes(cluster_id, role=None):
emr_client = connect_emr(role=role)
result = run_func(emr_client.list_instances, ClusterId=cluster_id,
InstanceStates=['AWAITING_FULFILLMENT', 'PROVISIONING', 'BOOTSTRAPPING', 'RUNNING'],
cache_duration_secs=QUERY_CACHE_TIMEOUT)
result = json.loads(result)
result = result['Instances']
# read domain name config
custom_dn = config.get_value(constants.KEY_CUSTOM_DOMAIN_NAME, section=SECTION_EMR, resource=cluster_id)
i = 0
while i < len(result):
inst = result[i]
if inst['Status']['State'] == INSTANCE_STATE_TERMINATED:
del result[i]
i -= 1
else:
inst['cid'] = inst['Id'] if 'Id' in inst else 'n/a'
inst['iid'] = inst['Ec2InstanceId'] if 'Ec2InstanceId' in inst else 'n/a'
inst['gid'] = inst['InstanceGroupId'] if 'InstanceGroupId' in inst else 'n/a'
inst['ip'] = inst['PrivateIpAddress'] if 'PrivateIpAddress' in inst else 'n/a'
inst['host'] = inst['PrivateDnsName'] if 'PrivateDnsName' in inst else 'n/a'
if custom_dn:
inst['host'] = ip_to_hostname(hostname_to_ip(inst['host']), custom_dn)
inst['type'] = get_instance_group_type(cluster_id, inst['InstanceGroupId'], role=role)
inst['state'] = inst['Status']['State']
inst['market'] = get_instance_group_details(cluster_id, inst['InstanceGroupId'], role=role)['Market']
i += 1
return result
def get_node_queries(cluster):
cmd = ('presto-cli --execute \\"SELECT n.http_uri,count(q.node_id) from system.runtime.nodes n ' +
'left join (select * from system.runtime.queries where state = \'RUNNING\' ) as q ' +
'on q.node_id = n.node_id group by n.http_uri\\"')
result = {}
if cluster.ip == 'localhost':
# for testing purposes
return result
# run ssh command
out = run_ssh(cmd, cluster.ip, user='******', cache_duration_secs=QUERY_CACHE_TIMEOUT)
# remove SSH log output line
out = remove_lines_from_string(out, r'.*Permanently added.*')
# read config for domain
custom_dn = config.get_value(constants.KEY_CUSTOM_DOMAIN_NAME, section=SECTION_EMR, resource=cluster.id)
# assume input is actually domain name (not ip)
dn = custom_dn if custom_dn else re.match(r'ip-[^\.]+\.(.+)', cluster.ip).group(1)
for line in out.splitlines():
ip = re.sub(r'.*http://([0-9\.]+):.*', r'\1', line)
if ip:
queries = re.sub(r'.*"([0-9\.]+)"$', r'\1', line)
host = aws_common.ip_to_hostname(ip, dn)
try:
result[host] = int(queries)
except Exception, e:
result[host] = 0
def get_node_queries(cluster):
cmd = ('presto-cli --execute \\"SELECT n.http_uri,count(q.node_id) from system.runtime.nodes n ' +
'left join (select * from system.runtime.queries where state = \'RUNNING\' ) as q ' +
'on q.node_id = n.node_id group by n.http_uri\\"')
result = {}
if cluster.ip == 'localhost':
# for testing purposes
return result
# run ssh command
out = run_ssh(cmd, cluster.ip, user='******', cache_duration_secs=QUERY_CACHE_TIMEOUT)
# remove SSH log output line
out = remove_lines_from_string(out, r'.*Permanently added.*')
# read config for domain
custom_dn = config.get_value(constants.KEY_CUSTOM_DOMAIN_NAME, section=SECTION_EMR, resource=cluster.id)
# assume input is actually domain name (not ip)
dn = custom_dn if custom_dn else re.match(r'ip-[^\.]+\.(.+)', cluster.ip).group(1)
for line in out.splitlines():
ip = re.sub(r'.*http://([0-9\.]+):.*', r'\1', line)
if ip:
queries = re.sub(r'.*"([0-9\.]+)"$', r'\1', line)
host = aws_common.ip_to_hostname(ip, dn)
try:
result[host] = int(queries)
except Exception, e:
result[host] = 0
def run_ssh(cmd, host, user=None, keys=None, via_hosts=[], cache_duration_secs=0):
if not keys:
keys = config.get_value(KEY_SSH_KEYS).split(',')
user = '******' % user if user else ''
agent_forward = ''
forward_addendum = ''
ssh_configs = ('-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no ' +
'-o PasswordAuthentication=no -o BatchMode=yes -o ConnectTimeout=3')
if len(via_hosts) > 0:
agent_forward = '-o ForwardAgent=yes'
for via_host in list(reversed(via_hosts)):
forward_addendum = ('ssh %s %s%s ' % (ssh_configs, user, via_host)) + forward_addendum
ssh_cmd_tmpl = 'ssh ' + ssh_configs + ' ' + agent_forward + ' -i %s %s%s "' + forward_addendum + '%s"'
for key in keys:
key = key.strip()
ssh_cmd = ssh_cmd_tmpl % (key, user, host, cmd)
if len(via_hosts) > 0:
run('ssh-add %s 2>&1 > /dev/null' % key)
try:
out = run(ssh_cmd, cache_duration_secs)
return out
except subprocess.CalledProcessError, e:
# TODO find a more elegant solution for this.
if 'Permission denied (publickey)' not in e.output:
raise e
def update_resources(resource_config):
for resource in resource_config:
id = resource.id
enabled = config.get_value('enable_enhanced_monitoring',
section=SECTION_KINESIS,
resource=id)
if enabled == 'true':
resource.enhanced_monitoring = [MONITORING_METRICS_ALL]
return resource_config
def select_tasknode_group(tasknodes_groups, cluster_id):
if len(tasknodes_groups) <= 0:
raise Exception("Empty list of task node instance groups for scaling: %s" % tasknodes_groups)
if len(tasknodes_groups) == 1:
return tasknodes_groups[0]
preferred = config.get_value(KEY_PREFERRED_UPSCALE_INSTANCE_MARKET, cluster_id)
for group in tasknodes_groups:
if group['market'] == preferred:
return group
raise Exception("Could not select task node instance group for preferred market '%s': %s" %
(preferred, tasknodes_groups))
def get_state(cluster_id):
""" Get cluster state
---
operationId: 'getState'
parameters:
- name: cluster_id
in: path
"""
monitoring_interval_secs = int(config.get_value(KEY_MONITORING_INTERVAL_SECS))
info = monitoring.collect_info(CLUSTERS[cluster_id], monitoring_interval_secs=monitoring_interval_secs)
return jsonify(info)
def loop():
while True:
LOG.info("Running next loop iteration")
try:
resource_list = resources.get_resources()
for resource in resource_list:
resource.fetch_data()
scaling_required = resource.needs_scaling()
if scaling_required:
resource.perform_scaling(scaling_required)
except Exception, e:
LOG.warning("Exception in main loop: %s" % (traceback.format_exc(e)))
time.sleep(int(config.get_value(KEY_LOOP_INTERVAL_SECS)))
def loop():
while True:
LOG.info("Running next loop iteration")
try:
resource_list = resources.get_resources()
for resource in resource_list:
resource.fetch_data()
scaling_required = resource.needs_scaling()
if scaling_required:
resource.perform_scaling(scaling_required)
except Exception, e:
LOG.warning("Exception in main loop: %s" %
(traceback.format_exc(e)))
time.sleep(int(config.get_value(KEY_LOOP_INTERVAL_SECS)))
def get_emr_costs():
""" Get summary of cluster costs and cost savings
---
operationId: 'getEmrCosts'
parameters:
- name: 'request'
in: body
"""
data = json.loads(request.data)
cluster_id = data['cluster_id']
num_datapoints = data['num_datapoints'] if 'num_datapoints' in data else 300
baseline_nodes = (data['baseline_nodes'] if 'baseline_nodes' in data else
config.get_value(KEY_BASELINE_COMPARISON_NODES, section=SECTION_EMR, resource=cluster_id, default=20))
baseline_nodes = int(baseline_nodes)
info = database.history_get(section=SECTION_EMR, resource=cluster_id, limit=num_datapoints)
common.remove_NaN(info)
result = aws_pricing.get_cluster_savings(info, baseline_nodes)
common.remove_NaN(result, delete_values=False, replacement=0)
return jsonify(results=result, baseline_nodes=baseline_nodes)
def tick():
LOG.info("Running next loop iteration")
monitoring_interval_secs = int(config.get_value(KEY_MONITORING_INTERVAL_SECS))
for cluster_id, details in CLUSTERS.iteritems():
cluster_ip = details['ip_public']
info = None
try:
info = monitoring.collect_info(details, monitoring_interval_secs=monitoring_interval_secs)
except Exception, e:
LOG.warning("Error getting monitoring info for cluster %s: %s" % (cluster_id, e))
if info:
action = 'N/A'
# Make sure we are only resizing Presto clusters atm
if details['type'] == 'Presto':
# Make sure we don't change clusters that are not configured
if cluster_id in get_autoscaling_clusters():
try:
nodes_to_terminate = get_nodes_to_terminate(info)
print nodes_to_terminate
if len(nodes_to_terminate) > 0:
for node in nodes_to_terminate:
terminate_node(cluster_ip, node['ip'], node['gid'])
action = 'DOWNSCALE(-%s)' % len(nodes_to_terminate)
else:
nodes_to_add = get_nodes_to_add(info)
if len(nodes_to_add) > 0:
tasknodes_groups = aws_common.get_instance_groups_tasknodes(cluster_id)
tasknodes_group = select_tasknode_group(tasknodes_groups, cluster_id)['id']
current_num_nodes = len([n for key, n in info['nodes'].iteritems()
if n['gid'] == tasknodes_group])
spawn_nodes(cluster_ip, tasknodes_group, current_num_nodes, len(nodes_to_add))
action = 'UPSCALE(+%s)' % len(nodes_to_add)
else:
action = 'NOTHING'
except Exception, e:
LOG.warning("WARNING: Error downscaling/upscaling cluster %s: %s" %
(cluster_id, traceback.format_exc(e)))
# clean up and terminate instances whose nodes are already in inactive state
aws_common.terminate_inactive_nodes(cluster_ip, info['nodes'])
# store the state for future reference
monitoring.history_add(cluster_id, info, action)
def get_emr_costs():
""" Get summary of cluster costs and cost savings
---
operationId: 'getEmrCosts'
parameters:
- name: 'request'
in: body
"""
data = json.loads(request.data)
cluster_id = data['cluster_id']
num_datapoints = data['num_datapoints'] if 'num_datapoints' in data else 300
baseline_nodes = (data['baseline_nodes'] if 'baseline_nodes' in data else
config.get_value(KEY_BASELINE_COMPARISON_NODES,
section=SECTION_EMR,
resource=cluster_id,
default=20))
baseline_nodes = int(baseline_nodes)
info = database.history_get(section=SECTION_EMR,
resource=cluster_id,
limit=num_datapoints)
common.remove_NaN(info)
result = aws_pricing.get_cluster_savings(info, baseline_nodes)
common.remove_NaN(result, delete_values=False, replacement=0)
return jsonify(results=result, baseline_nodes=baseline_nodes)
def get_iam_role_for_cluster(cluster):
if not isinstance(cluster, basestring):
cluster = cluster.id
return config.get_value('role_to_assume', section=SECTION_EMR, resource=cluster)
def get_iam_role_for_stream(stream):
if not isinstance(stream, basestring):
stream = stream.id
return config.get_value('role_to_assume',
section=SECTION_KINESIS,
resource=stream)
def get_iam_role_for_cluster(cluster):
if not isinstance(cluster, basestring):
cluster = cluster.id
return config.get_value('role_to_assume', section=SECTION_EMR, resource=cluster)
def get_autoscaling_clusters():
return re.split(r'\s*,\s*', config.get_value(KEY_AUTOSCALING_CLUSTERS))