def perform_scaling(cluster):
app_config = config.get_config()
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = cluster.monitoring_data
if info:
action = 'N/A'
# Make sure we don't change clusters that are not configured
if cluster.id in app_config.general.get_autoscaling_clusters():
role = emr_monitoring.get_iam_role_for_cluster(cluster)
try:
nodes_to_terminate = get_nodes_to_terminate(info)
if len(nodes_to_terminate) > 0:
for node in nodes_to_terminate:
terminate_node(cluster, node)
action = 'DOWNSCALE(-%s)' % len(nodes_to_terminate)
else:
nodes_to_add = get_nodes_to_add(info)
if len(nodes_to_add) > 0:
tasknodes_groups = aws_common.get_instance_groups_tasknodes(cluster.id, role=role)
tasknodes_group = select_tasknode_group(tasknodes_groups, cluster.id, info=info)['id']
current_num_nodes = len([n for key, n in info['nodes'].iteritems()
if n['gid'] == tasknodes_group])
spawn_nodes(cluster.ip, tasknodes_group, current_num_nodes, len(nodes_to_add), role=role)
action = 'UPSCALE(+%s)' % len(nodes_to_add)
else:
action = 'NOTHING'
except Exception, e:
LOG.warning("WARNING: Error downscaling/upscaling cluster %s: %s" %
(cluster.id, traceback.format_exc(e)))
# clean up and terminate instances whose nodes are already in inactive state
aws_common.terminate_inactive_nodes(cluster, info, role=role)
# store the state for future reference
add_history_entry(cluster, info, action)
def do_get_config(section, resource=None):
cfg = config.get_config()
cfg = cfg.get(section)
if resource:
cfg = cfg.get(resource)
cfg = cfg.to_dict() if cfg else {}
cfg = config.convert_to_list(cfg)
return jsonify({'config': cfg})
def do_get_config(section, resource=None):
cfg = config.get_config()
cfg = cfg.get(section)
if resource:
cfg = cfg.get(resource)
cfg = cfg.to_dict() if cfg else {}
cfg = config.convert_to_list(cfg)
return jsonify({'config': cfg})
def do_set_config(section, new_config, resource=None):
if isinstance(new_config, list):
new_config = config.convert_from_list(new_config)
config.write(new_config, section=section, resource=resource)
cfg = config.get_config(force_load=True)
cfg = cfg.get(section)
if resource:
cfg = cfg.get(resource)
cfg = cfg.to_dict() if cfg else {}
cfg = config.convert_to_list(cfg)
return jsonify({'config': cfg})
def do_set_config(section, new_config, resource=None):
if isinstance(new_config, list):
new_config = config.convert_from_list(new_config)
config.write(new_config, section=section, resource=resource)
cfg = config.get_config(force_load=True)
cfg = cfg.get(section)
if resource:
cfg = cfg.get(resource)
cfg = cfg.to_dict() if cfg else {}
cfg = config.convert_to_list(cfg)
return jsonify({'config': cfg})
def get_kinesis_state(stream_id):
""" Get Kinesis stream state
---
operationId: 'getKinesisState'
parameters:
- name: stream_id
in: path
"""
app_config = config.get_config()
stream = resources.get_resource(SECTION_KINESIS, stream_id, reload=True)
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = kinesis_monitoring.collect_info(stream, monitoring_interval_secs=monitoring_interval_secs)
return jsonify(info)
def get_emr_state(cluster_id):
""" Get EMR cluster state
---
operationId: 'getEmrState'
parameters:
- name: cluster_id
in: path
"""
app_config = config.get_config()
cluster = resources.get_resource(SECTION_EMR, cluster_id)
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = emr_monitoring.collect_info(cluster, monitoring_interval_secs=monitoring_interval_secs)
return jsonify(info)
def get_kinesis_state(stream_id):
""" Get Kinesis stream state
---
operationId: 'getKinesisState'
parameters:
- name: stream_id
in: path
"""
app_config = config.get_config()
stream = resources.get_resource(SECTION_KINESIS, stream_id)
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = kinesis_monitoring.collect_info(
stream, monitoring_interval_secs=monitoring_interval_secs)
return jsonify(info)
def get_emr_state(cluster_id):
""" Get EMR cluster state
---
operationId: 'getEmrState'
parameters:
- name: cluster_id
in: path
"""
app_config = config.get_config()
cluster = resources.get_resource(SECTION_EMR, cluster_id)
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = emr_monitoring.collect_info(
cluster, monitoring_interval_secs=monitoring_interval_secs)
return jsonify(info)
def restart_emr_node():
""" Restart a cluster node
---
operationId: 'restartEmrNode'
parameters:
- name: 'request'
in: body
"""
data = json.loads(request.data)
cluster_id = data['cluster_id']
node_host = data['node_host']
app_config = config.get_config()
clusters = app_config.emr.to_dict()
for c_id, details in clusters.iteritems():
if c_id == cluster_id:
cluster_ip = details.ip
tasknodes_group = aws_common.get_instance_group_for_node(cluster_id, node_host)
if tasknodes_group:
server.terminate_node(cluster_ip, node_host, tasknodes_group)
return jsonify({'result': 'SUCCESS'})
return jsonify({'result': 'Invalid cluster ID provided'})
def restart_emr_node():
""" Restart a cluster node
---
operationId: 'restartEmrNode'
parameters:
- name: 'request'
in: body
"""
data = json.loads(request.data)
cluster_id = data['cluster_id']
node_host = data['node_host']
app_config = config.get_config()
clusters = app_config.emr.to_dict()
for c_id, details in clusters.iteritems():
if c_id == cluster_id:
cluster_ip = details.ip
tasknodes_group = aws_common.get_instance_group_for_node(
cluster_id, node_host)
if tasknodes_group:
server.terminate_node(cluster_ip, node_host, tasknodes_group)
return jsonify({'result': 'SUCCESS'})
return jsonify({'result': 'Invalid cluster ID provided'})
def perform_scaling(cluster):
app_config = config.get_config()
monitoring_interval_secs = int(app_config.general.monitoring_time_window)
info = cluster.monitoring_data
LOG.debug('info={}'.format(json.dumps(info, indent=2)))
if info:
action = 'N/A'
# Make sure we don't change clusters that are not configured
autoscaling_clusters = app_config.general.get_autoscaling_clusters()
if cluster.id in autoscaling_clusters:
role = emr_monitoring.get_iam_role_for_cluster(cluster)
try:
nodes_to_terminate = get_nodes_to_terminate(info)
if len(nodes_to_terminate) > 0:
for node in nodes_to_terminate:
terminate_node(cluster, node, config=app_config)
action = 'DOWNSCALE(-%s)' % len(nodes_to_terminate)
else:
nodes_to_add = get_nodes_to_add(info)
if len(nodes_to_add) > 0:
if is_task_nodes(info):
nodes_groups = aws_common.get_instance_groups_tasknodes(cluster.id, role=role)
else:
nodes_groups = aws_common.get_instance_groups_nodes(cluster.id, role=role)
nodes_group = select_tasknode_group(nodes_groups, cluster.id, info=info)['id']
current_num_nodes = len([n for key, n in info['nodes'].iteritems()
if n['gid'] == nodes_group])
spawn_nodes(cluster.ip, nodes_group, current_num_nodes, len(nodes_to_add), role=role)
action = 'UPSCALE(+%s)' % len(nodes_to_add)
else:
action = 'NOTHING'
except Exception, e:
LOG.error("WARNING: Error downscaling/upscaling cluster %s: %s" %
(cluster.id, traceback.format_exc(e)))
# clean up and terminate instances whose nodes are already in inactive state
aws_common.terminate_inactive_nodes(cluster, info, role=role)
# store the state for future reference
add_history_entry(cluster, info, action)
def needs_scaling(self, params=None):
app_config = config.get_config()
cluster_ids = app_config.general.get_autoscaling_clusters()
return self.id in cluster_ids
def needs_scaling(self, params=None):
app_config = config.get_config()
stream_ids = app_config.general.get_autoscaling_kinesis_streams()
return self.id in stream_ids
def needs_scaling(self, params=None):
app_config = config.get_config()
stream_ids = app_config.general.get_autoscaling_kinesis_streams()
return self.id in stream_ids
def needs_scaling(self, params=None):
app_config = config.get_config()
cluster_ids = app_config.general.get_autoscaling_clusters()
return self.id in cluster_ids