def __init__(self, cluster_name):
self.dataproc = googleapiclient.discovery. \
build('dataproc', 'v1',
credentials=CREDENTIALS)
self.cluster_name = cluster_name
self.project_id = utils.get_project_id()
s = settings.get_cluster_settings(cluster_name)
if s.count(1) == 1:
for st in s:
self.cluster_settings = st
else:
raise DataProcException('Cluster not found!')
def __init__(self, payload):
data = json.loads(base64.b64decode(payload))
s = settings.get_cluster_settings(data['cluster'])
if s.count(1) == 1:
for st in s:
self.cluster_settings = st
else:
raise ScalingException('Cluster not found!')
self.total = 0
self.dataproc = dataproc_monitoring.DataProc(data['cluster'])
self.scale_to = data['scale_to']
self.scaling_direction = data['scaling_direction']
self.containerpendingratio = data['containerpendingratio']
self.cluster_name = self.cluster_settings.Cluster
self.preemptible_pct = self.cluster_settings.PreemptiblePct
self.min_instances = self.cluster_settings.MinInstances
self.max_instances = self.cluster_settings.MaxInstances
self.use_memory = self.cluster_settings.UseMemoryForScaling
self.up_container_pending_ratio = \
self.cluster_settings.UpContainerPendingRatio
self.down_container_pending_ratio = \
self.cluster_settings.DownContainerPendingRatio
if self.preemptible_pct != 100:
self.preemptibles_to_workers_ratio = self.preemptible_pct / (
100 - self.preemptible_pct)
else:
self.preemptibles_to_workers_ratio = -1
try:
self.cluster_status = self.dataproc.get_cluster_status()
self.current_nodes = \
int(self.dataproc.get_yarn_metric('yarn-nodes-active'))
except dataproc_monitoring.DataProcException as e:
logging.error(e)
raise e
def should_scale(payload):
"""
Make a decision to scale or not.
:param payload:
:return:
"""
cluster_settings = None
data = json.loads(base64.b64decode(payload))
yarn_memory_available_percentage = data['yarn_memory_available_percentage']
container_pending_ratio = data['container_pending_ratio']
number_of_nodes = data['number_of_nodes']
cluster_name = data['cluster']
yarn_containers_pending = data['yarn_containers_pending']
workers = data['worker_nodes']
preemptible_workers = data['preemptible_workers']
s = settings.get_cluster_settings(cluster_name)
for st in s:
cluster_settings = st
logging.info(
'Cluster %s YARNMemAvailPct %s ContainerPendingRatio %s number of '
'nodes %s', cluster_name, yarn_memory_available_percentage,
container_pending_ratio, number_of_nodes)
met = metrics.Metrics(cluster_name)
met.write_timeseries_value('YARNMemoryAvailablePercentage',
100 * yarn_memory_available_percentage)
met.write_timeseries_value('ContainerPendingRatio',
container_pending_ratio)
met.write_timeseries_value('YarnNodes',
str(int(workers) + int(preemptible_workers)))
met.write_timeseries_value('Workers', workers)
met.write_timeseries_value('PreemptibleWorkers', preemptible_workers)
scaling_direction = None
containerpendingratio = -1
scale_to = -1
# No memory is allocated so no needs for more nodes just scale down to the
# minimum
# pending containers are waiting....
if container_pending_ratio > cluster_settings.UpContainerPendingRatio:
scaling_direction = 'up'
containerpendingratio = container_pending_ratio
logging.info(
"container_pending_ratio %> cluster_settings.UpContainerPendingRatio"
)
elif container_pending_ratio < cluster_settings.DownContainerPendingRatio:
scaling_direction = 'down'
containerpendingratio = container_pending_ratio
logging.info(
"container_pending_ratio < cluster_settings.DownContainerPendingRatio"
)
elif yarn_memory_available_percentage == 1:
logging.info("yarn_memory_available_percentage == 1")
if number_of_nodes > cluster_settings.MinInstances:
logging.info("number_of_nodes > cluster_settings.MinInstances")
scaling_direction = 'down'
scale_to = cluster_settings.MinInstances
# We don't have enough memory lets go up
elif yarn_memory_available_percentage < cluster_settings.UpYARNMemAvailPct:
logging.info(
"yarn_memory_available_percentage < cluster_settings.UpYARNMemAvailPct"
)
scaling_direction = 'up'
# we have too much memory :)
elif yarn_memory_available_percentage > \
cluster_settings.DownYARNMemAvailePct:
logging.info(
"yarn_memory_available_percentage > cluster_settings.DownYARNMemAvailePct:"
)
scaling_direction = 'down'
body = {
'cluster': cluster_name,
'scaling_direction': scaling_direction,
'containerpendingratio': containerpendingratio,
'scale_to': scale_to
}
if scaling_direction is not None:
trigger_scaling(body)
return 'OK', 204