def __init__(self, cluster_name): self.dataproc = googleapiclient.discovery. \ build('dataproc', 'v1', credentials=CREDENTIALS) self.cluster_name = cluster_name self.project_id = utils.get_project_id() s = settings.get_cluster_settings(cluster_name) if s.count(1) == 1: for st in s: self.cluster_settings = st else: raise DataProcException('Cluster not found!')
def __init__(self, payload): data = json.loads(base64.b64decode(payload)) s = settings.get_cluster_settings(data['cluster']) if s.count(1) == 1: for st in s: self.cluster_settings = st else: raise ScalingException('Cluster not found!') self.total = 0 self.dataproc = dataproc_monitoring.DataProc(data['cluster']) self.scale_to = data['scale_to'] self.scaling_direction = data['scaling_direction'] self.containerpendingratio = data['containerpendingratio'] self.cluster_name = self.cluster_settings.Cluster self.preemptible_pct = self.cluster_settings.PreemptiblePct self.min_instances = self.cluster_settings.MinInstances self.max_instances = self.cluster_settings.MaxInstances self.use_memory = self.cluster_settings.UseMemoryForScaling self.up_container_pending_ratio = \ self.cluster_settings.UpContainerPendingRatio self.down_container_pending_ratio = \ self.cluster_settings.DownContainerPendingRatio if self.preemptible_pct != 100: self.preemptibles_to_workers_ratio = self.preemptible_pct / ( 100 - self.preemptible_pct) else: self.preemptibles_to_workers_ratio = -1 try: self.cluster_status = self.dataproc.get_cluster_status() self.current_nodes = \ int(self.dataproc.get_yarn_metric('yarn-nodes-active')) except dataproc_monitoring.DataProcException as e: logging.error(e) raise e
def should_scale(payload): """ Make a decision to scale or not. :param payload: :return: """ cluster_settings = None data = json.loads(base64.b64decode(payload)) yarn_memory_available_percentage = data['yarn_memory_available_percentage'] container_pending_ratio = data['container_pending_ratio'] number_of_nodes = data['number_of_nodes'] cluster_name = data['cluster'] yarn_containers_pending = data['yarn_containers_pending'] workers = data['worker_nodes'] preemptible_workers = data['preemptible_workers'] s = settings.get_cluster_settings(cluster_name) for st in s: cluster_settings = st logging.info( 'Cluster %s YARNMemAvailPct %s ContainerPendingRatio %s number of ' 'nodes %s', cluster_name, yarn_memory_available_percentage, container_pending_ratio, number_of_nodes) met = metrics.Metrics(cluster_name) met.write_timeseries_value('YARNMemoryAvailablePercentage', 100 * yarn_memory_available_percentage) met.write_timeseries_value('ContainerPendingRatio', container_pending_ratio) met.write_timeseries_value('YarnNodes', str(int(workers) + int(preemptible_workers))) met.write_timeseries_value('Workers', workers) met.write_timeseries_value('PreemptibleWorkers', preemptible_workers) scaling_direction = None containerpendingratio = -1 scale_to = -1 # No memory is allocated so no needs for more nodes just scale down to the # minimum # pending containers are waiting.... if container_pending_ratio > cluster_settings.UpContainerPendingRatio: scaling_direction = 'up' containerpendingratio = container_pending_ratio logging.info( "container_pending_ratio %> cluster_settings.UpContainerPendingRatio" ) elif container_pending_ratio < cluster_settings.DownContainerPendingRatio: scaling_direction = 'down' containerpendingratio = container_pending_ratio logging.info( "container_pending_ratio < cluster_settings.DownContainerPendingRatio" ) elif yarn_memory_available_percentage == 1: logging.info("yarn_memory_available_percentage == 1") if number_of_nodes > cluster_settings.MinInstances: logging.info("number_of_nodes > cluster_settings.MinInstances") scaling_direction = 'down' scale_to = cluster_settings.MinInstances # We don't have enough memory lets go up elif yarn_memory_available_percentage < cluster_settings.UpYARNMemAvailPct: logging.info( "yarn_memory_available_percentage < cluster_settings.UpYARNMemAvailPct" ) scaling_direction = 'up' # we have too much memory :) elif yarn_memory_available_percentage > \ cluster_settings.DownYARNMemAvailePct: logging.info( "yarn_memory_available_percentage > cluster_settings.DownYARNMemAvailePct:" ) scaling_direction = 'down' body = { 'cluster': cluster_name, 'scaling_direction': scaling_direction, 'containerpendingratio': containerpendingratio, 'scale_to': scale_to } if scaling_direction is not None: trigger_scaling(body) return 'OK', 204