def aggregate_region_operation_metric_for_table_and_cluster(self): allClusterOperationMetric = {} # because the number of regions could be huge. We read out region operation metrics # by table, then table operation metrics and cluster operation metrics could be aggregated tables = Table.objects.all() for table in tables: clusterName = table.cluster.name clusterOperationMetric = allClusterOperationMetric.setdefault( clusterName, {}) tableOperationMetric = {} regions = dbutil.get_region_by_table(table) logger.info( "TableOperationMetricAggregation aggregate %d regions metric for table %s, cluster %s", len(regions), table.name, clusterName) for region in regions: if region.operationMetrics is None or region.operationMetrics == '': continue regionOperationMetrics = json.loads(region.operationMetrics) for regionOperationName in regionOperationMetrics.keys(): regionOperation = regionOperationMetrics[ regionOperationName] self.aggregate_one_region_operation_metric( tableOperationMetric.setdefault( regionOperationName, self.make_empty_operation_metric()), regionOperation) self.aggregate_one_region_operation_metric( clusterOperationMetric.setdefault( regionOperationName, self.make_empty_operation_metric()), regionOperation) # compute avgTime for table operation metrics self.compute_avg_time_and_num_ops_after_aggregation( tableOperationMetric) table.operationMetrics = json.dumps(tableOperationMetric) table.save() # compute avgTime for clusetr operation metrics clusters = HBaseCluster.objects.all() for cluster in clusters: clusterName = cluster.cluster.name if clusterName in allClusterOperationMetric: clusterOperationMetric = allClusterOperationMetric[clusterName] self.compute_avg_time_and_num_ops_after_aggregation( clusterOperationMetric) cluster.operationMetrics = json.dumps(clusterOperationMetric) cluster.save() return
def aggregate_region_operation_metric_in_process(output_queue, task_data): allClusterOperationMetric = {} # because the number of regions could be huge. We read out region operation metrics # by table, then table operation metrics and cluster operation metrics could be aggregated tables = Table.objects.all() for table in tables: clusterName = table.cluster.name clusterOperationMetric = allClusterOperationMetric.setdefault(clusterName, {}) tableOperationMetric = {} regions = dbutil.get_region_by_table(table) logger.info( "TableOperationMetricAggregation aggregate %d " "regions metric for table %s, cluster %s", len(regions), table.name, clusterName, ) for region in regions: if region.operationMetrics is None or region.operationMetrics == "": continue regionOperationMetrics = json.loads(region.operationMetrics) for regionOperationName in regionOperationMetrics.keys(): regionOperation = regionOperationMetrics[regionOperationName] aggregate_one_region_operation_metric( tableOperationMetric.setdefault(regionOperationName, make_empty_operation_metric()), regionOperation ) aggregate_one_region_operation_metric( clusterOperationMetric.setdefault(regionOperationName, make_empty_operation_metric()), regionOperation, ) # compute avgTime for table operation metrics compute_avg_time_and_num_ops_after_aggregation(tableOperationMetric) table.operationMetrics = json.dumps(tableOperationMetric) table.save() # compute avgTime for clusetr operation metrics clusters = HBaseCluster.objects.all() for cluster in clusters: clusterName = cluster.cluster.name if clusterName in allClusterOperationMetric: clusterOperationMetric = allClusterOperationMetric[clusterName] compute_avg_time_and_num_ops_after_aggregation(clusterOperationMetric) cluster.operationMetrics = json.dumps(clusterOperationMetric) cluster.save() output_queue.put(QueueTask(AGGREGATE_TASK_TYPE, None))
def aggregate_region_operation_metric_for_table_and_cluster(self): allClusterOperationMetric = {} # because the number of regions could be huge. We read out region operation metrics # by table, then table operation metrics and cluster operation metrics could be aggregated tables = Table.objects.all() for table in tables: clusterName = table.cluster.name clusterOperationMetric = allClusterOperationMetric.setdefault(clusterName, {}) tableOperationMetric = {} regions = dbutil.get_region_by_table(table) logger.info( "TableOperationMetricAggregation aggregate %d regions metric for table %s, cluster %s" , len(regions), table.name, clusterName) for region in regions: if region.operationMetrics is None or region.operationMetrics == '': continue; regionOperationMetrics = json.loads(region.operationMetrics) for regionOperationName in regionOperationMetrics.keys(): regionOperation = regionOperationMetrics[regionOperationName] self.aggregate_one_region_operation_metric(tableOperationMetric.setdefault(regionOperationName, self.make_empty_operation_metric()), regionOperation) self.aggregate_one_region_operation_metric(clusterOperationMetric.setdefault(regionOperationName, self.make_empty_operation_metric()), regionOperation) # compute avgTime for table operation metrics self.compute_avg_time_and_num_ops_after_aggregation(tableOperationMetric) table.operationMetrics = json.dumps(tableOperationMetric) table.save() # compute avgTime for clusetr operation metrics clusters = HBaseCluster.objects.all() for cluster in clusters: clusterName = cluster.cluster.name if clusterName in allClusterOperationMetric: clusterOperationMetric = allClusterOperationMetric[clusterName] self.compute_avg_time_and_num_ops_after_aggregation(clusterOperationMetric) cluster.operationMetrics = json.dumps(clusterOperationMetric) cluster.save() return