def show_storm_topology(request, id): cluster = dbutil.get_cluster(id) storm_id = request.GET.get('topology_id') spout_keys = ["__ack-count", "__fail-count", "__emit-count", "__transfer-count", "__complete-latency",] bolt_keys = ["__ack-count", "__fail-count", "__emit-count", "__transfer-count", "__process-latency", "__execute-count", "__execute-latency",] storm_metrics = {"storm_id" : storm_id} storm_graphs = [] for key in spout_keys: title = storm_id + ":Spout:" + key query = ("&m=sum:%s{host=%s,group=STORM_BUILTIN_SPOUT_METRICS}&o=" % (key, dbutil.format_storm_name(storm_id))) graph = { "title" : title, "query" : query, } storm_graphs.append(graph) for key in bolt_keys: title = storm_id + ":Bolt:" + key query = ("&m=sum:%s{host=%s,group=STORM_BUILTIN_BOLT_METRICS}&o=" % (key, dbutil.format_storm_name(storm_id))) graph = { "title" : title, "query" : query, } storm_graphs.append(graph) storm_metrics["graphs"] = storm_graphs params = { 'cluster' : cluster, 'storm_metrics' : storm_metrics, } return respond(request, 'monitor/storm_topology.html', params)
def show_cluster_regionserver_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hbase': # return empty paget for unsupported service return HttpResponse('') read_requests_dist_by_rs, write_requests_dist_by_rs = dbutil.get_requests_distribution_groupby(cluster, 'regionserver'); params = { 'chart_id': 'read_requests_on_rs', 'chart_title': 'read requests on region server', 'request_dist': read_requests_dist_by_rs, 'base_url': '/monitor/regionserver/', } read_requests_dist_by_rs_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render(Context(params)) params = { 'chart_id': 'write_requests_on_rs', 'chart_title': 'write requests on region server', 'request_dist': write_requests_dist_by_rs, 'base_url': '/monitor/regionserver/', } write_requests_dist_by_rs_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render(Context(params)) regionservers = dbutil.get_items_on_cluster(cluster, 'regionserver', order_by='name') params = { 'cluster': cluster, 'read_requests_dist_by_rs_chart': read_requests_dist_by_rs_chart, 'write_requests_dist_by_rs_chart': write_requests_dist_by_rs_chart, 'regionservers': regionservers, } return respond(request, 'monitor/hbase_regionserver_board.html', params)
def show_cluster_basic_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hbase': # return empty paget for unsupported service return HttpResponse('') basic_info = dbutil.get_hbase_basic_info(cluster) hdfs_cluster = dbutil.get_hdfs_cluster_by_name(cluster.name) group = 'Cluster' tsdb_read_query = [ metric_helper.make_metric_query(cluster.name, group, 'readRequestsCountPerSec') ] tsdb_write_query = [ metric_helper.make_metric_query(cluster.name, group, 'writeRequestsCountPerSec') ] params = { 'cluster': cluster, 'hdfs_cluster': hdfs_cluster, 'basic_info': basic_info, 'tsdb_read_query': tsdb_read_query, 'tsdb_write_query': tsdb_write_query, } return respond(request, 'monitor/hbase_basic_board.html', params)
def show_cluster_regionserver_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != "hbase": # return empty paget for unsupported service return HttpResponse("") read_requests_dist_by_rs, write_requests_dist_by_rs = dbutil.get_requests_distribution_groupby( cluster, "regionserver" ) params = { "chart_id": "read_requests_on_rs", "chart_title": "read requests on region server", "request_dist": read_requests_dist_by_rs, "base_url": "/monitor/regionserver/", } read_requests_dist_by_rs_chart = loader.get_template("monitor/requests_dist_pie_chart.tpl").render(Context(params)) params = { "chart_id": "write_requests_on_rs", "chart_title": "write requests on region server", "request_dist": write_requests_dist_by_rs, "base_url": "/monitor/regionserver/", } write_requests_dist_by_rs_chart = loader.get_template("monitor/requests_dist_pie_chart.tpl").render(Context(params)) regionservers = dbutil.get_items_on_cluster(cluster, "regionserver", order_by="name") params = { "cluster": cluster, "read_requests_dist_by_rs_chart": read_requests_dist_by_rs_chart, "write_requests_dist_by_rs_chart": write_requests_dist_by_rs_chart, "regionservers": regionservers, } return respond(request, "monitor/hbase_regionserver_board.html", params)
def show_quota_total_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hdfs': return HttpResponse('') tsdb_quota_total, tsdb_space_quota_total = dbutil.get_quota_distribution(cluster) params = { 'chart_id': 'used_quota_total', 'chart_title': 'total name quota on users', 'request_dist': tsdb_quota_total, 'base_url': '/monitor/user/', } tsdb_quota_total_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render(Context(params)) params = { 'chart_id': 'used_space_quota_total', 'chart_title': 'total used space on users', 'request_dist': tsdb_space_quota_total, 'base_url': '/monitor/user/', } tsdb_space_quota_total_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render(Context(params)) tsdb_quota_total_query = [metric_helper.make_quota_query(cluster.name, 'used_quota_total', 'used_quota')] tsdb_space_quota_total_query = [metric_helper.make_quota_query(cluster.name, 'used_space_quota_total', 'used_space_quota')] params = { 'cluster': cluster, 'tsdb_quota_total_chart': tsdb_quota_total_chart, 'tsdb_space_quota_total_chart': tsdb_space_quota_total_chart, 'tsdb_quota_total_query': tsdb_quota_total_query, 'tsdb_space_quota_total_query': tsdb_space_quota_total_query, } return respond(request, 'monitor/quota_total_board.html', params)
def show_cluster_table_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hbase': # return empty paget for unsupported service return HttpResponse('') read_requests_dist_by_table, write_requests_dist_by_table = dbutil.get_requests_distribution_groupby( cluster, 'table') params = { 'chart_id': 'read_requests_on_table', 'chart_title': 'read requests on table', 'request_dist': read_requests_dist_by_table, 'base_url': '/monitor/table/', } read_requests_dist_by_table_chart = loader.get_template( 'monitor/requests_dist_pie_chart.tpl').render(Context(params)) params = { 'chart_id': 'write_requests_on_table', 'chart_title': 'write requests on table', 'request_dist': write_requests_dist_by_table, 'base_url': '/monitor/table/', } write_requests_dist_by_table_chart = loader.get_template( 'monitor/requests_dist_pie_chart.tpl').render(Context(params)) tables = dbutil.get_items_on_cluster(cluster, 'table', order_by='-qps') system_tables = [table for table in tables if is_system_table(table)] user_tables = [table for table in tables if not is_system_table(table)] table_read_item_keys = '|'.join( ['%s-readRequestsCountPerSec' % (table.name) for table in user_tables]) table_write_item_keys = '|'.join([ '%s-writeRequestsCountPerSec' % (table.name) for table in user_tables ]) tsdb_read_query = [] tsdb_write_query = [] for table in user_tables: tsdb_read_query.append( metric_helper.make_metric_query(cluster.name, table.name, 'readRequestsCountPerSec')) tsdb_write_query.append( metric_helper.make_metric_query(cluster.name, table.name, 'writeRequestsCountPerSec')) params = { 'cluster': cluster, 'read_requests_dist_by_table_chart': read_requests_dist_by_table_chart, 'write_requests_dist_by_table_chart': write_requests_dist_by_table_chart, 'system_tables': system_tables, 'user_tables': user_tables, 'table_read_item_keys': table_read_item_keys, 'table_write_item_keys': table_write_item_keys, 'tsdb_read_query': tsdb_read_query, 'tsdb_write_query': tsdb_write_query, } return respond(request, 'monitor/hbase_table_board.html', params)
def show_cluster_replication(request, id): cluster = dbutil.get_cluster(id) region_servers = dbutil.get_regionservers_with_active_replication_metrics_by_cluster(cluster) peer_id_endpoint_map = metric_helper.get_peer_id_endpoint_map(region_servers) params = { 'cluster' : cluster, 'replication_metrics' : metric_helper.make_metrics_query_for_replication(peer_id_endpoint_map), } return respond(request, 'monitor/hbase_replication.html', params)
def show_cluster_replication(request, id): cluster = dbutil.get_cluster(id) region_servers = dbutil.get_regionservers_with_active_replication_metrics_by_cluster(cluster) peer_id_endpoint_map = metric_helper.get_peer_id_endpoint_map(region_servers) params = { "cluster": cluster, "replication_metrics": metric_helper.make_metrics_query_for_replication(peer_id_endpoint_map), } return respond(request, "monitor/hbase_replication.html", params)
def show_cluster_replication(request, id): cluster = dbutil.get_cluster(id) region_servers = dbutil.get_regionservers_with_active_replication_metrics_by_cluster(cluster) (peer_id_endpoint_map, peer_id_cluster_map) = metric_helper.get_peer_id_endpoint_map_and_cluster(region_servers) params = { 'cluster' : cluster, 'replication_metrics' : metric_helper.make_metrics_query_for_replication(peer_id_endpoint_map, peer_id_cluster_map), } return respond(request, 'monitor/hbase_replication.html', params)
def show_cluster_storm_user_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) # user metrics format is <storm_id, component_id:task_id, <key, value>>>; storm_metrics = {} for storm_task in storm_tasks: if storm_task.job.name != 'metricserver': continue try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): topology_metrics_dict = storm_metrics.setdefault(storm_id, {}) for group_name, group_metrics in topology_metrics.iteritems(): if group_name.find("STORM_SYSTEM_") == 0 or group_name == "STORM_BUILTIN_SPOUT_METRICS" or group_name == "STORM_BUILTIN_BOLT_METRICS": continue group_component_id = group_name.split(":")[0] group_task_id = group_name.split(":")[1] group_metrics_dict = topology_metrics_dict.setdefault(group_component_id, {}) task_metrics_dict = group_metrics_dict.setdefault(group_task_id, {}); for metrics_name, metrics in group_metrics.iteritems(): task_metrics_dict[metrics_name] = metrics # after upper handle, storm_metrics in format: <storm_id, <component_id, <task_id, <key, value>>>> format_storm_metrics = {} for storm_id in storm_metrics: topology_metrics = storm_metrics.get(storm_id) format_topology_metrics = format_storm_metrics.setdefault(storm_id, {}) for component_id in topology_metrics: group_metrics = topology_metrics.get(component_id) format_group_metrics = format_topology_metrics.setdefault(component_id, []) for task_id in group_metrics: metrics = group_metrics.get(task_id) key_set, value_set = add_key_set_for_format_group_metrics(format_group_metrics, metrics.keys()) format_metrics_list = [task_id] for key in key_set: if key == "TaskID": continue format_metrics_list.append(metrics.get(key, " ")) value_set.append(format_metrics_list) # after upper handle, format_storm_metrics in format: # <storm_id, <component_id,[<"key_set": [key1, key2, ...... ,keyn], "value_sets": # [[v11, v12, ...... v1n], ...... ,[vm1, vm2, ...... vmn]],> ...... <"key_set": [], "value_sets": []>] > > params = { 'cluster' : cluster, 'storm_metrics' : format_storm_metrics, } return respond(request, 'monitor/storm_user_board.html', params)
def show_cluster_task_board(request, id): cluster = dbutil.get_cluster(id) tasks = dbutil.get_tasks_by_cluster(id) params = {"cluster": cluster, "tasks": tasks} if cluster.service.name == "hdfs": return respond(request, "monitor/hdfs_task_board.html", params) elif cluster.service.name == "hbase": return respond(request, "monitor/hbase_task_board.html", params) else: return respond(request, "monitor/cluster.html", params)
def show_cluster_operation_table_comparison(request, id): cluster = dbutil.get_cluster(id) endpoint = dbutil.map_cluster_to_endpoint(cluster.name) params = { "cluster": cluster, "tsdb_metrics": metric_helper.make_operation_metrics_for_tables_in_cluster(cluster), "endpoint": endpoint, } print params["tsdb_metrics"] return respond(request, "monitor/hbase_cluster_operation_table_comparsion.html", params)
def show_cluster_task_board(request, id): cluster = dbutil.get_cluster(id) tasks = dbutil.get_tasks_by_cluster(id) params = {'cluster': cluster, 'tasks': tasks} if cluster.service.name == 'hdfs': return respond(request, 'monitor/hdfs_task_board.html', params) elif cluster.service.name == 'hbase': return respond(request, 'monitor/hbase_task_board.html', params) else: return respond(request, 'monitor/cluster.html', params)
def show_cluster_operation_table_comparison(request, id): cluster = dbutil.get_cluster(id) endpoint = dbutil.map_cluster_to_endpoint(cluster.name) params = { 'cluster' : cluster, 'tsdb_metrics' : metric_helper.make_operation_metrics_for_tables_in_cluster(cluster), 'endpoint' : endpoint } print params['tsdb_metrics'] return respond(request, 'monitor/hbase_cluster_operation_table_comparsion.html', params)
def show_cluster_table_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != "hbase": # return empty paget for unsupported service return HttpResponse("") read_requests_dist_by_table, write_requests_dist_by_table = dbutil.get_requests_distribution_groupby( cluster, "table" ) params = { "chart_id": "read_requests_on_table", "chart_title": "read requests on table", "request_dist": read_requests_dist_by_table, "base_url": "/monitor/table/", } read_requests_dist_by_table_chart = loader.get_template("monitor/requests_dist_pie_chart.tpl").render( Context(params) ) params = { "chart_id": "write_requests_on_table", "chart_title": "write requests on table", "request_dist": write_requests_dist_by_table, "base_url": "/monitor/table/", } write_requests_dist_by_table_chart = loader.get_template("monitor/requests_dist_pie_chart.tpl").render( Context(params) ) tables = dbutil.get_items_on_cluster(cluster, "table", order_by="-qps") system_tables = [table for table in tables if is_system_table(table)] user_tables = [table for table in tables if not is_system_table(table)] table_read_item_keys = "|".join(["%s-readRequestsCountPerSec" % (table.name) for table in user_tables]) table_write_item_keys = "|".join(["%s-writeRequestsCountPerSec" % (table.name) for table in user_tables]) tsdb_read_query = [] tsdb_write_query = [] for table in user_tables: tsdb_read_query.append(metric_helper.make_metric_query(cluster.name, table.name, "readRequestsCountPerSec")) tsdb_write_query.append(metric_helper.make_metric_query(cluster.name, table.name, "writeRequestsCountPerSec")) params = { "cluster": cluster, "read_requests_dist_by_table_chart": read_requests_dist_by_table_chart, "write_requests_dist_by_table_chart": write_requests_dist_by_table_chart, "system_tables": system_tables, "user_tables": user_tables, "table_read_item_keys": table_read_item_keys, "table_write_item_keys": table_write_item_keys, "tsdb_read_query": tsdb_read_query, "tsdb_write_query": tsdb_write_query, } return respond(request, "monitor/hbase_table_board.html", params)
def show_cluster_operation(request, id): cluster = dbutil.get_cluster(id) endpoint = dbutil.map_cluster_to_endpoint(cluster.name) group = "Cluster" params = { "cluster": cluster, "tsdb_metrics": metric_helper.make_operation_metrics(endpoint, cluster.hbasecluster, group), "endpoint": endpoint, } return respond(request, "monitor/hbase_cluster_operation.html", params)
def show_cluster_operation(request, id): cluster = dbutil.get_cluster(id) endpoint = dbutil.map_cluster_to_endpoint(cluster.name) group = 'Cluster' params = { 'cluster' : cluster, 'tsdb_metrics' : metric_helper.make_operation_metrics(endpoint, cluster.hbasecluster, group), 'endpoint' : endpoint } return respond(request, 'monitor/hbase_cluster_operation.html', params)
def show_storm_topology(request, id): cluster = dbutil.get_cluster(id) storm_id = request.GET.get('topology_id') spout_keys = [ "__ack-count", "__fail-count", "__emit-count", "__transfer-count", "__complete-latency", ] bolt_keys = [ "__ack-count", "__fail-count", "__emit-count", "__transfer-count", "__process-latency", "__execute-count", "__execute-latency", ] storm_metrics = {"storm_id": storm_id} storm_graphs = [] for key in spout_keys: title = storm_id + ":Spout:" + key query = ("&m=sum:%s{host=%s,group=STORM_BUILTIN_SPOUT_METRICS}&o=" % (key, dbutil.format_storm_name(storm_id))) graph = { "title": title, "query": query, } storm_graphs.append(graph) for key in bolt_keys: title = storm_id + ":Bolt:" + key query = ("&m=sum:%s{host=%s,group=STORM_BUILTIN_BOLT_METRICS}&o=" % (key, dbutil.format_storm_name(storm_id))) graph = { "title": title, "query": query, } storm_graphs.append(graph) storm_metrics["graphs"] = storm_graphs params = { 'cluster': cluster, 'storm_metrics': storm_metrics, } return respond(request, 'monitor/storm_topology.html', params)
def show_cluster_table_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hbase': # return empty paget for unsupported service return HttpResponse('') read_requests_dist_by_table, write_requests_dist_by_table = dbutil.get_requests_distribution_groupby(cluster, 'table'); params = { 'chart_id': 'read_requests_on_table', 'chart_title': 'read requests on table', 'request_dist': read_requests_dist_by_table, 'base_url': '/monitor/table/', } read_requests_dist_by_table_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render(Context(params)) params = { 'chart_id': 'write_requests_on_table', 'chart_title': 'write requests on table', 'request_dist': write_requests_dist_by_table, 'base_url': '/monitor/table/', } write_requests_dist_by_table_chart = loader.get_template('monitor/requests_dist_pie_chart.tpl').render( Context(params)) tables = dbutil.get_items_on_cluster(cluster, 'table', order_by='-qps') system_tables = [table for table in tables if is_system_table(table)] user_tables = [table for table in tables if not is_system_table(table)] table_read_item_keys = '|'.join(['%s-readRequestsCountPerSec' % (table.name) for table in user_tables]) table_write_item_keys ='|'.join(['%s-writeRequestsCountPerSec' % (table.name) for table in user_tables]) tsdb_read_query = [] tsdb_write_query = [] for table in user_tables: tsdb_read_query.append(metric_helper.make_metric_query(cluster.name, table.name, 'readRequestsCountPerSec')) tsdb_write_query.append(metric_helper.make_metric_query(cluster.name, table.name, 'writeRequestsCountPerSec')) params = { 'cluster': cluster, 'read_requests_dist_by_table_chart': read_requests_dist_by_table_chart, 'write_requests_dist_by_table_chart': write_requests_dist_by_table_chart, 'system_tables': system_tables, 'user_tables': user_tables, 'table_read_item_keys': table_read_item_keys, 'table_write_item_keys': table_write_item_keys, 'tsdb_read_query': tsdb_read_query, 'tsdb_write_query': tsdb_write_query, } return respond(request, 'monitor/hbase_table_board.html', params)
def show_cluster_basic_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != "hbase": # return empty paget for unsupported service return HttpResponse("") basic_info = dbutil.get_hbase_basic_info(cluster) group = "Cluster" tsdb_read_query = [metric_helper.make_metric_query(cluster.name, group, "readRequestsCountPerSec")] tsdb_write_query = [metric_helper.make_metric_query(cluster.name, group, "writeRequestsCountPerSec")] params = { "cluster": cluster, "basic_info": basic_info, "tsdb_read_query": tsdb_read_query, "tsdb_write_query": tsdb_write_query, } return respond(request, "monitor/hbase_basic_board.html", params)
def show_cluster_basic_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hbase': # return empty paget for unsupported service return HttpResponse('') basic_info = dbutil.get_hbase_basic_info(cluster) group = 'Cluster' tsdb_read_query = [metric_helper.make_metric_query(cluster.name, group, 'readRequestsCountPerSec')] tsdb_write_query = [metric_helper.make_metric_query(cluster.name, group, 'writeRequestsCountPerSec')] params = { 'cluster': cluster, 'basic_info': basic_info, 'tsdb_read_query': tsdb_read_query, 'tsdb_write_query': tsdb_write_query, } return respond(request, 'monitor/hbase_basic_board.html', params)
def show_quota_total_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name != 'hdfs': return HttpResponse('') tsdb_quota_total, tsdb_space_quota_total = dbutil.get_quota_distribution( cluster) params = { 'chart_id': 'used_quota_total', 'chart_title': 'total name quota on users', 'request_dist': tsdb_quota_total, 'base_url': '/monitor/user/', } tsdb_quota_total_chart = loader.get_template( 'monitor/requests_dist_pie_chart.tpl').render(Context(params)) params = { 'chart_id': 'used_space_quota_total', 'chart_title': 'total used space on users', 'request_dist': tsdb_space_quota_total, 'base_url': '/monitor/user/', } tsdb_space_quota_total_chart = loader.get_template( 'monitor/requests_dist_pie_chart.tpl').render(Context(params)) tsdb_quota_total_query = [ metric_helper.make_quota_query(cluster.name, 'used_quota_total', 'used_quota') ] tsdb_space_quota_total_query = [ metric_helper.make_quota_query(cluster.name, 'used_space_quota_total', 'used_space_quota') ] params = { 'cluster': cluster, 'tsdb_quota_total_chart': tsdb_quota_total_chart, 'tsdb_space_quota_total_chart': tsdb_space_quota_total_chart, 'tsdb_quota_total_query': tsdb_quota_total_query, 'tsdb_space_quota_total_query': tsdb_space_quota_total_query, } return respond(request, 'monitor/quota_total_board.html', params)
def show_cluster_storm_builtin_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) type = request.GET.get('type') type_dict = { "Spout": "STORM_BUILTIN_SPOUT_METRICS", "Bolt": "STORM_BUILTIN_BOLT_METRICS", } # builtin metrics format is <storm_id, STORM_BUILTIN_SPOUT_METRICS|STORM_BUILTIN_BOLT_METRICS, <key, value>>> storm_metrics = [] for storm_task in storm_tasks: if storm_task.job.name != 'metricserver': continue try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): element = {"storm_id": storm_id} for group_name, group_metrics in topology_metrics.iteritems(): if group_name == type_dict.get(type): for metrics_name, metrics in group_metrics.iteritems(): metrics_name = metrics_name.lstrip('_') metrics_name = metrics_name.replace('-', '_') element[metrics_name] = metrics storm_metrics.append(element) params = { 'cluster': cluster, 'storm_metrics': storm_metrics, } if type == "Spout": return respond(request, 'monitor/storm_spout_board.html', params) elif type == "Bolt": return respond(request, 'monitor/storm_bolt_board.html', params) else: return HttpResponse('Unsupported type: ' + type)
def show_cluster_storm_builtin_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) type = request.GET.get('type') type_dict = { "Spout": "STORM_BUILTIN_SPOUT_METRICS", "Bolt": "STORM_BUILTIN_BOLT_METRICS", } # builtin metrics format is <storm_id, STORM_BUILTIN_SPOUT_METRICS|STORM_BUILTIN_BOLT_METRICS, <key, value>>> storm_metrics = []; for storm_task in storm_tasks: if storm_task.job.name != 'metricserver': continue try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): element = {"storm_id": storm_id} for group_name, group_metrics in topology_metrics.iteritems(): if group_name == type_dict.get(type): for metrics_name, metrics in group_metrics.iteritems(): metrics_name = metrics_name.lstrip('_') metrics_name = metrics_name.replace('-', '_') element[metrics_name] = metrics storm_metrics.append(element) params = { 'cluster' : cluster, 'storm_metrics' : storm_metrics, } if type == "Spout": return respond(request, 'monitor/storm_spout_board.html', params) elif type == "Bolt": return respond(request, 'monitor/storm_bolt_board.html', params) else: return HttpResponse('Unsupported type: ' + type)
def show_cluster_user_board(request, id): cluster = dbutil.get_cluster(id) if cluster.service.name == 'hdfs': return show_hdfs_user_board(request, cluster); # return empty paget for unsupported service return HttpResponse('')
def show_cluster_storm_system_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) # system metrics format is <storm_id, STORM_SYSTEM_*, <key, value>>>; # and key may in format: "GC/*", "memory/heap:*", ""memory/nonHeap:*" or ".*"; storm_metrics = [] for storm_task in storm_tasks: try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task.last_metrics_raw) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): topology_element = [] for group_name, group_metrics in topology_metrics.iteritems(): if group_name.find("STORM_SYSTEM_") != 0: continue group_name = group_name.lstrip("STORM_SYSTEM_") element = { "worker_endpoint": group_name } gc_value = "" memory_heap_value = "" memory_non_heap_value = "" for metrics_name, metrics in group_metrics.iteritems(): if metrics_name.find("GC/") == 0: if len(gc_value) != 0: gc_value += ", \n" gc_value += metrics_name.lstrip("GC/") + ":" + str( metrics) if metrics_name.find("memory/heap:") == 0: if len(memory_heap_value) != 0: memory_heap_value += ", \n" memory_heap_value += metrics_name.lstrip( "memory/heap:") + ":" + str(metrics) if metrics_name.find("memory/nonHeap:") == 0: if len(memory_non_heap_value) != 0: memory_non_heap_value += ", \n" memory_non_heap_value += metrics_name.lstrip( "memory/nonHeap:") + ":" + str(metrics) if metrics_name == "startTimeSecs": element["start_time_sec"] = metrics if metrics_name == "uptimeSecs": element["uptime_sec"] = metrics element["GC"] = gc_value element["memory_heap"] = memory_heap_value element["memory_non_heap"] = memory_non_heap_value topology_element.append(element) metrics = { "storm_id": storm_id, "topology_metrics": topology_element } storm_metrics.append(metrics) params = { 'cluster': cluster, 'storm_metrics': storm_metrics, } return respond(request, 'monitor/storm_system_metrics_board.html', params)
def show_cluster_storm_system_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) # system metrics format is <storm_id, STORM_SYSTEM_*, <key, value>>>; # and key may in format: "GC/*", "memory/heap:*", ""memory/nonHeap:*" or ".*"; storm_metrics = [] for storm_task in storm_tasks: try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task.last_metrics_raw) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): topology_element = [] for group_name, group_metrics in topology_metrics.iteritems(): if group_name.find("STORM_SYSTEM_") != 0: continue group_name = group_name.lstrip("STORM_SYSTEM_") element = {"worker_endpoint" : group_name}; gc_value = "" memory_heap_value = "" memory_non_heap_value = "" for metrics_name, metrics in group_metrics.iteritems(): if metrics_name.find("GC/") == 0: if len(gc_value) != 0: gc_value += ", \n" gc_value += metrics_name.lstrip("GC/") + ":" + str(metrics) if metrics_name.find("memory/heap:") == 0: if len(memory_heap_value) != 0: memory_heap_value += ", \n" memory_heap_value += metrics_name.lstrip("memory/heap:") + ":" + str(metrics) if metrics_name.find("memory/nonHeap:") == 0: if len(memory_non_heap_value) != 0: memory_non_heap_value += ", \n" memory_non_heap_value += metrics_name.lstrip("memory/nonHeap:") + ":" + str(metrics) if metrics_name == "startTimeSecs": element["start_time_sec"] = metrics if metrics_name == "uptimeSecs": element["uptime_sec"] = metrics element["GC"] = gc_value element["memory_heap"] = memory_heap_value element["memory_non_heap"] = memory_non_heap_value topology_element.append(element) metrics = { "storm_id" : storm_id, "topology_metrics" : topology_element } storm_metrics.append(metrics) params = { 'cluster' : cluster, 'storm_metrics' : storm_metrics, } return respond(request, 'monitor/storm_system_metrics_board.html', params)
def show_cluster_storm_user_metrics(request, id): cluster = dbutil.get_cluster(id) storm_tasks = dbutil.get_storm_task_by_cluster(cluster) # user metrics format is <storm_id, component_id:task_id, <key, value>>>; storm_metrics = {} for storm_task in storm_tasks: if storm_task.job.name != 'metricserver': continue try: json_metrics = json.loads(storm_task.last_metrics_raw) except: logger.warning("Failed to parse metrics of task: %s", storm_task) return HttpResponse('') for storm_id, topology_metrics in json_metrics.iteritems(): topology_metrics_dict = storm_metrics.setdefault(storm_id, {}) for group_name, group_metrics in topology_metrics.iteritems(): if group_name.find( "STORM_SYSTEM_" ) == 0 or group_name == "STORM_BUILTIN_SPOUT_METRICS" or group_name == "STORM_BUILTIN_BOLT_METRICS": continue group_component_id = group_name.split(":")[0] group_task_id = group_name.split(":")[1] group_metrics_dict = topology_metrics_dict.setdefault( group_component_id, {}) task_metrics_dict = group_metrics_dict.setdefault( group_task_id, {}) for metrics_name, metrics in group_metrics.iteritems(): task_metrics_dict[metrics_name] = metrics # after upper handle, storm_metrics in format: <storm_id, <component_id, <task_id, <key, value>>>> format_storm_metrics = {} for storm_id in storm_metrics: topology_metrics = storm_metrics.get(storm_id) format_topology_metrics = format_storm_metrics.setdefault(storm_id, {}) for component_id in topology_metrics: group_metrics = topology_metrics.get(component_id) format_group_metrics = format_topology_metrics.setdefault( component_id, []) for task_id in group_metrics: metrics = group_metrics.get(task_id) key_set, value_set = add_key_set_for_format_group_metrics( format_group_metrics, metrics.keys()) format_metrics_list = [task_id] for key in key_set: if key == "TaskID": continue format_metrics_list.append(metrics.get(key, " ")) value_set.append(format_metrics_list) # after upper handle, format_storm_metrics in format: # <storm_id, <component_id,[<"key_set": [key1, key2, ...... ,keyn], "value_sets": # [[v11, v12, ...... v1n], ...... ,[vm1, vm2, ...... vmn]],> ...... <"key_set": [], "value_sets": []>] > > params = { 'cluster': cluster, 'storm_metrics': format_storm_metrics, } return respond(request, 'monitor/storm_user_board.html', params)