def dbappsum(): where = getRequestParam("where", "1") cursor = database.getCursor() sumKey = [ "mapsTotal", "mapsCompleted", "successfulMapAttempts", "killedMapAttempts", "failedMapAttempts", "localMap", "rackMap", "reducesTotal", "reducesCompleted", "successfulReduceAttempts", "killedReduceAttempts", "failedReduceAttempts", "fileRead", "fileWrite", "hdfsRead", "hdfsWrite", ] select = "count(appid) as appidCount" for key in sumKey: select = select + " , sum(" + key + ") as " + key + "Sum " sql = ("select " + select + " from app where %s order by appid ") % (where) cursor.execute(sql) sumRecord = cursor.fetchone() col_name_list = [tuple[0] for tuple in cursor.description] resultRecord = {} for i in range(len(col_name_list)): resultRecord[col_name_list[i]] = sumRecord[i] retult = {"resultRecord": resultRecord} return json.dumps(retult)
def dbapplist(): offset = getRequestInt("offset", 0) limit = getRequestInt("limit", 50) where = getRequestParam("where", "1") orderField = getRequestParam("orderField", "appid") orderDirection = getRequestParam("orderDirection", "desc") cursor = database.getCursor() selectKeyArray = [ "appid", "user", "name", "queue", "startedTime", "finishedTime", "state", "finalStatus", "attemptNumber", "mapsTotal", "mapsCompleted", "localMap", "reducesTotal", "reducesCompleted", "fileRead", "fileWrite", "hdfsRead", "hdfsWrite" ] if orderField == "appid": selectKeyArray.append( "CAST ( SUBSTR ( appid ,27,length(appid)-26 ) AS int ) as appnum") orderby = "appnum " + orderDirection else: orderby = orderField + " " + orderDirection selectKey = ",".join(selectKeyArray) sql = ("select " + selectKey + " from app where %s order by %s LIMIT %d OFFSET %d " % (where, orderby, limit, offset)) print sql # return sql cursor.execute(sql) queryResult = cursor.fetchall() retult = { "applist": queryResult, "rmhost": config.rmhost, "rmport": config.rmport } return json.dumps(retult)
def dbmetricsquery(): fields = getRequestArray("fields", []) recordTimeMax = getRequestInt("recordTimeMax", 0) recordTimeMin = getRequestInt("recordTimeMin", 0) recordTimeSplit = getRequestInt("recordTimeSplit", 600) if len(fields) != 0: recordTimeFilter = " ( recordTime % " + str( recordTimeSplit) + " = 0 ) " where = ("recordTime >= %d and recordTime <= %d and %s" % (recordTimeMin, recordTimeMax, recordTimeFilter)) sqlFields = "" for field in fields: sqlFields = sqlFields + " , " + field sql = ("select recordTime %s from metrics where %s " % (sqlFields, where)) cursor = database.getCursor() cursor.execute(sql) sqlResult = cursor.fetchall() queryResult = [] for record in sqlResult: queryResult.append(list(record)) result = {"result": queryResult, "sql": sql} return json.dumps(result)
def dbnmquery(): hosts = getRequestArray("hosts", []) fields = getRequestArray("fields", []) happenTimeMax = getRequestInt("happenTimeMax", 0) happenTimeMin = getRequestInt("happenTimeMin", 0) happenTimeSplit = getRequestInt("happenTimeSplit", 600) if len(hosts) != 0 and len(fields) != 0: where = "1" if happenTimeMax != 0: where = where + " and happenTime < " + str(happenTimeMax) if happenTimeMin != 0: where = where + " and happenTime > " + str(happenTimeMin) hostWhere = "" for host in hosts: hostWhere = hostWhere + '"' + host + '",' hostWhere = hostWhere.rstrip(",") where = where + ' and host in (' + hostWhere + ')' sqlFields = "" for field in fields: sqlFields = sqlFields + " , sum(" + field + ") as " + field sql = ( "select (happenTime/%d)*%d as printTime , host %s from nm where %s group by printTime,host" % (happenTimeSplit, happenTimeSplit, sqlFields, where)) cursor = database.getCursor() cursor.execute(sql) sqlResult = cursor.fetchall() queryResult = [] #特殊处理mapTime和reduceTime #由于前面有printTime和host两个位置,所以偏移两位 fieldOffset = 2 for record in sqlResult: queryResult.append(list(record)) if "mapTime" in fields and "mapNum" in fields: timeIndex = fields.index("mapTime") + fieldOffset numIndex = fields.index("mapNum") + fieldOffset for record in queryResult: # return json.dumps(record) if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] if "reduceTime" in fields and "reduceNum" in fields: timeIndex = fields.index("reduceTime") + fieldOffset numIndex = fields.index("reduceNum") + fieldOffset for record in queryResult: if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] else: sql = "" queryResult = [] result = {"result": queryResult, "sql": sql} return json.dumps(result)
def dbnmquery(): hosts = getRequestArray("hosts",[]); fields = getRequestArray("fields",[]); happenTimeMax = getRequestInt("happenTimeMax",0); happenTimeMin = getRequestInt("happenTimeMin",0); happenTimeSplit = getRequestInt("happenTimeSplit",600); if len(hosts)!=0 and len(fields)!=0 : where = "1" if happenTimeMax != 0: where = where + " and happenTime < "+str(happenTimeMax) if happenTimeMin != 0: where = where + " and happenTime > "+str(happenTimeMin) hostWhere = "" for host in hosts: hostWhere = hostWhere +'"' + host +'",' hostWhere = hostWhere.rstrip(",") where = where +' and host in ('+hostWhere+')' sqlFields="" for field in fields: sqlFields = sqlFields+" , sum("+field+") as " + field sql = ("select (happenTime/%d)*%d as printTime , host %s from nm where %s group by printTime,host" % (happenTimeSplit,happenTimeSplit,sqlFields,where) ) cursor = database.getCursor() cursor.execute(sql) sqlResult = cursor.fetchall() queryResult = [] #特殊处理mapTime和reduceTime #由于前面有printTime和host两个位置,所以偏移两位 fieldOffset = 2 for record in sqlResult: queryResult.append(list(record)) if "mapTime" in fields and "mapNum" in fields : timeIndex = fields.index("mapTime") + fieldOffset numIndex = fields.index("mapNum") + fieldOffset for record in queryResult: # return json.dumps(record) if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] if "reduceTime" in fields and "reduceNum" in fields : timeIndex = fields.index("reduceTime") + fieldOffset numIndex = fields.index("reduceNum") + fieldOffset for record in queryResult: if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] else: sql = "" queryResult = [] result= {"result":queryResult,"sql":sql} return json.dumps(result)
def dbrmquery(): fields = getRequestArray("fields", []) happenTimeMax = getRequestInt("happenTimeMax", 0) happenTimeMin = getRequestInt("happenTimeMin", 0) happenTimeSplit = getRequestInt("happenTimeSplit", 600) if len(fields) != 0: where = "1" if happenTimeMax != 0: where = where + " and happenTime < " + str(happenTimeMax) if happenTimeMin != 0: where = where + " and happenTime > " + str(happenTimeMin) sqlFields = "" for field in fields: sqlFields = sqlFields + " , sum(" + field + ") as " + field sql = "select (happenTime/%d)*%d as printTime %s from rm where %s group by printTime" % ( happenTimeSplit, happenTimeSplit, sqlFields, where, ) cursor = database.getCursor() cursor.execute(sql) sqlResult = cursor.fetchall() queryResult = [] # 特殊处理mapTime和reduceTime # 由于前面有一个printTime的查询,所以偏移一位 fieldOffset = 1 for record in sqlResult: queryResult.append(list(record)) if "mapTime" in fields and "mapNum" in fields: timeIndex = fields.index("mapTime") + fieldOffset numIndex = fields.index("mapNum") + fieldOffset for record in queryResult: # return json.dumps(record) if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] if "reduceTime" in fields and "reduceNum" in fields: timeIndex = fields.index("reduceTime") + fieldOffset numIndex = fields.index("reduceNum") + fieldOffset for record in queryResult: if record[numIndex] != None and record[numIndex] != 0: record[timeIndex] = record[timeIndex] / record[numIndex] else: sql = "" queryResult = [] result = {"result": queryResult, "sql": sql} return json.dumps(result)
def dbapplist(): offset = getRequestInt("offset", 0) limit = getRequestInt("limit", 50) where = getRequestParam("where", "1") orderField = getRequestParam("orderField", "appid") orderDirection = getRequestParam("orderDirection", "desc") cursor = database.getCursor() selectKeyArray = [ "appid", "user", "name", "queue", "startedTime", "finishedTime", "state", "finalStatus", "attemptNumber", "mapsTotal", "mapsCompleted", "localMap", "reducesTotal", "reducesCompleted", "fileRead", "fileWrite", "hdfsRead", "hdfsWrite", ] if orderField == "appid": selectKeyArray.append("CAST ( SUBSTR ( appid ,27,length(appid)-26 ) AS int ) as appnum") orderby = "appnum " + orderDirection else: orderby = orderField + " " + orderDirection selectKey = ",".join(selectKeyArray) sql = "select " + selectKey + " from app where %s order by %s LIMIT %d OFFSET %d " % (where, orderby, limit, offset) print sql # return sql cursor.execute(sql) queryResult = cursor.fetchall() retult = {"applist": queryResult, "rmhost": config.rmhost, "rmport": config.rmport} return json.dumps(retult)
def dbmetricsquery(): fields = getRequestArray("fields",[]); recordTimeMax = getRequestInt("recordTimeMax",0); recordTimeMin = getRequestInt("recordTimeMin",0); recordTimeSplit = getRequestInt("recordTimeSplit",600); if len(fields)!=0 : recordTimeFilter = " ( recordTime % " + str(recordTimeSplit) + " = 0 ) "; where = ( "recordTime >= %d and recordTime <= %d and %s" % ( recordTimeMin , recordTimeMax , recordTimeFilter ) ) sqlFields="" for field in fields: sqlFields = sqlFields+" , "+field; sql = ("select recordTime %s from metrics where %s " % (sqlFields,where) ) cursor = database.getCursor() cursor.execute(sql) sqlResult = cursor.fetchall() queryResult = [] for record in sqlResult: queryResult.append(list(record)) result= {"result":queryResult,"sql":sql} return json.dumps(result)
def dbappsum(): where = getRequestParam("where", "1") cursor = database.getCursor() sumKey = [ "mapsTotal", "mapsCompleted", "successfulMapAttempts", "killedMapAttempts", "failedMapAttempts", "localMap", "rackMap", "reducesTotal", "reducesCompleted", "successfulReduceAttempts", "killedReduceAttempts", "failedReduceAttempts", "fileRead", "fileWrite", "hdfsRead", "hdfsWrite" ] select = "count(appid) as appidCount" for key in sumKey: select = select + " , sum(" + key + ") as " + key + "Sum " sql = (("select " + select + " from app where %s order by appid ") % (where)) cursor.execute(sql) sumRecord = cursor.fetchone() col_name_list = [tuple[0] for tuple in cursor.description] resultRecord = {} for i in range(len(col_name_list)): resultRecord[col_name_list[i]] = sumRecord[i] retult = {"resultRecord": resultRecord} return json.dumps(retult)