def runRawCommand(command, server): client = LepDClient(server=server) data = client.sendRequest(command) data['splitted'] = client.split_to_lines(data['result']) return jsonify(data)
class PerfProfiler: def __init__(self, server, config='release'): self.server = server self.client = LepDClient(self.server) self.config = config self.flame_burner = FlameBurner() self.dataCount = 25 def get_perf_cpu_clock(self, response_lines=None): lepd_command = 'GetCmdPerfCpuclock' if not response_lines: response_lines = self.client.getResponse(lepd_command) elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) if len(response_lines) == 0: return {} response_data = {} if self.config == 'debug': response_data['rawResult'] = response_lines[:] response_data['lepd_command'] = lepd_command column_header_line_prefix = '# Overhead' try: while not response_lines[0].startswith(column_header_line_prefix): response_lines.pop(0) except Exception as e: print(response_lines, "-----------") return {} response_lines.pop(0) response_lines.pop(0) response_lines.pop(0) result_list = [] for line in response_lines: if line.strip() == '': continue line_values = line.split() if len(line_values) < 5: # print(' --------------- skip it.') continue if '%' not in line_values[0]: # print(' --------------- skip it.') continue result_line = {} result_line['Overhead'] = line_values[0] result_line["Command"] = line_values[1] result_line["Shared Object"] = line_values[2] result_line['Symbol'] = ' '.join([str(x) for x in line_values[3:]]) result_list.append(result_line) if len(result_list) >= self.dataCount: # print('now the length of the array is greater than the max, break here') break response_data['data'] = result_list return response_data def get_cmd_perf_flame(self, response_lines=None): lepd_command = 'GetCmdPerfFlame' if not response_lines: response_lines = self.client.getResponse(lepd_command) elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) if len(response_lines) == 0: return {} flame_data = self.flame_burner.burn(response_lines) flame_data_hierarchy = [] # self.flame_burner.generate_json_hierarchy(flame_data, [], flame_data_hierarchy) return { 'flame': flame_data, 'perf_script_output': response_lines, 'hierarchy': flame_data_hierarchy }
class PerfProfiler: def __init__(self, server, config='release'): self.server = server self.client = LepDClient(self.server) self.config = config self.dataCount = 25 def parse_perf_block(self, block_lines): first_line = block_lines.pop(0) # first line is like this: # swapper 0 [000] 1212828.569239: 10101010 cpu-clock: header_line_entities = first_line.split() perf_block = { 'process': header_line_entities[0], 'pid': header_line_entities[1], 'CCC': header_line_entities[2], 'DDD': header_line_entities[3], 'EEE': header_line_entities[4], 'FFF': header_line_entities[5], 'children': [] } for child_line in block_lines: child_entities = child_line.split() perf_block['children'].append({ 'address': child_entities[0], 'process_name': child_entities[1], 'symbol': child_entities[2] }) return perf_block def get_cmd_perf_flame(self, response_lines=None): lepd_command = 'GetCmdPerfFlame' if not response_lines: response_lines = self.client.getResponse(lepd_command) elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) if len(response_lines) == 0: return {} # for line in response_lines: # print(line) parsed_blocks = [] block_lines = [] for line in response_lines: # each block is separated by an empty line if line.strip(): block_lines.append(line) continue parsed_block = self.parse_perf_block(block_lines) parsed_blocks.append(parsed_block) block_lines = [] return parsed_blocks def get_perf_cpu_clock(self, response_lines=None): lepd_command = 'GetCmdPerfCpuclock' if not response_lines: response_lines = self.client.getResponse(lepd_command) elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) if len(response_lines) == 0: return {} response_data = {} if self.config == 'debug': response_data['rawResult'] = response_lines[:] response_data['lepd_command'] = lepd_command column_header_line_prefix = '# Overhead' while not response_lines[0].startswith(column_header_line_prefix): response_lines.pop(0) response_lines.pop(0) response_lines.pop(0) response_lines.pop(0) result_list = [] for line in response_lines: if line.strip() == '': continue line_values = line.split() if len(line_values) < 5: # print(' --------------- skip it.') continue if '%' not in line_values[0]: # print(' --------------- skip it.') continue result_line = {} result_line['Overhead'] = line_values[0] result_line["Command"] = line_values[1] result_line["Shared Object"] = line_values[2] result_line['Symbol'] = ' '.join([str(x) for x in line_values[3:]]) result_list.append(result_line) if len(result_list) >= self.dataCount: # print('now the length of the array is greater than the max, break here') break response_data['data'] = result_list return response_data
class CPUProfiler: def __init__(self, server, config='release'): self.server = server self.client = LepDClient(self.server) self.config = config # this maxDataCount should match the one defined for UI. self.maxDataCount = 25 self.loadBalanceBenchMark = Decimal(40) def getCpuInfoForArm(self, lines): results = {} line = lines.pop(0) results['architecture'] = "ARM" results['model name'] = line.split(':')[1].strip() results['processors'] = {} line = lines.pop(0) while (not line.startswith("Features")): if (line.startswith("processor")): processorId = line.split(":")[1].strip() results['processors'][processorId] = {} bogoMips = lines.pop(0).split(":")[1].strip() results['processors'][processorId]["processorId"] = processorId results['processors'][processorId]["bogomips"] = bogoMips line = lines.pop(0) return results def getCpuInfoForArmArch64(self, lines): results = {} line = lines.pop(0) results['architecture'] = "ARM" results['model name'] = line.split(":")[1].strip() results['processors'] = {} line = lines.pop(0) while (not line.startswith("Features")): if (line.startswith("processor")): processorId = line.split(":")[1].strip() results['processors'][processorId] = {} results['processors'][processorId]["processorId"] = processorId results['processors'][processorId]["bogomips"] = '' line = lines.pop(0) return results def getCpuInfoForX86(self, lines): results = {} results['architecture'] = "X86" results['processors'] = {} for line in lines: if (line.strip() == ""): continue if re.match(r'processor\W+:\W+\d.*', line, re.M | re.I): linePairs = line.split(":") processorId = linePairs[1].strip() results['processors'][processorId] = {} continue if (":" in line): linePairs = line.split(":") lineKey = linePairs[0].strip() lineValue = '' if (len(linePairs) > 1): lineValue = linePairs[1].strip() results['processors'][processorId][lineKey] = lineValue return results def getCpuInfo(self, cpuInfoLines=None): if (cpuInfoLines == None): cpuInfoLines = self.client.getResponse('GetProcCpuinfo') responseData = {} if (self.config == 'debug'): responseData['rawResult'] = cpuInfoLines firstLine = cpuInfoLines[0] if ("ARM" in firstLine): responseData['data'] = self.getCpuInfoForArm(cpuInfoLines) elif ('AArch64' in firstLine): responseData['data'] = self.getCpuInfoForArmArch64(cpuInfoLines) else: secondLine = cpuInfoLines[1] responseData['data'] = self.getCpuInfoForX86(cpuInfoLines) if ('GenuineIntel' not in secondLine): responseData['data']['architecture'] = 'ARM' responseData['data']['processorCount'] = 0 for line in cpuInfoLines: if re.match(r'\W*processor\W*:\W*\d+', line, re.M | re.I): responseData['data']['processorCount'] += 1 return responseData def getProcessorCount(self, cpuInfoLines=None): if (cpuInfoLines == None): cpuInfoLines = self.client.getResponse('GetCpuInfo') responseData = {} for line in cpuInfoLines: if line.startswith('cpunr'): responseData['count'] = int(line.split(":")[1].strip()) break if ('count' not in responseData): print('failed in getting processor count by GetCpuInfo') print(cpuInfoLines) return responseData def getCapacity(self): cpuInfoData = self.getCpuInfo() if (not cpuInfoData): return {} responseData = {} if (self.config == 'debug'): responseData['rawResult'] = cpuInfoData['rawResult'] responseData['lepd_command'] = 'GetProcCpuinfo' capacity = {} capacity['processors'] = cpuInfoData['data']['processors'] coresString = 'Core' coreCount = len(cpuInfoData['data']['processors']) capacity['coresCount'] = coreCount if (coreCount > 1): coresString = "Cores" for processorId, processorData in cpuInfoData['data'][ 'processors'].items(): if (cpuInfoData['data']['architecture'] == "ARM"): if ('model name' in cpuInfoData['data']): processorData['model'] = cpuInfoData['data']['model name'] else: processorData['model'] = '' # Summary is a string to briefly describe the CPU, like "2GHZ x 2", meaning it's a 2-core cpu with 2GHZ speed. if ('bogomips' not in processorData): capacity['bogomips'] = '' capacity['summary'] = '' else: capacity['bogomips'] = processorData['bogomips'] capacity['summary'] = processorData[ 'bogomips'] + " MHz x " + str(coreCount) + coresString capacity['model'] = processorData['model'] capacity['architecture'] = 'ARM' else: modelName = processorData['model name'].replace("(R)", "").replace( " CPU", "") if (" @" in modelName): modelName = modelName[0:modelName.find(" @")] processorData['model'] = modelName processorSpeed = Decimal(processorData['cpu MHz']).quantize( Decimal('0')) # Summary is a string to briefly describe the CPU, like "2GHZ x 2", meaning it's a 2-core cpu with 2GHZ speed. capacity['summary'] = str(processorSpeed) + " MHz x " + str( coreCount) + coresString capacity['model'] = modelName capacity['bogomips'] = processorData['bogomips'] capacity['architecture'] = 'X86' break responseData['data'] = capacity return responseData def get_irq(self, statData): if len(statData) < 10: return None results = statData[10:len(statData)] responseData = {} responseData['data'] = {} for line in results: if (line.strip() == ''): break line_values = line.split() cpu_name = line_values[1] irq_value = 0 for index, value in enumerate(line_values): if index < 2: continue irq_value = irq_value + \ self.client.toDecimal(value) responseData['data'][cpu_name] = irq_value # print(responseData) return responseData def get_soft_irq(self, statData): if len(statData) < 14: return None results = statData[14:len(statData)] responseData = {} responseData['data'] = {} for line in results: if (line.strip() == ''): break line_values = line.split() cpu_name = line_values[1] responseData['data'][cpu_name] = {} responseData['data'][cpu_name]['HRTIMER'] = self.client.toDecimal( line_values[-2]) responseData['data'][cpu_name]['TASKLET'] = self.client.toDecimal( line_values[-4]) responseData['data'][cpu_name]['NET_RX'] = self.client.toDecimal( line_values[-7]) responseData['data'][cpu_name]['NET_TX'] = self.client.toDecimal( line_values[-8]) # print(responseData) return responseData def getStatus(self): statData = self.get_stat() allIdleRatio = self.client.toDecimal( statData['data']['cpu_stat']['all']['idle']) componentInfo = {} componentInfo["name"] = "cpu" componentInfo["ratio"] = 100 - allIdleRatio componentInfo['server'] = self.server if (self.config == 'debug'): componentInfo['rawResult'] = statData['rawResult'] return componentInfo def get_stat(self): results = self.client.getCmdMpStat() if not results: return None results.pop(0) # Basic data, basically for debugging stat_data = { "lepd_command": "GetCmdMpstat", "rawResult": results, "server": self.server } # this is for analysis irq_numbers = [] softirq_numbers = [] # Core data, for displaying stat_data['data'] = {} stat_data['data']['cpu_stat'] = {} for line in results: if (line.strip() == ''): break line_values = line.split() cpu_stat = {} cpu_stat['idle'] = self.client.toDecimal(line_values[-1]) cpu_stat['gnice'] = self.client.toDecimal(line_values[-2]) cpu_stat['guest'] = self.client.toDecimal(line_values[-3]) cpu_stat['steal'] = self.client.toDecimal(line_values[-4]) cpu_stat['soft'] = self.client.toDecimal(line_values[-5]) cpu_stat['irq'] = self.client.toDecimal(line_values[-6]) cpu_stat['iowait'] = self.client.toDecimal(line_values[-7]) cpu_stat['system'] = self.client.toDecimal(line_values[-8]) cpu_stat['nice'] = self.client.toDecimal(line_values[-9]) cpu_stat['user'] = self.client.toDecimal(line_values[-10]) cpu_name = line_values[-11] # this is for mocking data # current_minute = datetime.now().minute # if current_minute % 2 == 0: # if cpu_name == '0': # cpu_stat['irq'] = Decimal(80) # else: # cpu_stat['irq'] = Decimal(20) stat_data['data']['cpu_stat'][cpu_name] = cpu_stat # analysis for load balance analysis_report = self.analyze_irq_for_load_balance( stat_data['data']['cpu_stat']) if analysis_report: if 'messages' not in stat_data: stat_data['messages'] = [] analysis_report['source'] = 'irq' stat_data['messages'].append(analysis_report) #get irq info from stat_data irq_info = self.get_irq(results) if (irq_info != None): stat_data['data']['irq'] = irq_info['data'] #get soft irq info from stat_data softirq_info = self.get_soft_irq(results) if (softirq_info != None): stat_data['data']['softirq'] = softirq_info['data'] return stat_data def analyze_irq_for_load_balance(self, cpu_stat_data): if not cpu_stat_data: return None if len(cpu_stat_data) < 2: return None irq_list = [] for core_name in cpu_stat_data: if core_name == 'all': continue irq_list.append(cpu_stat_data[core_name]) # TODO: will refactor in the future, the logic below is just for demo # a very simple logic: if any two irq values has a difference of over 30% variance, we say it's not load balanced. for index, item in enumerate(irq_list): if index == len(irq_list) - 1: break irqValue = item['irq'] + item['soft'] nextIrqValue = irq_list[index + 1]['irq'] + irq_list[index + 1]['soft'] variance = abs(irqValue - nextIrqValue) print("variance: " + str(variance)) if variance >= self.loadBalanceBenchMark: # if randrange(10) > 4: # this is just for mocking print("IRQ variance=" + str(variance) + ">=0.4, load NOT balanced") return { 'level': "warning", "message": "Load NOT balanced! ", "time": strftime("%Y-%m-%d %H:%M:%S", gmtime()) } else: print("IRQ variance less than 0.3, load balanced") return None def get_average_load(self, response_lines=None): if not response_lines: response_lines = self.client.getResponse('GetProcLoadavg') elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) response_data = {} # if options['debug']: # response_data['rawResult'] = response_lines[:] # response_data['lepd_command'] = 'GetProcLoadavg' response = response_lines[0].split(" ") # '0.00 0.01 0.05 1/103 24750 # 'avg system load of 1 minute ago, 5 minutes ago, 15 minutes ago, # the fourth is A/B, A is the number of running processes # B is the total process count. # last number, like 24750 is the ID of the most recently running process. result_data = { 'last1': self.client.toDecimal(response[0]), 'last5': self.client.toDecimal(response[1]), 'last15': self.client.toDecimal(response[2]) } response_data['data'] = result_data return response_data def getTopOutput(self, responseLines=None): if (responseLines == None): responseLines = self.client.getResponse('GetCmdTop') if (len(responseLines) == 0): return {} responseData = {} if (self.config == 'debug'): responseData['rawResult'] = responseLines[:] headerLine = responseLines.pop(0) while (not re.match(r'\W*PID\W+USER\W+.*', headerLine, re.M | re.I)): headerLine = responseLines.pop(0) headerColumns = headerLine.split() result = {} for lineIndex, responseLine in enumerate(responseLines): if (self.client.LEPDENDINGSTRING in responseLine): break if (lineIndex > self.maxDataCount): break lineValues = responseLine.split() result[lineIndex] = {} # print(headerLine) for columnIndex, columnName in enumerate(headerColumns): if (columnName == 'Name' or columnName == 'CMD'): result[lineIndex][columnName] = ' '.join( [str(x) for x in lineValues[columnIndex:]]) else: result[lineIndex][columnName] = lineValues[columnIndex] responseData['data'] = {} responseData['data']['top'] = result responseData['data']['headerline'] = headerLine if (re.match(r'\W*PID\W+USER\W+PR\W+.*', headerLine, re.M | re.I)): # android : # PID USER PR NI CPU% S #THR VSS RSS PCY Name responseData['data']['os'] = 'android' elif (re.match(r'\W*PID\W+USER\W+PRI\W+NI\W+VSZ\W+RSS\W+.*', headerLine, re.M | re.I)): # for Linux: # PID USER PRI NI VSZ RSS S %CPU %MEM TIME CMD responseData['data']['os'] = 'linux' else: print("GetCmdTop command returned data from unrecognized system") return responseData
class MemoryProfiler: def __init__(self, server, config='release'): self.server = server self.client = LepDClient(self.server) self.config = config self.dataCount = 25 def convertKbToMb(self, strKbValue): return Decimal(int(strKbValue) / 1024).quantize(Decimal('0')) def getStatus(self, response_lines=None): lepd_command = "GetProcMeminfo" if not response_lines: response_lines = self.client.getResponse(lepd_command) elif isinstance(response_lines, str): response_lines = self.client.split_to_lines(response_lines) response_data = {} if self.config == 'debug': response_data['rawResult'] = response_lines[:] response_data['lepd_command'] = lepd_command if len(response_lines) == 0: return response_data results = {} for line in response_lines: linePairs = line.split(":") lineKey = linePairs[0].strip() lineValue = linePairs[1].replace('kB', '').strip() results[lineKey] = lineValue componentInfo = {} componentInfo["name"] = "memory" componentInfo['total'] = self.client.toDecimal(Decimal(int(results['MemTotal']) / 1024).quantize(Decimal('0'))) componentInfo['free'] = self.client.toDecimal(Decimal(int(results['MemFree']) / 1024).quantize(Decimal('0'))) componentInfo['buffers'] = self.client.toDecimal(Decimal(int(results['Buffers']) / 1024).quantize(Decimal('0'))) componentInfo['cached'] = self.client.toDecimal(Decimal(int(results['Cached']) / 1024).quantize(Decimal('0'))) componentInfo['used'] = componentInfo['total'] - componentInfo['free'] - componentInfo['buffers'] - componentInfo['cached'] usedRatio = (componentInfo['used'] / componentInfo['total']) * 100 #usedRatio = Decimal(usedRatio).quantize(Decimal('0.00')) usedRatio = ("%.2f" % usedRatio) componentInfo["ratio"] = usedRatio componentInfo['unit'] = 'MB' response_data['data'] = componentInfo return response_data def getCapacity(self, sampleDataLines = None): responseLines = [] if (sampleDataLines == None): responseLines = self.client.getResponse("GetProcMeminfo") else: responseLines = sampleDataLines if (len(responseLines) == 0): return {} responseData = {} if (self.config == 'debug'): responseData['rawResult'] = responseLines[:] results = {} for line in responseLines: # print(line) if (self.client.LEPDENDINGSTRING in line): continue linePairs = line.split(":") lineKey = linePairs[0].strip() lineValue = linePairs[1].replace('kB', '').strip() results[lineKey] = lineValue componentInfo = {} componentInfo["name"] = "memory" componentInfo["capacity"] = int(int(results['MemTotal']) / 1024) componentInfo["unit"] = "MB" componentInfo["summary"] = str(componentInfo["capacity"]) + " " + componentInfo["unit"] responseData['data'] = componentInfo return responseData def normalizeValue(self, valueString): # 1.23% -> 0.012 if (valueString.endswith("%")): valueString = valueString.replace("%", "") valueString = Decimal(Decimal(valueString) / 100).quantize(Decimal('0.0000')) elif (valueString == "N/A"): valueString = 0 return valueString # def getMemoryStat(self): # # memoryStatData = {} # # results = self.client.getResponse('GetCmdSmem') # if (self.config == 'debug'): # memoryStatData['rawResult'] = results[:] # # headerLine = results.pop(0) # headers = headerLine.split() # # # sMemInfo['headerLine'] = headerLine # for line in results: # # print(line) # lineValues = line.split() # # pid = lineValues[0] # memoryStatData[pid] = {} # # sMemInfo['line'] = line # memoryStatData[pid]['pid'] = lineValues.pop(0) # memoryStatData[pid]['user'] = lineValues.pop(0) # # # the command section is likely to have whitespaces in it thus hard to locate it. workaround here. # memoryStatData[pid]['rss'] = self.normalizeValue(lineValues.pop()) # memoryStatData[pid]['pss'] = self.normalizeValue(lineValues.pop()) # memoryStatData[pid]['uss'] = self.normalizeValue(lineValues.pop()) # memoryStatData[pid]['swap'] = self.normalizeValue(lineValues.pop()) # # memoryStatData[pid]['command'] = ' '.join([str(x) for x in lineValues]) # # if(len(memoryStatData) >= self.dataCount): # break # # return def getProcrank(self): procrankData = {} resultLines = self.client.getResponse('GetCmdProcrank') if (len(resultLines) == 0): return {} if (self.config == 'debug'): procrankData['rawResult'] = resultLines[:] procrankData['data'] = {} procrankData['data']['procranks'] = {} headerLine = resultLines.pop(0) lineIndex = 0 for line in resultLines: if (re.match( r'\W+-+\W+-+\W-+.*', line, re.M|re.I)): break lineValues = line.split() procrankData['data']['procranks'][lineIndex] = {} procrankData['data']['procranks'][lineIndex]['pid'] = lineValues.pop(0) procrankData['data']['procranks'][lineIndex]['vss'] = self.client.toDecimal(Decimal(Decimal(lineValues.pop(0)[:-1]))) procrankData['data']['procranks'][lineIndex]['rss'] = self.client.toDecimal(Decimal(Decimal(lineValues.pop(0)[:-1]))) procrankData['data']['procranks'][lineIndex]['pss'] = self.client.toDecimal(Decimal(Decimal(lineValues.pop(0)[:-1]))) procrankData['data']['procranks'][lineIndex]['uss'] = self.client.toDecimal(Decimal(Decimal(lineValues.pop(0)[:-1]))) procrankData['data']['procranks'][lineIndex]['cmdline'] = ' '.join([str(x) for x in lineValues]) lineIndex += 1 if(len(procrankData) >= self.dataCount): break # now parse from end, which contains summary info lastLine = resultLines[-1] procrankData['data']['sum'] = {} if (lastLine.startswith('RAM:')): lastLine = lastLine.replace("RAM:", '') lastLineValuePairs = lastLine.split(", ") for valuePair in lastLineValuePairs: keyValuePair = valuePair.split() keyName = keyValuePair[1].strip() keyValue = keyValuePair[0].strip() procrankData['data']['sum'][keyName + "Unit"] = keyValue[-1:] procrankData['data']['sum'][keyName] = self.client.toDecimal(Decimal(Decimal(keyValue[:-1]))) xssSumLine = resultLines[-3].strip() if (xssSumLine.endswith('TOTAL')): xssValues = xssSumLine.split() ussTotalString = xssValues[-2] procrankData['data']['sum']['ussTotalUnit'] = ussTotalString[-1:] procrankData['data']['sum']['ussTotal'] = self.client.toDecimal(Decimal(Decimal(ussTotalString[:-1]))) pssTotalString = xssValues[-3] procrankData['data']['sum']['pssTotalUnit'] = pssTotalString[-1:] procrankData['data']['sum']['pssTotal'] = self.client.toDecimal(Decimal(Decimal(pssTotalString[:-1]))) return procrankData