def _parse_awsdata_section(imap, parsed_map): sec_id_1 = 'ID_70' raw_section_name_1, final_section_name_1, _ = get_section_name_from_id( sec_id_1) sec_id_2 = 'ID_85' raw_section_name_2, final_section_name_2, _ = get_section_name_from_id( sec_id_2) logger.info("Parsing section: " + final_section_name_1) if not imap: logger.warning("Null section json") return awsdata = {} field_count = 0 total_fields = 2 aws_section_list = None # If both sections are present, select which has more number of lines. if raw_section_name_1 in imap and raw_section_name_2 in imap: if len(imap[raw_section_name_1]) > len(imap[raw_section_name_2]): aws_section_list = imap[raw_section_name_1] else: aws_section_list = imap[raw_section_name_2] elif raw_section_name_1 in imap: aws_section_list = imap[raw_section_name_1] elif raw_section_name_2 in imap: aws_section_list = imap[raw_section_name_2] else: logger.warning(raw_section_name_1 + " and " + raw_section_name_2 + " section is not present in section json.") return if len(aws_section_list) > 1: logger.warning( "More than one entries detected, There is a collision for this section(aws_info).") aws_section = aws_section_list[0] for index, line in enumerate(aws_section): if field_count >= total_fields: break if 'in_aWS' not in awsdata and re.search("This .* in AWS", line, re.IGNORECASE): awsdata['in_aws'] = True field_count += 1 continue if 'in_aWS' not in awsdata and re.search("not .* in aws", line, re.IGNORECASE): awsdata['in_aws'] = False field_count += 1 continue if 'instance_type' not in awsdata and re.search("instance-type", line): awsdata['instance_type'] = (aws_section[index + 1]).split('\n')[0] field_count += 1 if 'in_aWS' not in awsdata: awsdata['in_aws'] = True field_count += 1 continue parsed_map[final_section_name_1] = awsdata
def _parse_ipaddr_section(imap, parsed_map): sec_id = 'ID_72' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return ip_section = imap[raw_section_name][0] ip_list = [] for line in ip_section: # inet 127.0.0.1/8 scope host lo if 'inet' in line and 'inet6' not in line: tok_list = line.rstrip().split() ip_list.append(tok_list[1].split('/')[0]) continue # inet6 fe80::a236:9fff:fe82:7fde/64 scope link if 'inet6' in line: tok_list = line.rstrip().split() ip = '[' + tok_list[1].split('/')[0] + ']' ip_list.append(ip) continue parsed_map[final_section_name] = {} parsed_map[final_section_name]['hosts'] = ip_list
def _parse_meminfo_section(imap, parsed_map): sec_id = 'ID_92' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return meminfodata = {} meminfo_section = imap[raw_section_name][0] # If this section is not empty then there would be more than 5-6 lines, # defensive check. if len(meminfo_section) < 3: logger.info("meminfo section seems empty.") return for line in meminfo_section: # If line is a newline char, skip it. line size 4 (defensive check) if len(line) < 4 or line.strip() == '': continue # "Mem_total: 32653368 k_b\n", if ':' not in line: continue keyval = line.split(':') key = keyval[0].replace(' ', '_') meminfodata[key] = int(keyval[1].split()[0]) * 1024 parsed_map[final_section_name] = meminfodata
def _parse_dmesg_section(imap, parsed_map): sec_id = 'ID_42' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return dmesg_section = imap[raw_section_name][0] parsed_map[final_section_name] = {} cpu_list = [] parsed_map[final_section_name]["OOM"] = False parsed_map[final_section_name]["Blocked"] = False parsed_map[final_section_name]["ENA_enabled"] = False for line in dmesg_section: if 'OOM' in line: parsed_map[final_section_name]["OOM"] |= True if 'blocked for more than 120 seconds' in line: parsed_map[final_section_name]["Blocked"] |= True if 'Linux version' in line: parsed_map[final_section_name]["OS"] = line if ' ena ' in line or ' ena:' in line: parsed_map[final_section_name]["ENA_enabled"] = True
def _parse_hdparm_section(imap, parsed_map): sec_id = 'ID_110' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return device_info = {} hdparm_section = imap[raw_section_name][0] for line in hdparm_section: if re.search("/dev.*:", line, re.IGNORECASE): device = line if ('Sector size' in line or 'device size' in line or 'Model Number' in line or 'Serial Number' in line or 'Firmware Revision' in line or 'Transport' in line or 'Queue Depth' in line): lineobj = line.rstrip().split(':') key = str(device) + str(lineobj[0]).strip() val = str(lineobj[1]).strip() device_info[key] = val parsed_map[final_section_name] = device_info
def _parse_free_m_section(imap, parsed_map): sec_id = 'ID_37' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return free_m_data = {} tok_list = [] alltok_list = ['total', 'used', 'free', 'shared', 'buffers', 'cached', 'buff/cache', 'available'] found_sec_start = False free_m_section = imap[raw_section_name][0] for line in free_m_section: if 'total' in line and 'used' in line and 'free' in line: sectok_list = line.rstrip().split() if set(sectok_list).intersection(set(alltok_list)) != set(sectok_list): logger.error("Free-m section format changed. union list: " + str(alltok_list) + " new sec list: " + str(sectok_list)) return tok_list = sectok_list found_sec_start = True if found_sec_start and 'Mem:' in line: data_list = line.rstrip().split() mem_obj = {} for idx, val in enumerate(tok_list): mem_obj[val] = data_list[idx + 1] free_m_data['mem'] = mem_obj continue if found_sec_start and '-/+ buffers/cache:' in line: data_list = line.rstrip().split() buffer_obj = {} buffer_obj[tok_list[1]] = data_list[2] buffer_obj[tok_list[2]] = data_list[3] free_m_data['buffers/cache'] = buffer_obj continue if found_sec_start and 'Swap:' in line: data_list = line.rstrip().split() swap_obj = {} swap_obj[tok_list[0]] = data_list[1] swap_obj[tok_list[1]] = data_list[2] swap_obj[tok_list[2]] = data_list[3] free_m_data['swap'] = swap_obj continue parsed_map[final_section_name] = free_m_data
def _parse_df_section(imap, parsed_map): sec_id = 'ID_38' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return df_data = [] tok_count = 6 found_sec_start = False size_in_kb = False df_section = imap[raw_section_name][0] for index, line in enumerate(df_section): if re.search(r'id.*enabled', line): break if line.strip() == '': continue if 'Filesystem' in line: found_sec_start = True continue if '1_k-block' in line: size_in_kb = True continue if found_sec_start: tok_list = line.strip().split() if len(tok_list) != tok_count: if index > len(df_section) - 1: continue if len(tok_list) == 1 and (len( df_section[index + 1].rstrip().split()) == tok_count - 1): tok_list = tok_list + df_section[index + 1].rstrip().split() df_section[index + 1] = '' else: continue file_system = {} file_system['name'] = tok_list[0] file_system['size'] = _get_mem_in_byte_from_str(tok_list[1], 1) file_system['used'] = _get_mem_in_byte_from_str(tok_list[2], 1) file_system['avail'] = _get_mem_in_byte_from_str(tok_list[3], 1) file_system['%use'] = tok_list[4].replace('%', '') file_system['mount_point'] = tok_list[5] if size_in_kb: file_system['size'] = file_system['size'] * 1024 df_data.append(file_system) parsed_map[final_section_name] = {} parsed_map[final_section_name]['Filesystems'] = df_data
def _parse_df_section(imap, parsed_map): sec_id = 'ID_38' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return df_data = [] tok_count = 6 found_sec_start = False size_in_kb = False df_section = imap[raw_section_name][0] for index, line in enumerate(df_section): if re.search(r'id.*enabled', line): break if line.strip() == '': continue if 'Filesystem' in line: found_sec_start = True continue if '1_k-block' in line: size_in_kb = True continue if found_sec_start: tok_list = line.strip().split() if len(tok_list) != tok_count: if index > len(df_section) - 1: continue if len(tok_list) == 1 and (len(df_section[index + 1].rstrip().split()) == tok_count - 1): tok_list = tok_list + df_section[index + 1].rstrip().split() df_section[index + 1] = '' else: continue file_system = {} file_system['name'] = tok_list[0] file_system['size'] = _get_mem_in_byte_from_str(tok_list[1], 1) file_system['used'] = _get_mem_in_byte_from_str(tok_list[2], 1) file_system['avail'] = _get_mem_in_byte_from_str(tok_list[3], 1) file_system['%use'] = tok_list[4].replace('%', '') file_system['mount_point'] = tok_list[5] if size_in_kb: file_system['size'] = file_system['size'] * 1024 df_data.append(file_system) parsed_map[final_section_name] = {} parsed_map[final_section_name]['Filesystems'] = df_data
def _parse_scheduler_section(imap, parsed_map): sec_id = 'ID_100' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return scheduler_section = imap[raw_section_name][0] schedulers = [] scheduler = "" device = "" for line in scheduler_section: line = line.strip() if not line or "cannot access" in line: continue if "scheduler" in line: l = line.split("/sys/block/") if not l: continue l = l[1].split("/queue/scheduler") if not l: continue device = l[0].strip() continue if not device: # device not found yet, no need to proceed with this line continue # find scheduler for s in line.split(): if not s: continue if s.startswith('[') and s.endswith(']'): scheduler = s[1:len(s) - 1].lower() # if scheduler found, set details if scheduler: schedulers.append({"device": device, "scheduler": scheduler}) scheduler = "" device = "" if final_section_name not in parsed_map: parsed_map[final_section_name] = {} parsed_map[final_section_name]["scheduler_stat"] = schedulers
def _parse_scheduler_section(imap, parsed_map): sec_id = 'ID_100' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return scheduler_section = imap[raw_section_name][0] schedulers = [] scheduler = "" device = "" for line in scheduler_section: line = line.strip() if not line or "cannot access" in line: continue if "scheduler" in line: l = line.split("/sys/block/") if not l: continue l = l[1].split("/queue/scheduler") if not l: continue device = l[0].strip() continue if not device: # device not found yet, no need to proceed with this line continue # find scheduler for s in line.split(): if not s: continue if s.startswith('[') and s.endswith(']'): scheduler = s[1:len(s)-1].lower() # if scheduler found, set details if scheduler: schedulers.append({"device":device, "scheduler": scheduler}) scheduler = "" device = "" if final_section_name not in parsed_map: parsed_map[final_section_name] = {} parsed_map[final_section_name]["scheduler_stat"] = schedulers
def _parse_iptables_section(imap, parsed_map): sec_id = 'ID_108' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return iptables_section = imap[raw_section_name][0] parsed_map[final_section_name] = {} for line in iptables_section: if "DROP" in line: parsed_map[final_section_name]["has_firewall"] = True return parsed_map[final_section_name]["has_firewall"] = False
def _parse_hostname_section(imap, parsed_map): sec_id = 'ID_22' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return hnamedata = {} hname_section = imap[raw_section_name][0] for line in hname_section: if line == '\n' or line == '.' or 'hostname' in line: continue else: hnamedata['hosts'] = line.rstrip().split() break parsed_map[final_section_name] = hnamedata
def _parse_sysctlall_section(imap, parsed_map): sec_id = 'ID_109' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return sysctlall_section = imap[raw_section_name][0] parsed_map[final_section_name] = {} for line in sysctlall_section: if line == "": continue lineobj = line.rstrip().split('=') key = str(lineobj[0]) val = str(lineobj[1]) parsed_map[final_section_name][key.strip()] = val.strip()
def _parse_uname_section(imap, parsed_map): sec_id = 'ID_24' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return unamedata = {} uname_section = imap[raw_section_name][0] for line in uname_section: if re.search('uname -a', line) or line.strip() == '': continue # "Linux e-asmem-01.ame.admarketplace.net 2.6.32-279.el6.x86_64 #1 SMP Fri Jun 22 12:19:21 UTC 2012 x86_64 x86_64 x86_64 GNU/Linux\n" l = re.split('\ +', (line.split('#')[0])) unamedata['kernel_name'] = l[0] unamedata['nodename'] = l[1] unamedata['kernel_release'] = l[2] break parsed_map[final_section_name] = unamedata
def _parse_environment_section(imap, parsed_map): sec_id = 'ID_112' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return env_section = imap[raw_section_name][0] platform = "baremetal" for line in env_section: if line.strip() == "meta-data": platform = "aws" break if final_section_name not in parsed_map: parsed_map[final_section_name] = {} parsed_map[final_section_name]["platform"] = platform
def _parse_interrupts_section(imap, parsed_map): sec_id = 'ID_93' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return irq_section = imap[raw_section_name][0] tok_list = [] int_list = [] for line in irq_section: if 'cat /proc' in line or line == '\n': continue if 'CPU' in line: cpu_tok = line.rstrip().split() continue if 'Tx_rx' in line: tok_list = line.rstrip().split() device_name = tok_list[-1] int_type = tok_list[-2] int_id = tok_list[0] cpu_list = tok_list[1:-2] dev_obj = {} dev_obj['device_name'] = device_name dev_obj['interrupt_id'] = int_id.replace(':', '') dev_obj['interrupt_type'] = int_type dev_obj['interrupts'] = {} for idx, cpu in enumerate(cpu_tok): dev_obj['interrupts'][cpu] = cpu_list[idx] int_list.append(dev_obj) parsed_map[final_section_name] = {} parsed_map[final_section_name]['device_interrupts'] = int_list
def _parse_limits_section(imap, parsed_map): sec_id = 'ID_111' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return limits = {} limits_section = imap[raw_section_name][0] for line in limits_section: if "Max" not in line: continue lineobj = filter(None, line.rstrip().split(' ')) key = str(lineobj[0]).strip() limits["Soft " + key] = str(lineobj[1]).strip() limits["Hard " + key] = str(lineobj[2]).strip() parsed_map[final_section_name] = limits
def _parse_iostat_section(imap, parsed_map): sec_id = 'ID_43' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return iostat_section = imap[raw_section_name][0] # Create a List of all instances of iostat data. section_list = [] start = False section = [] for line in iostat_section: if 'avg-cpu' in line and 'user' in line: if start: section_list.append(section) section = [] start = True section.append(line) section_list.append(section) iostat_data = [] avgcpu_line = False #tok_cpuline = [] tok_cpuline = ['avg-cpu:', '%user', '%nice', '%system', '%iowait', '%steal', '%idle'] device_line = False tok_deviceline = [] tok_devicelist = ['Device:', 'rrqm/s', 'wrqm/s', 'r/s', 'w/s', 'rk_b/s', 'rkB/s', 'wk_b/s', 'wkB/s', 'avgrq-sz', 'avgqu-sz', 'await', 'r_await', 'w_await', 'svctm', '%util', 'rsec/s', 'wsec/s'] # Iterate over all instances and create list of maps for iostat_section in section_list: section_data = {} cpuobj = {} deviceobj_list = [] for line in iostat_section: deviceobj = {} if 'avg-cpu' in line and 'user' in line: avgcpu_line = True sectok_cpuline = line.rstrip().split() if tok_cpuline != sectok_cpuline: logger.error("iostat section format changed. old sec list: " + str(tok_cpuline) + " new sec list: " + str(sectok_cpuline)) return continue if 'Device:' in line and 'rrqm/s' in line: avgcpu_line = False device_line = True sectok_deviceline = line.rstrip().split() if set(sectok_deviceline).intersection(set(tok_devicelist)) == set(sectok_deviceline): tok_deviceline = sectok_deviceline else: logger.error("iostat section format changed. old sec union list: " + str(tok_devicelist) + " new sec list: " + str(sectok_deviceline)) return continue if avgcpu_line: data_list = line.rstrip().split() if len(data_list) + 1 != len(tok_cpuline): continue for idx, val in enumerate(data_list): cpuobj[tok_cpuline[idx + 1]] = val continue if device_line: data_list = line.rstrip().split() if len(data_list) != len(tok_deviceline): continue deviceobj[tok_deviceline[0].replace(':', '')] = data_list[0] for idx, val in enumerate(data_list): if idx == 0: continue deviceobj[tok_deviceline[idx]] = val deviceobj_list.append(deviceobj) section_data['avg-cpu'] = cpuobj section_data['device_stat'] = deviceobj_list # Change rkB/s -> rk_b/s and wkB/S -> wk_b/s _modify_keys_in_iostat_section(deviceobj_list) iostat_data.append(section_data) parsed_map[final_section_name] = {} parsed_map[final_section_name]['iostats'] = iostat_data
def _parse_lsb_release_section(imap, parsed_map): sec_id_1 = 'ID_25' raw_section_name_1, final_section_name_1, _ = get_section_name_from_id( sec_id_1) sec_id_2 = 'ID_26' raw_section_name_2, final_section_name_2, _ = get_section_name_from_id( sec_id_2) logger.info("Parsing section: " + final_section_name_1) if not imap: logger.warning("Null section json") return if raw_section_name_1 not in imap and raw_section_name_2 not in imap: logger.warning(raw_section_name_1 + " and " + raw_section_name_1 + " section not present.") return lsbdata = {} lsb_section_names = [raw_section_name_1, raw_section_name_2] for section in lsb_section_names: lsb_section_list = None if section not in imap: continue logger.info("Section: " + section) lsb_section_list = imap[section] if len(lsb_section_list) > 1: logger.warning( "More than one entries detected, There is a collision for this section: " + section) lsb_section = lsb_section_list[0] for index, line in enumerate(lsb_section): # "LSB Version:\t:base-4.0-amd64:base-4.0-noarch:core-4.0-amd64: # "Description:\t_cent_oS release 6.4 (Final)\n" matchobj = re.match(r'Description:\t(.*)', line) if matchobj: lsbdata['description'] = matchobj.group(1).strip() break # "Red Hat Enterprise Linux Server release 6.7 (Santiago)\n" # "Cent_oS release 6.7 (Final)\n" if re.search('.* release [0-9]+', line): lsbdata['description'] = line.strip() break # Few formats have only PRETTY_NAME, so need to add this condition. # "PRETTY_NAME=\"Ubuntu 14.04.2 LTS\"\n" matchobj = re.match(r'PRETTY_NAME=\"(.*)\"', line) if matchobj: lsbdata['description'] = matchobj.group(1) break if 'description' in lsbdata and ( 'amazon' in lsbdata['description'].lower() and 'ami' in lsbdata['description'].lower()): # For amazon linux ami for index, line in enumerate(lsb_section): matchobj = re.match(r'version=(.*)', line.lower()) if matchobj: v = matchobj.group(1).strip() try: v = v.split('.') y, m = v[0], v[1] if y.startswith("'") or y.startswith('"'): y = y[1:] if m.endswith("'") or m.endswith('"'): m = m[:-1] y = int(y) m = int(m) lsbdata['os_age_months'] = _get_age_month(y, m) except Exception: # Error while parsing version pass parsed_map[final_section_name_1] = lsbdata
def _parse_free_m_section(imap, parsed_map): sec_id = 'ID_37' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return free_m_data = {} tok_list = [] alltok_list = [ 'total', 'used', 'free', 'shared', 'buffers', 'cached', 'buff/cache', 'available' ] found_sec_start = False free_m_section = imap[raw_section_name][0] for line in free_m_section: if 'total' in line and 'used' in line and 'free' in line: sectok_list = line.rstrip().split() if set(sectok_list).intersection( set(alltok_list)) != set(sectok_list): logger.error("Free-m section format changed. union list: " + str(alltok_list) + " new sec list: " + str(sectok_list)) return tok_list = sectok_list found_sec_start = True if found_sec_start and 'Mem:' in line: data_list = line.rstrip().split() mem_obj = {} for idx, val in enumerate(tok_list): try: mem_obj[val] = int(data_list[idx + 1]) except Exception: mem_obj[val] = data_list[idx + 1] free_m_data['mem'] = mem_obj continue if found_sec_start and '-/+ buffers/cache:' in line: data_list = line.rstrip().split() buffer_obj = {} try: buffer_obj[tok_list[1]] = int(data_list[2]) except Exception: buffer_obj[tok_list[1]] = data_list[2] try: buffer_obj[tok_list[2]] = int(data_list[3]) except Exception: buffer_obj[tok_list[2]] = data_list[3] free_m_data['buffers/cache'] = buffer_obj continue if found_sec_start and 'Swap:' in line: data_list = line.rstrip().split() swap_obj = {} try: swap_obj[tok_list[0]] = int(data_list[1]) except Exception: swap_obj[tok_list[0]] = data_list[1] try: swap_obj[tok_list[1]] = int(data_list[2]) except Exception: swap_obj[tok_list[1]] = data_list[2] try: swap_obj[tok_list[2]] = int(data_list[3]) except Exception: swap_obj[tok_list[2]] = data_list[3] free_m_data['swap'] = swap_obj continue parsed_map[final_section_name] = free_m_data
def _parse_iostat_section(imap, parsed_map): sec_id = 'ID_43' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return iostat_section = imap[raw_section_name][0] # Create a List of all instances of iostat data. section_list = [] start = False section = [] for line in iostat_section: if 'avg-cpu' in line and 'user' in line: if start: section_list.append(section) section = [] start = True section.append(line) section_list.append(section) iostat_data = [] avgcpu_line = False #tok_cpuline = [] tok_cpuline = [ 'avg-cpu:', '%user', '%nice', '%system', '%iowait', '%steal', '%idle' ] device_line = False tok_deviceline = [] tok_devicelist = [ 'Device:', 'rrqm/s', 'wrqm/s', 'r/s', 'w/s', 'rk_b/s', 'rkB/s', 'wk_b/s', 'wkB/s', 'avgrq-sz', 'avgqu-sz', 'await', 'r_await', 'w_await', 'svctm', '%util', 'rsec/s', 'wsec/s' ] # Iterate over all instances and create list of maps for iostat_section in section_list: section_data = {} cpuobj = {} deviceobj_list = [] for line in iostat_section: deviceobj = {} if 'avg-cpu' in line and 'user' in line: avgcpu_line = True sectok_cpuline = line.rstrip().split() if tok_cpuline != sectok_cpuline: logger.error( "iostat section format changed. old sec list: " + str(tok_cpuline) + " new sec list: " + str(sectok_cpuline)) return continue if 'Device:' in line and 'rrqm/s' in line: avgcpu_line = False device_line = True sectok_deviceline = line.rstrip().split() if set(sectok_deviceline).intersection( set(tok_devicelist)) == set(sectok_deviceline): tok_deviceline = sectok_deviceline else: logger.error( "iostat section format changed. old sec union list: " + str(tok_devicelist) + " new sec list: " + str(sectok_deviceline)) return continue if avgcpu_line: data_list = line.rstrip().split() if len(data_list) + 1 != len(tok_cpuline): continue for idx, val in enumerate(data_list): cpuobj[tok_cpuline[idx + 1]] = val continue if device_line: data_list = line.rstrip().split() if len(data_list) != len(tok_deviceline): continue deviceobj[tok_deviceline[0].replace(':', '')] = data_list[0] for idx, val in enumerate(data_list): if idx == 0: continue deviceobj[tok_deviceline[idx]] = val deviceobj_list.append(deviceobj) section_data['avg-cpu'] = cpuobj section_data['device_stat'] = deviceobj_list # Change rkB/s -> rk_b/s and wkB/S -> wk_b/s _modify_keys_in_iostat_section(deviceobj_list) iostat_data.append(section_data) parsed_map[final_section_name] = {} parsed_map[final_section_name]['iostats'] = iostat_data
def _parse_top_section(imap, parsed_map): sec_id = 'ID_36' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return topdata = { 'uptime': {}, 'tasks': {}, 'cpu_utilization': {}, 'ram': {}, 'swap': {}, 'asd_process': {}, 'xdr_process': {} } top_section = imap[raw_section_name][0] asd_flag = False xdr_flag = False for index, line in enumerate(top_section): line = line.strip() if re.search('top -n3 -b', line): continue # Match object to get uptime in days. # "top - 18:56:45 up 103 days, 13:00, 2 users, load average: 1.29, 1.34, 1.35\n" matchobj_1 = re.match(r'.*up (.*?) days.*', line) # Match object to get total task running. # "Tasks: 149 total, 1 running, 148 sleeping, 0 stopped, 0 zombie\n" # Match object to get cpu utilization info. # "%Cpu(s): 11.3 us, 1.0 sy, 0.0 ni, 85.0 id, 1.7 wa, 0.0 hi, 0.7 si, 0.3 st\n" # Match object to get RAM info. # "Ki_b Mem: 62916356 total, 54829756 used, 8086600 free, 194440 buffers\n" # Match object to get Swap Mem info. # "Ki_b Swap: 0 total, 0 used, 0 free. 52694652 cached Mem\n" matchobj_2 = re.match( r'.*Swap:.* (.*?).total.* (.*?).used.* (.*?).free.* (.*?).ca.*', line) matchobj_3 = re.match( r'.*Swap:.* (.*?).total.* (.*?).free.* (.*?).used.* (.*?).av.*', line) obj = None if 'up' in line and 'load' in line: obj1 = re.match(r'.*up.* (.*?):(.*?),.* load .*', line) obj2 = re.match(r'.* (.*?) min', line) hr = 0 mn = 0 days = 0 if matchobj_1: days = int(matchobj_1.group(1)) if obj1: hr = int(obj1.group(1)) mn = int(obj1.group(2)) if obj2: mn = int(obj2.group(1)) topdata['uptime']['seconds'] = (days * 24 * 60 * 60) + (hr * 60 * 60) + (mn * 60) #topdata['uptime']['days'] = matchobj_1.group(1) if re.search(r'Tasks.*total', line): obj = _parse_top_section_line(line, ',', [' ']) topdata['tasks'] = obj elif re.search(r'Cpu.*us', line): obj = _parse_top_section_line(line, ',', [' ', '%']) topdata['cpu_utilization'] = obj elif re.search(r'Mem.*total', line): shift = 1 if 'Ki_b' in line or 'KiB' in line: shift = 10 if 'Mi_b' in line or 'MiB' in line: shift = 20 obj = _parse_top_section_line(line, ',', [' ', '+']) topdata['ram'] = obj for mem in topdata['ram']: topdata['ram'][mem] = _get_mem_in_byte_from_str( topdata['ram'][mem], 1, shift=shift) elif matchobj_2 or matchobj_3: shift = 1 if 'Ki_b' in line or 'KiB' in line: shift = 10 if 'Mi_b' in line or 'MiB' in line: shift = 20 if matchobj_2: topdata['swap']['total'] = matchobj_2.group(1) topdata['swap']['used'] = matchobj_2.group(2) topdata['swap']['free'] = matchobj_2.group(3) topdata['swap']['cached'] = matchobj_2.group(4) elif matchobj_3: topdata['swap']['total'] = matchobj_3.group(1) topdata['swap']['free'] = matchobj_3.group(2) topdata['swap']['used'] = matchobj_3.group(3) topdata['swap']['avail'] = matchobj_3.group(4) for mem in topdata['swap']: topdata['swap'][mem] = _get_mem_in_byte_from_str( topdata['swap'][mem], 1, shift=shift) else: # Break, If we found data for both process. # Also break if it chacked more the top 15 process. if (asd_flag and xdr_flag) or index > 25: break # " PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND\n" # "26937 root 20 0 59.975g 0.049t 0.048t S 117.6 83.9 164251:27 asd\n" if not asd_flag and re.search('asd', line): asd_flag = True l = re.split('\ +', line) topdata['asd_process']['virtual_memory'] = l[4] topdata['asd_process']['resident_memory'] = l[5] topdata['asd_process']['shared_memory'] = l[6] topdata['asd_process']['%cpu'] = l[8] topdata['asd_process']['%mem'] = l[9] for field in topdata['asd_process']: if field == '%cpu' or field == '%mem': continue topdata['asd_process'][field] = _get_mem_in_byte_from_str( topdata['asd_process'][field], 1) elif not xdr_flag and re.search('xdr', line): xdr_flag = True l = re.split('\ +', line) topdata['xdr_process']['virtual_memory'] = l[4] topdata['xdr_process']['resident_memory'] = l[5] topdata['xdr_process']['shared_memory'] = l[6] topdata['xdr_process']['%cpu'] = l[8] topdata['xdr_process']['%mem'] = l[9] for field in topdata['xdr_process']: if field == '%cpu' or field == '%mem': continue topdata['xdr_process'][field] = _get_mem_in_byte_from_str( topdata['xdr_process'][field], 1) _replace_comma_from_map_value_field(topdata) datalist = ['tasks', 'cpu_utilization', 'ram', 'swap'] for sec in datalist: if not topdata[sec] or len(topdata[sec]) == 0: logger.error( "Top format chaned. data could be missing. section: " + str(sec)) parsed_map[final_section_name] = topdata
def _parse_lsb_release_section(imap, parsed_map): sec_id_1 = 'ID_25' raw_section_name_1, final_section_name_1, _ = get_section_name_from_id(sec_id_1) sec_id_2 = 'ID_26' raw_section_name_2, final_section_name_2, _ = get_section_name_from_id(sec_id_2) logger.info("Parsing section: " + final_section_name_1) if not imap: logger.warning("Null section json") return if raw_section_name_1 not in imap and raw_section_name_2 not in imap: logger.warning(raw_section_name_1 + " and " + raw_section_name_1 + " section not present.") return lsbdata = {} lsb_section_names = [raw_section_name_1, raw_section_name_2] for section in lsb_section_names: lsb_section_list = None if section not in imap: continue logger.info("Section: " + section) lsb_section_list = imap[section] if len(lsb_section_list) > 1: logger.warning( "More than one entries detected, There is a collision for this section: " + section) lsb_section = lsb_section_list[0] for index, line in enumerate(lsb_section): # "LSB Version:\t:base-4.0-amd64:base-4.0-noarch:core-4.0-amd64: # "Description:\t_cent_oS release 6.4 (Final)\n" matchobj = re.match(r'Description:\t(.*)', line) if matchobj: lsbdata['description'] = matchobj.group(1).strip() break # "Red Hat Enterprise Linux Server release 6.7 (Santiago)\n" # "Cent_oS release 6.7 (Final)\n" if re.search('.* release [0-9]+', line): lsbdata['description'] = line.strip() break # Few formats have only PRETTY_NAME, so need to add this condition. # "PRETTY_NAME=\"Ubuntu 14.04.2 LTS\"\n" matchobj = re.match(r'PRETTY_NAME=\"(.*)\"', line) if matchobj: lsbdata['description'] = matchobj.group(1) break if 'description' in lsbdata and ('amazon' in lsbdata['description'].lower() and 'ami' in lsbdata['description'].lower()): # For amazon linux ami for index, line in enumerate(lsb_section): matchobj = re.match(r'version=(.*)', line.lower()) if matchobj: v = matchobj.group(1).strip() try: v = v.split('.') y, m = v[0], v[1] if y.startswith("'") or y.startswith('"'): y = y[1:] if m.endswith("'") or m.endswith('"'): m = m[:-1] y = int(y) m = int(m) lsbdata['os_age_months'] = _get_age_month(y, m) except Exception: # Error while parsing version pass parsed_map[final_section_name_1] = lsbdata
def _parse_top_section(imap, parsed_map): sec_id = 'ID_36' raw_section_name, final_section_name, _ = get_section_name_from_id(sec_id) logger.info("Parsing section: " + final_section_name) if not is_valid_section(imap, raw_section_name, final_section_name): return topdata = {'uptime': {}, 'tasks': {}, 'cpu_utilization': {}, 'ram': {}, 'swap': {}, 'asd_process': {}, 'xdr_process': {}} top_section = imap[raw_section_name][0] asd_flag = False xdr_flag = False kib_format = False for index, line in enumerate(top_section): line = line.strip() if re.search('top -n3 -b', line): continue if 'Ki_b' in line or 'KiB' in line: kib_format = True # Match object to get uptime in days. # "top - 18:56:45 up 103 days, 13:00, 2 users, load average: 1.29, 1.34, 1.35\n" matchobj_1 = re.match(r'.*up (.*?) days.*', line) # Match object to get total task running. # "Tasks: 149 total, 1 running, 148 sleeping, 0 stopped, 0 zombie\n" # Match object to get cpu utilization info. # "%Cpu(s): 11.3 us, 1.0 sy, 0.0 ni, 85.0 id, 1.7 wa, 0.0 hi, 0.7 si, 0.3 st\n" # Match object to get RAM info. # "Ki_b Mem: 62916356 total, 54829756 used, 8086600 free, 194440 buffers\n" # Match object to get Swap Mem info. # "Ki_b Swap: 0 total, 0 used, 0 free. 52694652 cached Mem\n" matchobj_2 = re.match( r'.*Swap:.* (.*?).total.* (.*?).used.* (.*?).free.* (.*?).ca.*', line) matchobj_3 = re.match( r'.*Swap:.* (.*?).total.* (.*?).free.* (.*?).used.* (.*?).av.*', line) obj = None if 'up' in line and 'load' in line: obj1 = re.match(r'.*up.* (.*?):(.*?),.* load .*', line) obj2 = re.match(r'.* (.*?) min', line) hr = 0 mn = 0 days = 0 if matchobj_1: days = int(matchobj_1.group(1)) if obj1: hr = int(obj1.group(1)) mn = int(obj1.group(2)) if obj2: mn = int(obj2.group(1)) topdata['uptime']['seconds'] = ( days * 24 * 60 * 60) + (hr * 60 * 60) + (mn * 60) #topdata['uptime']['days'] = matchobj_1.group(1) if re.search(r'Tasks.*total', line): obj = _parse_top_section_line(line, ',', [' ']) topdata['tasks'] = obj elif re.search(r'Cpu.*us', line): obj = _parse_top_section_line(line, ',', [' ', '%']) topdata['cpu_utilization'] = obj elif re.search(r'Mem.*total', line): obj = _parse_top_section_line(line, ',', [' ', '+']) topdata['ram'] = obj for mem in topdata['ram']: topdata['ram'][mem] = _get_mem_in_byte_from_str(topdata['ram'][mem], 1) elif matchobj_2 or matchobj_3: if matchobj_2: topdata['swap']['total'] = matchobj_2.group(1) topdata['swap']['used'] = matchobj_2.group(2) topdata['swap']['free'] = matchobj_2.group(3) topdata['swap']['cached'] = matchobj_2.group(4) elif matchobj_3: topdata['swap']['total'] = matchobj_3.group(1) topdata['swap']['free'] = matchobj_3.group(2) topdata['swap']['used'] = matchobj_3.group(3) topdata['swap']['avail'] = matchobj_3.group(4) for mem in topdata['swap']: topdata['swap'][mem] = _get_mem_in_byte_from_str(topdata['swap'][mem], 1) else: # Break, If we found data for both process. # Also break if it chacked more the top 15 process. if (asd_flag and xdr_flag) or index > 25: break # " PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND\n" # "26937 root 20 0 59.975g 0.049t 0.048t S 117.6 83.9 164251:27 asd\n" if not asd_flag and re.search('asd', line): asd_flag = True l = re.split('\ +', line) topdata['asd_process']['virtual_memory'] = l[4] topdata['asd_process']['resident_memory'] = l[5] topdata['asd_process']['shared_memory'] = l[6] topdata['asd_process']['%cpu'] = l[8] topdata['asd_process']['%mem'] = l[9] for field in topdata['asd_process']: if field == '%cpu' or field == '%mem': continue topdata['asd_process'][field] = _get_mem_in_byte_from_str(topdata['asd_process'][field], 1) elif not xdr_flag and re.search('xdr', line): xdr_flag = True l = re.split('\ +', line) topdata['xdr_process']['virtual_memory'] = l[4] topdata['xdr_process']['resident_memory'] = l[5] topdata['xdr_process']['shared_memory'] = l[6] topdata['xdr_process']['%cpu'] = l[8] topdata['xdr_process']['%mem'] = l[9] for field in topdata['xdr_process']: if field == '%cpu' or field == '%mem': continue topdata['xdr_process'][field] = _get_mem_in_byte_from_str( topdata['xdr_process'][field], 1) if kib_format: for key in topdata['ram']: try: topdata['ram'][key] = _get_bytes_from_float(topdata['ram'][key], 10, 0) except Exception: pass for key in topdata['swap']: try: topdata['swap'][key] = _get_bytes_from_float(topdata['swap'][key], 10, 0) except Exception: pass _replace_comma_from_map_value_field(topdata) datalist = ['tasks', 'cpu_utilization', 'ram', 'swap'] for sec in datalist: if not topdata[sec] or len(topdata[sec]) == 0: logger.error( "Top format chaned. data could be missing. section: " + str(sec)) parsed_map[final_section_name] = topdata
def _parse_awsdata_section(imap, parsed_map): sec_id_1 = 'ID_70' raw_section_name_1, final_section_name_1, _ = get_section_name_from_id( sec_id_1) sec_id_2 = 'ID_85' raw_section_name_2, final_section_name_2, _ = get_section_name_from_id( sec_id_2) logger.info("Parsing section: " + final_section_name_1) if not imap: logger.warning("Null section json") return awsdata = {} field_count = 0 total_fields = 2 aws_section_list = None # If both sections are present, select which has more number of lines. if raw_section_name_1 in imap and raw_section_name_2 in imap: if len(imap[raw_section_name_1]) > len(imap[raw_section_name_2]): aws_section_list = imap[raw_section_name_1] else: aws_section_list = imap[raw_section_name_2] elif raw_section_name_1 in imap: aws_section_list = imap[raw_section_name_1] elif raw_section_name_2 in imap: aws_section_list = imap[raw_section_name_2] else: logger.warning(raw_section_name_1 + " and " + raw_section_name_2 + " section is not present in section json.") return if len(aws_section_list) > 1: logger.warning( "More than one entries detected, There is a collision for this section(aws_info)." ) aws_section = aws_section_list[0] for index, line in enumerate(aws_section): if field_count >= total_fields: break if 'in_aWS' not in awsdata and re.search("This .* in AWS", line, re.IGNORECASE): awsdata['in_aws'] = True field_count += 1 continue if 'in_aWS' not in awsdata and re.search("not .* in aws", line, re.IGNORECASE): awsdata['in_aws'] = False field_count += 1 continue if 'instance_type' not in awsdata and re.search("instance-type", line): awsdata['instance_type'] = (aws_section[index + 1]).split('\n')[0] field_count += 1 if 'in_aWS' not in awsdata: awsdata['in_aws'] = True field_count += 1 continue parsed_map[final_section_name_1] = awsdata