Beispiel #1
0
class ServerMgrIPMIMonitoring():
    host_disklist = dict()
    host_nw_info_list = dict()
    _default_ipmi_username = None
    _default_ipmi_password = None
    DEBUG = "debug"
    INFO = "info"
    WARN = "warn"
    ERROR = "error"
    CRITICAL = "critical"
    _serverDb = None
    sleep_period = 10
    ssh_access_method = ""
    log_file = '/opt/contrail/server_manager/logger.conf'

    def __init__(self, val, frequency, smgr_ip=None, smgr_port=None, collectors_ip=None, introspect_port=None,
                 rev_tags_dict=None):
        ''' Constructor '''
        self.base_obj = ServerMgrMonBasePlugin()
        logging.config.fileConfig(self.log_file)
        # create logger
        self._monitoring_log = logging.getLogger('MONITORING')
        self.val = val
        self.smgr_ip = smgr_ip
        self.smgr_port = smgr_port
        self.introspect_port = introspect_port
        self.freq = float(frequency)
        self._collectors_ip = collectors_ip
        self.rev_tags_dict = rev_tags_dict
        self.ssh_access_method = "key"

    def log(self, level, msg):
        frame, filename, line_number, function_name, lines, index = inspect.stack()[1]
        log_dict = dict()
        log_dict['log_frame'] = frame
        log_dict['log_filename'] = os.path.basename(filename)
        log_dict['log_line_number'] = line_number
        log_dict['log_function_name'] = function_name
        log_dict['log_line'] = lines
        log_dict['log_index'] = index
        try:
            if level == self.DEBUG:
                self._monitoring_log.debug(msg, extra=log_dict)
            elif level == self.INFO:
                self._monitoring_log.info(msg, extra=log_dict)
            elif level == self.WARN:
                self._monitoring_log.warn(msg, extra=log_dict)
            elif level == self.ERROR:
                self._monitoring_log.error(msg, extra=log_dict)
            elif level == self.CRITICAL:
                self._monitoring_log.critical(msg, extra=log_dict)
        except Exception as e:
            print "Error logging msg in Mon" + e.message

    def set_serverdb(self, server_db):
        self._serverDb = server_db
        self.base_obj.set_serverdb(server_db=server_db)

    def set_ipmi_defaults(self, ipmi_username, ipmi_password):
        self._default_ipmi_username = ipmi_username
        self._default_ipmi_password = ipmi_password
        self.base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)

    # call_send function is the sending function of the sandesh object (send_inst)
    def call_send(self, send_inst):
        #self.log("info", "Sending UVE Info over Sandesh")
        send_inst.send()

    #This function returns the boolean value needed in the Sandesh Payload
    def str2bool(self,value):
        return value.lower() in ("true")

    # send_ipmi_stats function packages and sends the IPMI info gathered from server polling
    # to the analytics node
    def send_ipmi_stats(self, ip, ipmi_data, hostname, data_type):
        base_send_type = base_mon_uve.UVE_NONE
        if data_type == "ipmi_data":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.sensor_stats = []
            for ipmidata in ipmi_data:
                sm_ipmi_info.sensor_stats.append(ipmidata)
        elif data_type == "ipmi_chassis_data":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.chassis_state = ipmi_data
        elif data_type == "disk_list":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.disk_usage_stats = []
            for data in ipmi_data:
                sm_ipmi_info.disk_usage_stats.append(data)
        elif data_type == "disk_list_tot":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.disk_usage_totals = []
            for data in ipmi_data:
                sm_ipmi_info.disk_usage_totals.append(data)
        elif data_type == "resource_info_stats":
            base_send_type = base_mon_uve.UVE_SUMMARY
            sm_ipmi_info = ServerMonitoringSummary()
            sm_ipmi_info.resource_info_stats = ipmi_data
        elif data_type == "intinfo_list_tot":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.network_info_totals = []
            for data in ipmi_data:
                sm_ipmi_info.network_info_totals.append(data)
        elif data_type == "intinfo_list":
            base_send_type = base_mon_uve.UVE_SUMMARY
            sm_ipmi_info = ServerMonitoringSummary()
            sm_ipmi_info.network_info_stats = []
            for data in ipmi_data:
                sm_ipmi_info.network_info_stats.append(data)
        elif data_type == "file_system_view_list":
            base_send_type = base_mon_uve.UVE_INFO
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.file_system_view_stats = []
            for data in ipmi_data:
                sm_ipmi_info.file_system_view_stats.append(data)
            # self.log("info", "Sending Monitoring UVE Info for: " + str(data_type))
            # self.log("info", "UVE Interface Info = " + str(sm_ipmi_info))

        # assign hostname
        sm_ipmi_info.name = str(hostname)
        sm_ipmi_info.deleted = False
        # Send info based on base type
        if base_send_type == base_mon_uve.UVE_INFO:
            ipmi_stats_trace = ServerMonitoringInfoUve(data=sm_ipmi_info)
        elif base_send_type == base_mon_uve.UVE_SUMMARY:
            ipmi_stats_trace = ServerMonitoringSummaryUve(data=sm_ipmi_info)
        elif base_send_type == base_mon_uve.UVE_NONE:
            self.log("error", "Error base mon uve type for " + str(data_type))

        # self.log("info", "UVE Info Sent= " + str(sm_ipmi_info))
        self.call_send(ipmi_stats_trace)

    # Packages and sends a REST API call to the ServerManager node
    def send_run_inventory_request(self, ip, port, payload):
        try:
            url = "http://%s:%s/run_inventory" % (ip, port)
            args_str = ''
            match_key, match_value = payload.popitem()
            if match_key and match_value:
                args_str += urllib.quote_plus(str(match_key)) + "=" \
                            + urllib.quote_plus(str(match_value))
            if args_str != '':
                url += "?" + args_str
            headers = {'content-type': 'application/json'}
            resp = requests.post(url, headers=headers, timeout=300)
            return resp.text
        except Exception as e:
            self.log("error", "Error running inventory on  " + str(payload) + " : " + str(e))
            return None

    def return_collector_ip(self):
        return self._collectors_ip

    def isfloat(self, num):
        try:
            float(num)
        except ValueError:
           return False
        return True

    def fetch_and_process_monitoring(self, hostname, ipmi, ip, username, password, supported_sensors):
        try:
            ipmi_data = []
            cmd = 'ipmitool -H %s -U %s -P %s sdr list all' % (ipmi, username, password)
            result = self.base_obj.call_subprocess(cmd)
            if result is not None and "|" in result:
                fileoutput = cStringIO.StringIO(result)
                for line in fileoutput:
                    reading = line.split("|")
                    sensor = reading[0].strip()
                    reading_value = reading[1].strip()
                    status = reading[2].strip()
                    for i in supported_sensors:
                        if re.search(i, sensor, re.IGNORECASE) is not None:
                            sensor_type = 'unknown'
                            if 'FAN' in sensor or 'fan' in sensor or 'Fan' in sensor:
                                sensor_type = 'fan'
                            elif 'PWR' in sensor or 'Power' in sensor:
                                sensor_type = 'power'
                            elif 'Temp' in sensor or 'TEMP' in sensor or 'temp' in sensor:
                                sensor_type = 'temperature'
                            value = reading_value.split()
                            ipmidata = IpmiSensor()
                            ipmidata.sensor = sensor
                            ipmidata.status = status
                            if status == "ns":
                                pass
                            elif status == "ok" and self.isfloat(value[0].strip()):
                                ipmidata.reading = long(value[0].strip())
                                ipmidata.unit = value[len(value) - 1].strip()
                                ipmidata.sensor_type = sensor_type
                                ipmi_data.append(ipmidata)
                self.send_ipmi_stats(ip, ipmi_data, hostname, "ipmi_data")
                return True
            else:
                self.log("error", "Error getting Sesnsor info for: " + str(hostname) +
                         " Data returned was: " + str(result))
                return False
        except Exception as e:
            self.log("error", "Error getting Sesnsor info for: " + str(hostname) + " Error is: " + str(e))
            raise

    def fetch_and_process_chassis(self, hostname, ipmi, ip, username, password):
        ipmi_chassis_data = IpmiChassis_status_info()
        cmd = 'ipmitool -H %s -U %s -P %s chassis status' % (ipmi, username, password)
        try:
            result = self.base_obj.call_subprocess(cmd)
            if result is not None:
                fileoutput = cStringIO.StringIO(result)
                ipmichassisdata = IpmiChassis_status_info()
                for line in fileoutput:
                    if len(line.split(":")) >= 2:
                        reading = line.split(":")
                        chassis_key = reading[0].strip()
                        chassis_value = reading[1].strip()
                        if chassis_key == "System Power":
                            ipmichassisdata.system_power = chassis_value
                        elif chassis_key == "Power Overload" and chassis_value:
                            ipmichassisdata.power_overload = self.str2bool(chassis_value)
                        elif chassis_key == "Power Interlock" and chassis_value:
                            ipmichassisdata.power_interlock = chassis_value
                        elif chassis_key == "Main Power Fault" and chassis_value:
                            ipmichassisdata.main_power_fault = self.str2bool(chassis_value)
                        elif chassis_key == "Power Control Fault" and chassis_value:
                            ipmichassisdata.power_control_fault = self.str2bool(chassis_value)
                        elif chassis_key == "Power Restore Policy" and chassis_value:
                            ipmichassisdata.power_restore_policy = chassis_value
                        elif chassis_key == "Last Power Event" and chassis_value:
                            ipmichassisdata.last_power_event = chassis_value
                        elif chassis_key == "Chassis Intrusion" and chassis_value:
                            ipmichassisdata.chassis_intrusion = chassis_value
                        elif chassis_key == "Front-Panel Lockout" and chassis_value:
                            ipmichassisdata.front_panel_lockout = chassis_value
                        elif chassis_key == "Drive Fault" and chassis_value:
                            ipmichassisdata.drive_fault = self.str2bool(chassis_value)
                        elif chassis_value:
                            ipmichassisdata.cooling_fan_fault = self.str2bool(chassis_value)
                    else:
                        pass
                ipmi_chassis_data = ipmichassisdata
            self.send_ipmi_stats(ip, ipmi_chassis_data, hostname, "ipmi_chassis_data")
        except Exception as e:
            self.log("error", "Error getting chassis data for " + str(hostname) + " : " + str(e.message))
            raise

    def fetch_and_process_disk_info(self, hostname, ip, sshclient):
        disk_list = []
        disk_list_tot = []
        cmd = 'iostat -m'

        is_sysstat = sshclient.exec_command('which iostat')
        if not is_sysstat:
            #self.log("info", "sysstat package not installed on " + str(ip))
            disk_data = Disk()
            disk_data_tot = Disk_totals()
            disk_data.disk_name = "N/A"
            disk_data.read_bytes = int(0)
            disk_data.write_bytes = int(0)
            disk_data_tot.total_read_bytes = int(0)
            disk_data_tot.total_write_bytes = int(0)
            disk_list.append(disk_data)
            disk_list_tot.append(disk_data_tot)
            self.send_ipmi_stats(ip, disk_list, hostname, "disk_list")
            self.send_ipmi_stats(ip, disk_list_tot, hostname, "disk_list_tot")
        else:
            try:
                filestr = sshclient.exec_command(cmd=cmd)
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    #lookup for host dictionary
                    if hostname not in self.host_disklist:
                        #if empty insert the disklist dictionary
                        dict_disk = dict()
                        self.host_disklist[hostname] = dict_disk
                    else:
                        # disklist entry found
                        dict_disk = self.host_disklist[hostname]

                    for line in fileoutput:
                        if line is not None:
                            if (line.find('sd') != -1 or line.find('dm') != -1) and line.find('Linux')!=0:
                                disk_data = Disk()
                                disk_data_tot = Disk_totals()
                                prev_disk_info = Disk_totals()
                                res = re.sub('\s+', ' ', line).strip()
                                arr = res.split()
                                disk_data.disk_name = arr[0]
                                disk_data_tot.disk_name = arr[0]
                                disk_data_tot.total_read_bytes = int(arr[4])
                                disk_data_tot.total_write_bytes = int(arr[5])
                                if len(dict_disk) != 0:
                                    if disk_data_tot.disk_name in dict_disk:
                                        disk_data.read_bytes = disk_data_tot.total_read_bytes - dict_disk[
                                            disk_data_tot.disk_name].total_read_bytes
                                        disk_data.write_bytes = disk_data_tot.total_write_bytes - dict_disk[
                                            disk_data_tot.disk_name].total_write_bytes
                                        disk_list.append(disk_data)

                                prev_disk_info.total_read_bytes = disk_data_tot.total_read_bytes
                                prev_disk_info.total_write_bytes = disk_data_tot.total_write_bytes
                                dict_disk[disk_data_tot.disk_name] = prev_disk_info
                                disk_list_tot.append(disk_data_tot)
                    if disk_list:
                        self.send_ipmi_stats(ip, disk_list, hostname, "disk_list")
                        #return True  no return
                    if disk_list_tot:
                        self.send_ipmi_stats(ip, disk_list_tot, hostname, "disk_list_tot")
                        return True
                    else:
                        return False
            except Exception as e:
                self.log("error", "Error getting disk info for " + str(hostname) + " : " + str(e))
                raise

    #This function gets the mounted file system view. This will be the output of the df command
    def fetch_and_process_file_system_view(self, hostname, ip, sshclient):
        try:
            fs_view = file_system_disk_view()
            file_system_view_list = fs_view.get_file_system_view(sshclient)
            self.send_ipmi_stats(ip, file_system_view_list, hostname, "file_system_view_list")
        except Exception as e:
            self.log("error", "Error getting file system view info for " + str(hostname) + " : " + str(e))
            raise e

    def fetch_and_process_resource_info(self, hostname, ip, sshclient):
        try:
            resource_info1 = resource_info()
            is_mpstat = sshclient.exec_command('which mpstat')
            if not is_mpstat:
                resource_info1.cpu_usage_percentage = 0.0
            else:
                cmd = 'mpstat'
                filestr = sshclient.exec_command(cmd=cmd)
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    idle_index = -1
                    for line in fileoutput:
                        res = re.sub('\s+', ' ', line).strip()
                        arr = res.split()
                        if idle_index != -1:
                            resource_info1.cpu_usage_percentage = (100.0 - float(arr[idle_index]))
                            break
                        if "%idle" not in arr:
                            continue
                        else:
                            idle_index = arr.index('%idle')
            is_vmstat = sshclient.exec_command('which vmstat')
            if not is_vmstat:
                resource_info1.mem_usage_mb = 0
                resource_info1.mem_usage_percent = 0
            else:
                cmd = 'vmstat -s | grep "used memory"'
                filestr = sshclient.exec_command(cmd=cmd)
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    for line in fileoutput:
                        arr = line.split()
                        resource_info1.mem_usage_mb = int(int(arr[0])/1024)
                cmd = 'vmstat -s | grep "total memory"'
                filestr = sshclient.exec_command(cmd=cmd)
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    for line in fileoutput:
                        arr = line.split()
                        resource_info1.mem_usage_percent = float("{0:.2f}".format(
                            (resource_info1.mem_usage_mb/float(int(arr[0])/1024))*100))

            self.send_ipmi_stats(ip, resource_info1, hostname, "resource_info_stats")
        except Exception as e:
            self.log("error", "Error in getting resource info for  " + str(hostname) + str(e))
            raise e

    def fetch_and_process_network_info(self, hostname, ip, sshclient):
        try:
            intinfo_list = []
            intinfo_list_tot = []
            phys_intf_checker = sshclient.exec_command("ls -l /sys/class/net/")
            phys_intf_list = list()
            if phys_intf_checker:
                checker_output = cStringIO.StringIO(phys_intf_checker)
                for line in checker_output:
                    line = str(line)
                    if "virtual" not in line and "total" not in line and len(line.split(' ')) > 8:
                        line = line.split('/')
                        phys_intf_list.append(line[len(line) - 1].rstrip('\n'))

            result = sshclient.exec_command("ls /sys/class/net/")
            if result:
                output = cStringIO.StringIO(result)
                net_dictinfo = None
                #lookup for host dictionary
                if hostname not in self.host_nw_info_list:
                    #if empty insert the net dict info
                    net_dictinfo = dict()
                    self.host_nw_info_list[hostname] = net_dictinfo
                else:
                    # disklist entry found
                    net_dictinfo = self.host_nw_info_list[hostname]

                for line in output:
                    intinfo = network_info()
                    intinfo_tot = network_info_totals()
                    prev_nw = network_info_totals()
                    if line.rstrip() in phys_intf_list:
                        cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/tx_bytes"
                        tx_bytes = sshclient.exec_command(cmd=cmd)
                        cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/tx_packets"
                        tx_packets = sshclient.exec_command(cmd=cmd)
                        cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/rx_bytes"
                        rx_bytes = sshclient.exec_command(cmd=cmd)
                        cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/rx_packets"
                        rx_packets = sshclient.exec_command(cmd=cmd)
                        intinfo_tot.interface_name = line.rstrip()
                        intinfo.interface_name = line.rstrip()
                        intinfo_tot.total_tx_bytes = int(tx_bytes.rstrip())
                        intinfo_tot.total_tx_packets = int(tx_packets.rstrip())
                        intinfo_tot.total_rx_bytes = int(rx_bytes.rstrip())
                        intinfo_tot.total_rx_packets = int(rx_packets.rstrip())
                        if len(net_dictinfo) != 0:
                            if intinfo_tot.interface_name in net_dictinfo:
                                intinfo.tx_bytes = intinfo_tot.total_tx_bytes - net_dictinfo[
                                    intinfo_tot.interface_name].total_tx_bytes
                                intinfo.rx_bytes = intinfo_tot.total_rx_bytes - net_dictinfo[
                                    intinfo_tot.interface_name].total_rx_bytes
                                intinfo.tx_packets = intinfo_tot.total_tx_packets - net_dictinfo[
                                    intinfo_tot.interface_name].total_tx_packets
                                intinfo.rx_packets = intinfo_tot.total_rx_packets - net_dictinfo[
                                    intinfo_tot.interface_name].total_rx_packets
                                intinfo_list.append(intinfo)
                        prev_nw.total_tx_bytes = intinfo_tot.total_tx_bytes
                        prev_nw.total_rx_bytes = intinfo_tot.total_rx_bytes
                        prev_nw.total_tx_packets = intinfo_tot.total_tx_packets
                        prev_nw.total_rx_packets = intinfo_tot.total_rx_packets
                        net_dictinfo[intinfo_tot.interface_name] = prev_nw
                        intinfo_list_tot.append(intinfo_tot)
                if intinfo_list:
                    self.send_ipmi_stats(ip, intinfo_list, hostname, "intinfo_list")
                if intinfo_list_tot:
                    self.send_ipmi_stats(ip, intinfo_list_tot, hostname, "intinfo_list_tot")
                    return True
                else:
                    return False
        except Exception as e:
            self.log("error", "Error in getting network info for " + str(hostname) + " Error: " + str(e))
            raise e

    def fetch_and_process_sel_logs(self, hostname, ip, username, password, sel_event_log_list):
        sel_cmd = 'ipmitool -H %s -U %s -P %s sel elist' % (ip, username, password)
        sel_result = self.base_obj.call_subprocess(sel_cmd)
        try:
            if sel_result is not None:
                fileoutput = cStringIO.StringIO(sel_result)
                for line in fileoutput:
                    sellog = IpmiSystemEventLog()
                    sellog.name = str(hostname)
                    col = line.split("|")
                    hex_event_id = col[0].strip()
                    if hex_event_id.isdigit():
                        event_id = int(hex_event_id, 16)
                    else:
                        event_id = None
                    if event_id and event_id not in sel_event_log_list:
                        sel_event_log_list.append(event_id)
                        sellog.event_id = event_id
                        sellog.ipmi_timestamp = str(col[1]) + " " + str(col[2])
                        sensor_data = str(col[3])
                        sensor_data = sensor_data.split(" ")
                        sellog.sensor_type = str(sensor_data[1]) + " " + str(sensor_data[2])
                        sellog.sensor_name = str(sensor_data[3])
                        if len(sensor_data) >= 5:
                            sellog.sensor_name += " " + str(sensor_data[4])
                        sellog.ipmi_message = str(col[4])
                        if len(col) >= 6:
                            sellog.ipmi_message += " " + str(col[5])
                        sellog.send()
                    else:
                        pass
            return sel_event_log_list
        except Exception as e:
            self.log("error", "Error getting SEL Logs for " + str(hostname) + " : " + str(e.message))
            raise e

    def delete_monitoring_info(self, hostname_list):
        for hostname in hostname_list:
            sm_ipmi_info = ServerMonitoringInfo()
            sm_ipmi_info.name = str(hostname)
            sm_ipmi_info.deleted = True
            sm_ipmi_info.chassis_state = None
            ipmi_stats_trace = ServerMonitoringInfoUve(data=sm_ipmi_info)
            self.call_send(ipmi_stats_trace)

    def gevent_runner_func(self, hostname, ipmi, ip, username, password, supported_sensors, ipmi_state,
                           sel_event_log_list, option="key"):
        return_dict = dict()
        #self.log("info", "Gevent Thread created for %s" % hostname)
        try:
            sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
            sshclient.connect(ip, hostname, option)
            self.fetch_and_process_resource_info(hostname, ip, sshclient)
            self.fetch_and_process_network_info(hostname, ip, sshclient)
            self.fetch_and_process_disk_info(hostname, ip, sshclient)
            self.fetch_and_process_file_system_view(hostname, ip, sshclient)
            sshclient.close()
        except Exception as e:
            self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e))
            sshclient.close()
            pass
        try:
            return_dict["ipmi_status"] = \
                self.fetch_and_process_monitoring(hostname, ipmi, ip, username, password, supported_sensors)
            self.fetch_and_process_chassis(hostname, ipmi, ip, username, password)
            if sel_event_log_list:
                return_dict["sel_log"] = \
                    self.fetch_and_process_sel_logs(hostname, ipmi, username, password, sel_event_log_list)
            else:
                return_dict["sel_log"] = self.fetch_and_process_sel_logs(hostname, ipmi, username, password, [])
            if not ipmi_state and return_dict["ipmi_status"]:
                # Trigger REST API CALL to inventory for Server Hostname
                payload = dict()
                payload["id"] = str(hostname)
                self.send_run_inventory_request(self.smgr_ip, self.smgr_port, payload=payload)
            # self.log("info", "Payload return dict:\n" + str(return_dict))
            return return_dict
        except Exception as e:
            self.log("error", "Error in getting monitoring info through IPMI for server id: " + str(hostname) + " Error : " + str(e))
            pass


    # ####### MONITORING GET INFO SECTION ###########

    def convert_type(self, field_dict):
        data_type = field_dict["@type"]
        if "#text" in field_dict:
            if data_type == "bool":
                return json.loads(field_dict["#text"])
            elif data_type == "double":
                return float(field_dict["#text"])
            elif data_type == "u64":
                return int(field_dict["#text"])
            else:
                return str(field_dict["#text"])
        else:
            return "N/A"

    # Filters the data returned from REST API call for requested information
    def filter_monitoring_results(self, xml_dict, type_list):
        return_dict = {}
        if "all" in type_list:
            return_dict = dict(xml_dict)
        else:
            selected_fields = set(xml_dict.keys()).intersection(type_list)
            for selected_field in selected_fields:
                return_dict[selected_field] = xml_dict[selected_field]
        return return_dict

    def get_mon_conf_details(self):
        list_return_dict = list()
        main_return_dict = dict()
        return_dict = dict()
        return_dict["monitoring_frequency"] = self.freq
        return_dict["http_introspect_port"] = self.introspect_port
        return_dict["analytics_node_ips"] = list(self._collectors_ip)
        main_return_dict["config"] = dict(return_dict)
        list_return_dict.append(main_return_dict)
        return json.dumps(list_return_dict)

    def get_monitoring_info(self):
        list_return_dict = list()
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        server_cluster_list = list()
        server_tag_dict_list = list()
        self.log("debug", "get_monitoring_info")
        #self.log("debug", "Entered get_monitoring_info " + str(datetime.now()))
        uve_name = "ServerMonitoringInfo"
        summary_uve_name = "ServerMonitoringSummary"
        try:
            entity = bottle.request
            ret_data = self.base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
            #self.log("debug", "Validated rest api params " + str(datetime.now()))
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
            if match_key == "tag":
                match_dict = self.base_obj.process_server_tags(self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            #self.log("debug", "Before server db read " + str(datetime.now()))
            if match_dict.keys():
                servers = self._serverDb.get_server(
                    match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            #self.log("debug", "After server read " + str(datetime.now()))
            #self.log("debug", "Getting monitoring info of following servers: " + str(server_hostname_list))
            if len(servers) == 1:
                url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name,
                                                    dict(servers[0])['id'])
                summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name,
                                                            dict(servers[0])['id'])
            else:
                url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name)
                summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name)
            headers = {'content-type': 'application/json'}
            #self.log("debug", "After get_sandesh_url, before REST API call " + str(datetime.now()))
            #time_before = time.time()
            resp = requests.get(url, timeout=300, headers=headers)
            sum_resp = requests.get(summary_url, timeout=300, headers=headers)
            xml_data = resp.text
            sum_xml_data = sum_resp.text
            #time_after = time.time()
            #time_sec = time_after - time_before
            #self.log("debug", "Sandesh REST API Call : Time taken = " + str(time_sec)
            #         + " Resp length = " + str(len(xml_data)))
            #self.log("debug", "After REST API call " + str(datetime.now()))
            time_before = time.time()
            data = xmltodict.parse(str(xml_data))
            sum_data = xmltodict.parse(str(sum_xml_data))
            #self.log("debug", "After XMLtoDict" + str(datetime.now()))
            time_after = time.time()
            time_sec = time_after - time_before
            #self.log("debug", "XMLtoDict Call : Time taken = " + str(time_sec))
            data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
            sum_data_dict = dict(sum_data["__" + str(summary_uve_name) + "Uve_list"])
            #self.log("debug", "Before  processing " + str(datetime.now()))
            parsed_data_list = self.base_obj.parse_sandesh_xml(data_dict, uve_name)
            sum_parsed_data_list = self.base_obj.parse_sandesh_xml(sum_data_dict, summary_uve_name)
            parsed_data_dict = dict()
            sum_parsed_data_dict = dict()
            if parsed_data_list and sum_parsed_data_list and servers:
                for parsed_server in parsed_data_list:
                    parsed_server = dict(parsed_server)
                    parsed_data_dict[str(parsed_server["data"]["name"])] = dict(parsed_server["data"])
                for sum_parsed_server in sum_parsed_data_list:
                    sum_parsed_server = dict(sum_parsed_server)
                    sum_parsed_data_dict[str(sum_parsed_server["data"]["name"])] = dict(sum_parsed_server["data"])
                for server in servers:
                    server = dict(server)
                    server_hostname = str(server['id'])
                    if server_hostname in parsed_data_dict.keys():
                        return_dict = dict()
                        return_dict["name"] = str(server['id'])
                        return_dict["cluster_id"] = server['cluster_id']
                        if str(server['id']) in parsed_data_dict.keys():
                            main_dict = self.filter_monitoring_results(
                                parsed_data_dict[str(server['id'])],
                                ret_data["type"])
                        else:
                            main_dict = {}
                        if str(server['id']) in sum_parsed_data_dict.keys():
                            summary_dict = self.filter_monitoring_results(
                                sum_parsed_data_dict[str(server['id'])],
                                ret_data["type"])
                        else:
                            summary_dict = {}
                        for summary_key in summary_dict:
                            main_dict[str(summary_key)] = summary_dict[summary_key]
                        return_dict[str(uve_name)] = main_dict
                        #self.log("info", "All Keys:" + str(return_dict.keys()))
                        #self.log("info", "All Smgr Keys:" + str(return_dict[str(uve_name)].keys()))
                        list_return_dict.append(return_dict)
                    else:
                        self.log(self.ERROR, "Server Details missing in cache. ")
                        self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
                        pass
            else:
                self.log(self.ERROR, "Server Details missing in db. ")
                pass
        except ServerMgrException as e:
            self.log("error", "Get Monitoring Info Exception: " + str(e.message))
            return_dict = {}
            list_return_dict = list()
            list_return_dict.append(return_dict)
            return json.dumps(list_return_dict)
        except Exception as e:
            self.log("error", "Get Monitoring Info Exception: " + str(e.message))
            return_dict = {}
            list_return_dict = list()
            list_return_dict.append(return_dict)
            return json.dumps(list_return_dict)
        #self.log("debug", "Exited get_monitoring_info " + str(datetime.now()))
        return json.dumps(list_return_dict)

    def get_monitoring_info_summary(self):
        list_return_dict = list()
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        server_cluster_list = list()
        server_tag_dict_list = list()
        self.log("debug", "get_monitoring_info_summary")
        #self.log("debug", "Entered get_monitoring_info " + str(datetime.now()))
        uve_name = "ServerMonitoringInfo"
        summary_uve_name = "ServerMonitoringSummary"
        try:
            entity = bottle.request
            ret_data = self.base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
            #self.log("debug", "Validated rest api params " + str(datetime.now()))
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
            if match_key == "tag":
                match_dict = self.base_obj.process_server_tags(self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            #self.log("debug", "Before server db read " + str(datetime.now()))
            if match_dict.keys():
                servers = self._serverDb.get_server(
                    match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            #self.log("debug", "After server read " + str(datetime.now()))
            #self.log("debug", "Getting monitoring info of following servers: " + str(server_hostname_list))
            if len(servers) == 1:
                summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name,
                                                            dict(servers[0])['id'])
            else:
                summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name)
            headers = {'content-type': 'application/json'}
            #self.log("debug", "After get_sandesh_url, before REST API call " + str(datetime.now()))
            #time_before = time.time()
            sum_resp = requests.get(summary_url, timeout=300, headers=headers)
            sum_xml_data = sum_resp.text
            #time_after = time.time()
            #time_sec = time_after - time_before
            #self.log("debug", "Sandesh REST API Call : Time taken = " + str(time_sec)
            #         + " Resp length = " + str(len(xml_data)))
            #self.log("debug", "After REST API call " + str(datetime.now()))
            time_before = time.time()
            sum_data = xmltodict.parse(str(sum_xml_data))
            #self.log("debug", "After XMLtoDict" + str(datetime.now()))
            time_after = time.time()
            time_sec = time_after - time_before
            #self.log("debug", "XMLtoDict Call : Time taken = " + str(time_sec))
            sum_data_dict = dict(sum_data["__" + str(summary_uve_name) + "Uve_list"])
            #self.log("debug", "Before  processing " + str(datetime.now()))
            sum_parsed_data_list = self.base_obj.parse_sandesh_xml(sum_data_dict, summary_uve_name)
            sum_parsed_data_dict = dict()
            if sum_parsed_data_list and servers:
                for sum_parsed_server in sum_parsed_data_list:
                    sum_parsed_server = dict(sum_parsed_server)
                    sum_parsed_data_dict[str(sum_parsed_server["data"]["name"])] = dict(sum_parsed_server["data"])
                for server in servers:
                    server = dict(server)
                    server_hostname = str(server['id'])
                    if server_hostname in sum_parsed_data_dict.keys():
                        return_dict = dict()
                        return_dict["name"] = str(server['id'])
                        return_dict["cluster_id"] = server['cluster_id']
                        return_dict[str(summary_uve_name)] = self.filter_monitoring_results(
                            sum_parsed_data_dict[str(server['id'])],
                            ret_data["type"]
                        )
                        return_dict[str(uve_name)] = return_dict[str(summary_uve_name)]
                        return_dict.pop(str(summary_uve_name))
                        list_return_dict.append(return_dict)
                    else:
                        self.log(self.ERROR, "Server Details missing in cache. ")
                        self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
                        pass
            else:
                self.log(self.ERROR, "Server Details missing in db. ")
                pass
        except ServerMgrException as e:
            self.log("error", "Get Monitoring Info Summary Exception: " + str(e.message))
            return_dict = {}
            list_return_dict = list()
            list_return_dict.append(return_dict)
            return json.dumps(list_return_dict)
        except Exception as e:
            self.log("error", "Get Monitoring Info Summary Exception: " + str(e.message))
            return_dict = {}
            list_return_dict = list()
            list_return_dict.append(return_dict)
            return json.dumps(list_return_dict)
        #self.log("debug", "Exited get_monitoring_info " + str(datetime.now()))
        return json.dumps(list_return_dict)

    def cleanup(self, obj):
        if obj:
            obj.kill()

    # The Thread's run function continually checks the list of servers in the Server Mgr DB and polls them.
    # It then calls other functions to send the information to the correct analytics server.
    def run(self):
        print "Starting monitoring thread"
        #self.log("info", "Starting monitoring thread")
        sel_log_dict = dict()
        ipmi_list = list()
        hostname_list = list()
        server_ip_list = list()
        ipmi_username_list = list()
        ipmi_password_list = list()
        ipmi_state = dict()
        supported_sensors = ['FAN|.*_FAN', '^PWR', '.*Temp', '.*_Power']
        while True:
            servers = self._serverDb.get_server(
                None, detail=True)
            old_server_set = set(hostname_list)
            del ipmi_list[:]
            del hostname_list[:]
            del server_ip_list[:]
            del ipmi_username_list[:]
            del ipmi_password_list[:]
            #self.base_obj.populate_server_data_lists(servers, ipmi_list, hostname_list, server_ip_list,
             #                                        ipmi_username_list, ipmi_password_list, "monitoring")
            server_dict = self.base_obj.create_server_dict(servers)
            hostname_list = list(server_dict.keys())
            new_server_set = set(hostname_list)
            deleted_servers = set(old_server_set.difference(new_server_set))

            if len(hostname_list) == 0:
                time.sleep(self.freq)
                continue
            if len(deleted_servers) > 0:
                #self.log("info", "Deleting monitoring info of certain servers that have been removed")
                #self.log("info", "Deleted servers: " + str(list(deleted_servers)))
                self.delete_monitoring_info(list(deleted_servers))
            self.log("info", "Started IPMI Polling")
            gevent_threads = dict()
            gevent_priority_queue = gevent_queue.PriorityQueue()
            for server_id in server_dict:
                gevent_priority_queue.put(dict(server_dict[str(server_id)]))
            total_no_of_servers = gevent_priority_queue.qsize()
            sleep_period = self.sleep_period
            servers_per_period = int(math.floor(float(total_no_of_servers / self.freq * sleep_period)))
            #self.log("info", "Total number of servers this round: " + str(total_no_of_servers))
            #self.log("info", "Servers per period this round: " + str(servers_per_period))

            sleep_period = 1
            time_set_success = False
            while not time_set_success:
                servers_per_period = int(math.floor(float(total_no_of_servers / self.freq * sleep_period)))
                if servers_per_period >= 1:
                    time_set_success = True
                else:
                    sleep_period += 1
            try:
                counter = servers_per_period
                times_slept = 0
                spawned = 0
                total_spawned = 0
                for server_id in server_dict:
                    counter -= 1
                    spawned += 1
                    total_spawned += 1
                    server = dict(server_dict[str(server_id)])
                    if server['id'] not in ipmi_state and server['id'] not in sel_log_dict:
                        ipmi_state[str(server['id'])] = True
                        sel_log_dict[str(server['id'])] = None
                    if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
                            in server and 'ipmi_password' in server and server['id'] and server['ip_address'] \
                            and server['ipmi_address'] and server['ipmi_username'] and server['ipmi_password']:
                        thread = gevent.spawn(
                            self.gevent_runner_func, server['id'], server['ipmi_address'], server['ip_address'],
                            server['ipmi_username'], server['ipmi_password'],
                            supported_sensors, ipmi_state[str(server['id'])], sel_log_dict[str(server['id'])],
                            self.ssh_access_method)
                        gevent_threads[str(server['id'])] = thread
                    else:
                        self.log("error", "Missing fields in server dictionary - skipping monitoring run")
                    if counter > 0:
                        pass
                    else:
                        #self.log("debug", "Round of Spawning completed. Sleeping for 10 secs. ")
                        #self.log("debug", "Number of gevents spawned this round: " + str(spawned))
                        #self.log("debug", "Total spawned: " + str(total_spawned))
                        time.sleep(sleep_period)
                        times_slept += 1
                        counter = servers_per_period
                        spawned = 0
                #self.log("debug", "Slept for " + str(times_slept*sleep_period) + " s, sleeping for an additional " +
                         #str(self.freq - times_slept * sleep_period))
                if (self.freq - times_slept*sleep_period) > 0:
                    time.sleep(max(self.freq-times_slept*sleep_period, 0))
                else:
                    #self.log("debug", "No additional sleep. ")
                    pass

                for hostname in gevent_threads:
                    thread = gevent_threads[str(hostname)]
                    if thread.successful() and thread.value:
                        return_dict = dict(thread.value)
                        ipmi_state[str(hostname)] = return_dict["ipmi_status"]
                        sel_log_dict[str(hostname)] = return_dict["sel_log"]
                        thread.kill()
                    else:
                        self.log("error", "Greenlet for server " + str(hostname) + " didn't return successfully: "
                                 + str(thread.get()))
                        thread.kill()
                        pass

            except Exception as e:
                self.log("error", "Exception occured while spawning gevents. Error = " + str(e))
                pass
class ServerMgrInventory():
    _serverDb = None
    _inventory_log = None
    _collectors_ip = None
    _default_ipmi_username = None
    _default_ipmi_password = None
    _base_obj = None
    DEBUG = "debug"
    INFO = "info"
    WARN = "warn"
    ERROR = "error"
    CRITICAL = "critical"

    def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
        ''' Constructor '''
        self._base_obj = ServerMgrMonBasePlugin()
        logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
        # create logger
        self._inventory_log = logging.getLogger('INVENTORY')
        self.smgr_ip = smgr_ip
        self.smgr_port = smgr_port
        self.introspect_port = introspect_port
        self.rev_tags_dict = rev_tags_dict
        self.ssh_access_method = "key"

    def set_serverdb(self, server_db):
        self._serverDb = server_db
        self._base_obj.set_serverdb(server_db=server_db)

    def set_ipmi_defaults(self, ipmi_username, ipmi_password):
        self._default_ipmi_username = ipmi_username
        self._default_ipmi_password = ipmi_password
        self._base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)

    def log(self, level, msg):
        frame, filename, line_number, function_name, lines, index = inspect.stack()[1]
        log_dict = dict()
        log_dict['log_frame'] = frame
        log_dict['log_filename'] = os.path.basename(filename)
        log_dict['log_line_number'] = line_number
        log_dict['log_function_name'] = function_name
        log_dict['log_line'] = lines
        log_dict['log_index'] = index
        try:
            if level == self.DEBUG:
                self._inventory_log.debug(msg, extra=log_dict)
            elif level == self.INFO:
                self._inventory_log.info(msg, extra=log_dict)
            elif level == self.WARN:
                self._inventory_log.warn(msg, extra=log_dict)
            elif level == self.ERROR:
                self._inventory_log.error(msg, extra=log_dict)
            elif level == self.CRITICAL:
                self._inventory_log.critical(msg, extra=log_dict)
        except Exception as e:
            print "Error logging msg in Inv" + e.message

    # call_send function is the sending function of the sandesh object (send_inst)
    def call_send(self, send_inst):
        #self.log(self.INFO, "Sending UVE Info over Sandesh")
        #self.log("info", "UVE Info = " + str(send_inst.data))
        send_inst.send()

    @staticmethod
    def fru_dict_init(hostname):
        fru_dict = dict()
        fru_dict['id'] = hostname
        fru_dict['fru_description'] = "N/A"
        fru_dict['chassis_type'] = "N/A"
        fru_dict['chassis_serial_number'] = "N/A"
        fru_dict['board_mfg_date'] = "N/A"
        fru_dict['board_manufacturer'] = "N/A"
        fru_dict['board_product_name'] = "N/A"
        fru_dict['board_serial_number'] = "N/A"
        fru_dict['board_part_number'] = "N/A"
        fru_dict['product_manfacturer'] = "N/A"
        fru_dict['product_name'] = "N/A"
        fru_dict['product_part_number'] = "N/A"
        return fru_dict

    @staticmethod
    def fru_obj_init(hostname):
        fru_info_obj = fru_info()
        fru_info_obj.fru_description = "N/A"
        fru_info_obj.chassis_type = "N/A"
        fru_info_obj.chassis_serial_number = "N/A"
        fru_info_obj.board_mfg_date = "N/A"
        fru_info_obj.board_manufacturer = "N/A"
        fru_info_obj.board_product_name = "N/A"
        fru_info_obj.board_serial_number = "N/A"
        fru_info_obj.board_part_number = "N/A"
        fru_info_obj.product_manfacturer = "N/A"
        fru_info_obj.product_name = "N/A"
        fru_info_obj.product_part_number = "N/A"
        return fru_info_obj

    def get_fru_info(self, hostname, ip, username, password):
        cmd = 'ipmitool -H %s -U %s -P %s fru' % (ip, username, password)
        try:
            result = self._base_obj.call_subprocess(cmd)
            sensor = ""
            if result:
                inventory_info_obj = ServerInventoryInfo()
                inventory_info_obj.name = hostname
                fileoutput = cStringIO.StringIO(result)
                fru_obj_list = list()
                fru_dict = self.fru_dict_init(hostname)
                fru_info_obj = self.fru_obj_init(hostname)
                for line in fileoutput:
                    if ":" in line:
                        reading = line.split(":")
                        sensor = reading[0].strip()
                        reading_value = reading[1].strip()
                        if reading_value == "":
                            reading_value = "N/A"
                    else:
                        sensor = ""
                    if sensor == "FRU Device Description":
                        fru_dict = self.fru_dict_init(hostname)
                        fru_info_obj = self.fru_obj_init(hostname)
                        fru_info_obj.fru_description = reading_value
                        fru_dict['fru_description'] = str(hostname) + " " + reading_value
                    elif sensor == "Chassis Type":
                        fru_info_obj.chassis_type = reading_value
                        fru_dict['chassis_type'] = reading_value
                    elif sensor == "Chassis Serial":
                        fru_info_obj.chassis_serial_number = reading_value
                        fru_dict['chassis_serial_number'] = reading_value
                    elif sensor == "Board Mfg Date":
                        fru_info_obj.board_mfg_date = reading_value
                        fru_dict['board_mfg_date'] = reading_value
                    elif sensor == "Board Mfg":
                        fru_info_obj.board_manufacturer = reading_value
                        fru_dict['board_manufacturer'] = reading_value
                    elif sensor == "Board Product":
                        fru_info_obj.board_product_name = reading_value
                        fru_dict['board_product_name'] = reading_value
                    elif sensor == "Board Serial":
                        fru_info_obj.board_serial_number = reading_value
                        fru_dict['board_serial_number'] = reading_value
                    elif sensor == "Board Part Number":
                        fru_info_obj.board_part_number = reading_value
                        fru_dict['board_part_number'] = reading_value
                    elif sensor == "Product Manufacturer":
                        fru_info_obj.product_manfacturer = reading_value
                        fru_dict['product_manfacturer'] = reading_value
                    elif sensor == "Product Name":
                        fru_info_obj.product_name = reading_value
                        fru_dict['product_name'] = reading_value
                    elif sensor == "Product Part Number":
                        fru_info_obj.product_part_number = reading_value
                        fru_dict['product_part_number'] = reading_value
                    elif fru_dict['fru_description'] != "N/A" and sensor == "":
                        fru_obj_list.append(fru_info_obj)
                        rc = self._serverDb.add_inventory(fru_dict)
                        if rc != 0:
                            self.log(self.ERROR, "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
                        fru_dict = self.fru_dict_init(hostname)
                        fru_info_obj = self.fru_obj_init(hostname)
                if fru_dict['fru_description'] != "N/A":
                    fru_obj_list.append(fru_info_obj)
                    rc = self._serverDb.add_inventory(fru_dict)
                    if rc != 0:
                        self.log(self.ERROR, "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
                inventory_info_obj.fru_infos = fru_obj_list
            else:
                self.log(self.INFO, "Could not get the FRU info for IP: %s" % ip)
                inventory_info_obj = ServerInventoryInfo()
                inventory_info_obj.name = hostname
                inventory_info_obj.fru_infos = None
            self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
        except Exception as e:
            self.log(self.ERROR, "Could not get the FRU info for IP " + str(ip) + " Error: %s" + str(e))
            inventory_info_obj = ServerInventoryInfo()
            inventory_info_obj.name = hostname
            inventory_info_obj.fru_infos = None
            self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
            raise e

    @staticmethod
    def inventory_lookup(key):
        inv_dict = {
            'hostname': 'name',
            'boardproductname': 'board_product_name',
            'boardserialnumber': 'board_serial_number',
            'boardmanufacturer': 'board_manufacturer',
            'hardwaremodel': 'hardware_model',
            'interfaces': 'interface_name', 'physicalprocessorcount': 'physical_processor_count',
            'processorcount': 'cpu_cores_count',
            'virtual': 'virtual_machine',
            'memorytotal'		: 'total_memory_mb',
            'operatingsystem'		: 'os','operatingsystemrelease'	: 'os_version',
            'osfamily':'os_family',
            'kernelversion':'kernel_version',
            'uptime_seconds':'uptime_seconds',
            'ipaddress'		:'ip_addr',
            'netmask'			:'netmask',
            'macaddress'			: 'macaddress'
        }
        if key in inv_dict:
            return inv_dict[key]
        else:
            return None


    def get_facter_info(self, hostname, ip, sshclient):
        server_inventory_info = ServerInventoryInfo()
        # Get the total number of disks
        numdisks = self.get_field_value(sshclient, ip, 'lsblk | grep disk | wc -l')
        is_ethtool = sshclient.exec_command('which ethtool')
        server_inventory_info.total_numof_disks = int(numdisks)
        # Get the other inventory information from the facter tool
        try:
            filestr = sshclient.exec_command('facter')
            fileoutput = cStringIO.StringIO(filestr)
            if fileoutput is not None:
                interface_dict = {}
                intinfo_list = []
                for line in fileoutput:
                    inventory = line.split('=>')
                    try:
                        key = inventory[0].strip()
                        if len(inventory) > 1:
                            value = inventory[1].strip()
                        if key == 'interfaces':
                            interface_list = value.split(',')
                            for name in interface_list:
                                # Skip the loopback interface
                                if name.strip() == 'lo':
                                    continue
                                intinfo = interface_info()
                                intinfo.interface_name = name
                                intinfo.speed_Mb_per_sec = 0
                                intinfo.model = "N/A"
                                if not is_ethtool:
                                    self.log(self.DEBUG, "ethtool not installed on host : %s" % ip)
                                else:
                                    eth_cmd = 'ethtool ' + name + ' | grep Speed'
                                    driver_cmd = 'ethtool -i ' + name + ' | grep driver'
                                    intinfo.speed_Mb_per_sec = self.get_field_value(sshclient, ip, eth_cmd)
                                    if bool(re.search(r'\d', str(intinfo.speed_Mb_per_sec))):
                                        temp_var = re.findall('\d+|\D+', str(intinfo.speed_Mb_per_sec))
                                        intinfo.speed_Mb_per_sec = int(temp_var[0].strip())
                                    else:
                                        intinfo.speed_Mb_per_sec = 0
                                    intinfo.model = self.get_field_value(sshclient, ip, driver_cmd)

                                exp = '.*_' + name + '.*$'
                                # exp = '(^ipaddress_|^macaddress_|^netmask_).*'+name+'.*$'
                                res = re.findall(exp, filestr, re.MULTILINE)
                                for items in res:
                                    actualkey = items.split('=>')
                                    namekey = actualkey[0].split('_')
                                    objkey = self.inventory_lookup(key=namekey[0].strip())
                                    value = actualkey[1].strip()
                                    if objkey:
                                        setattr(intinfo, objkey, value)
                                if not getattr(intinfo, 'macaddress'):
                                    setattr(intinfo, 'macaddress', "N/A")
                                if not getattr(intinfo, 'ip_addr'):
                                    setattr(intinfo, 'ip_addr', "N/A")
                                if not getattr(intinfo, 'netmask'):
                                    setattr(intinfo, 'netmask', "N/A")
                                intinfo_list.append(intinfo)
                        else:
                            objkey = self.inventory_lookup(key)
                            if key == 'physicalprocessorcount' or key == 'processorcount' or key == 'uptime_seconds':
                                value = int(value)
                            elif key == 'memorytotal':
                                memval = value.split()
                                value = math.trunc(float(memval[0]))
                                if memval[1].strip() == 'GB':
                                    value *= 1024
                            if objkey:
                                setattr(server_inventory_info, objkey, value)
                    except Exception as KeyError:
                        self.log(self.INFO, "keyerror: %s " + str(KeyError) + " for IP: %s" % ip)
                        continue
                server_inventory_info.name = str(hostname)
                server_inventory_info.interface_infos = intinfo_list
                self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
            else:
                self.log(self.ERROR, "Could not get the Facter info for IP: %s" % ip)
        except Exception as e:
            self.log(self.ERROR, "Could not get the Facter info for IP " + str(ip) + " Error: %s" + str(e))
            raise e

    def get_field_value(self, sshclient, ip, cmd):
        try:
            filestr = sshclient.exec_command(cmd)
            if not filestr:
                return None
            if cmd == "lsblk" or cmd == "facter" or "statistics" in cmd or cmd == "vmstat" or cmd == "lspci":
                return filestr
            else:
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    for line in fileoutput:
                        if "=>" in line:
                            value = line.rstrip("\n").split("=>")[1].lstrip()
                            return value
                        elif ":" not in line:
                            return line
                        value = line.rstrip("\n").split(":")[1].lstrip()
                        return value
        except Exception as e:
            self.log(self.ERROR, "Error in get_field_value: " + str(ip) + " Command = " + str(cmd) + "Error: " + str(e))
            return None

    def get_cpu_info(self, hostname, ip, sshclient):
        try:
            # Get the CPU information from server
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.cpu_info_state = cpu_info()
            check_cmd = 'cat /proc/cpuinfo'
            lscpu__cmd = 'which lscpu'
            server_inventory_info.cpu_info_state.model = "N/A"
            server_inventory_info.cpu_info_state.core_count = 0
            server_inventory_info.cpu_info_state.clock_speed_MHz = 0.0
            server_inventory_info.cpu_info_state.num_of_threads = 0
            if self.get_field_value(sshclient, ip, check_cmd) and self.get_field_value(sshclient, ip, lscpu__cmd):
                model_cmd = 'cat /proc/cpuinfo | grep "model name" | head -n 1'
                core_cmd = 'cat /proc/cpuinfo | grep "cpu cores" | head -n 1'
                clock_cmd = 'cat /proc/cpuinfo | grep "cpu MHz" | head -n 1'
                thread_cmd = 'lscpu | grep "Thread"'
                server_inventory_info.cpu_info_state.model = self.get_field_value(sshclient, ip, model_cmd)
                server_inventory_info.cpu_info_state.core_count = int(self.get_field_value(sshclient, ip, core_cmd))
                server_inventory_info.cpu_info_state.clock_speed_MHz = float(
                    self.get_field_value(sshclient, ip, clock_cmd))
                server_inventory_info.cpu_info_state.num_of_threads = int(
                    self.get_field_value(sshclient, ip, thread_cmd))
            else:
                self.log(self.DEBUG, "lscpu not installed on host : %s" % ip)
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(self.ERROR, "Error in get_cpu_info: " + str(ip) + "Error: " + str(e))
            raise e


    def get_ethernet_info(self, hostname, ip, sshclient):
        try:
            # Get the Ethernet information from server
            is_lspci = sshclient.exec_command('which lspci')
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.eth_controller_state = ethernet_controller()
            server_inventory_info.eth_controller_state.num_of_ports = 0
            if not is_lspci:
                self.log(self.DEBUG, "lspci not installed on host : %s" % ip)
            else:
                port_cmd = 'lspci | grep Net | wc -l'
                server_inventory_info.eth_controller_state.num_of_ports = int(
                    self.get_field_value(sshclient, ip, port_cmd))
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(self.ERROR, "Error in get_ethernet_info: " + str(ip) + "Error: " + str(e))
            raise e

    def get_memory_info(self, hostname, ip, sshclient):
        try:
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.mem_state = memory_info()
            server_inventory_info.mem_state.mem_type = "N/A"
            server_inventory_info.mem_state.mem_speed_MHz = 0
            server_inventory_info.mem_state.num_of_dimms = 0
            server_inventory_info.mem_state.dimm_size_mb = 0
            server_inventory_info.mem_state.total_mem__mb = 0
            server_inventory_info.mem_state.swap_size_mb = 0.0
            dmi_cmd = 'which dmidecode'
            if self.get_field_value(sshclient, ip, dmi_cmd):
                type_cmd = 'dmidecode -t memory |  grep "Type:" | grep -v "Unknown" | grep -v "Error" | head -n1'
                mem_cmd = 'dmidecode -t memory | grep "Speed" | grep -v "Unknown" | head -n1'
                dimm_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | head -n1'
                num_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | wc -l'
                swap_cmd = 'facter | egrep -w swapsize_mb'
                total_mem_cmd = 'vmstat -s | grep "total memory"'
                server_inventory_info.mem_state.mem_type = self.get_field_value(sshclient, ip, type_cmd)
                mem_speed = self.get_field_value(sshclient, ip, mem_cmd)
                unit = mem_speed.split(" ")[1]
                if unit == "MHz":
                    server_inventory_info.mem_state.mem_speed_MHz = int(mem_speed.split(" ")[0])
                dimm_size = self.get_field_value(sshclient, ip, dimm_cmd)
                unit = dimm_size.split(" ")[1]
                if unit == "MB":
                    server_inventory_info.mem_state.dimm_size_mb = int(dimm_size.split(" ")[0])
                server_inventory_info.mem_state.num_of_dimms = int(self.get_field_value(sshclient, ip, num_cmd))
                swap_size = self.get_field_value(sshclient, ip, swap_cmd)
                server_inventory_info.mem_state.swap_size_mb = float(swap_size.split(" ")[0])
                total_mem = self.get_field_value(sshclient, ip, total_mem_cmd)
                server_inventory_info.mem_state.total_mem_mb = int(int(total_mem.lstrip().split(" ")[0]) / 1024)
            else:
                self.log(self.INFO, "Couldn't get the Memory info for IP: %s" % ip)
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(self.ERROR, "Error in get_memory_info: " + str(ip) + "Error: " + str(e))
            raise e

    def add_inventory(self):
        servers = self._serverDb.get_server(None, detail=True)
        gevent.spawn(self.handle_inventory_trigger, "add", servers)

    def delete_inventory_info(self, hostname):
        inventory_info_obj = ServerInventoryInfo()
        inventory_info_obj.name = str(hostname)
        inventory_info_obj.deleted = True
        inventory_info_obj.fru_infos = None
        inventory_info_obj.interface_infos = None
        inventory_info_obj.cpu_info_state = None
        inventory_info_obj.eth_controller_state = None
        self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))

    def gevent_runner_function(self, action, hostname, ip=None, ipmi=None, username=None, password=None, option="key"):
        sshclient = None
        if action == "add" and ip and ipmi and username and password:
            try:
                sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
                sshclient.connect(ip, hostname, option)
                self.get_facter_info(hostname, ip, sshclient)
                self.get_cpu_info(hostname, ip, sshclient)
                self.get_ethernet_info(hostname, ip, sshclient)
                self.get_memory_info(hostname, ip, sshclient)
                sshclient.close()
            except Exception as e:
                if sshclient:
                    sshclient.close()
                self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e))
                pass
            try:
                self.get_fru_info(hostname, ipmi, username, password)
            except Exception as e:
                self.log("error", "Error in getting Inventory Info through IPMI for server id: " + str(hostname) + " Error : " + str(e))
                pass
        elif action == "delete":
            self.log(self.INFO, "Deleted info of server: %s" % hostname)
            self.delete_inventory_info(hostname)

    ######## INVENTORY GET INFO SECTION ###########

    def convert_type(self, field_dict):
        data_type = field_dict["@type"]
        if "#text" in field_dict:
            if data_type == "bool":
                return json.loads(field_dict["#text"])
            elif data_type == "double":
                return float(field_dict["#text"])
            elif data_type == "u64":
                return int(field_dict["#text"])
            else:
                return str(field_dict["#text"])
        else:
            return "N/A"

    def filter_inventory_results(self, xml_dict, type_list):
        return_dict = {}
        if "all" in type_list:
            return_dict = dict(xml_dict)
        else:
            selected_fields = set(xml_dict.keys()).intersection(type_list)
            for selected_field in selected_fields:
                return_dict[selected_field] = xml_dict[selected_field]
        return return_dict

    @staticmethod
    def get_inv_conf_details(self):
        return "Configuration for Inventory set correctly."

    def get_inventory_info(self):
        list_return_dict = list()
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        server_cluster_list = list()
        server_tag_dict_list = list()
        self.log(self.DEBUG, "get_inventory_info")
        uve_name = "ServerInventoryInfo"
        try:
            entity = bottle.request
            ret_data = self._base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
            if match_key == "tag":
                match_dict = self._base_obj.process_server_tags(self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            if match_dict.keys():
                servers = self._serverDb.get_server(
                    match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            #self.log(self.DEBUG, "Getting inventory info of following servers: " + str(servers))
            if len(servers) == 1:
                url = self._base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name,
                                                     dict(servers[0])['id'])
            else:
                url = self._base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name)
            headers = {'content-type': 'application/json'}
            resp = requests.get(url, timeout=300, headers=headers)
            xml_data = resp.text
            data = xmltodict.parse(str(xml_data))
            data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
            parsed_data_list = self._base_obj.parse_sandesh_xml(data_dict, uve_name)
            parsed_data_dict = dict()
            if parsed_data_list and servers:
                for parsed_server in parsed_data_list:
                    parsed_server = dict(parsed_server)
                    parsed_data_dict[str(parsed_server["data"]["name"])] = dict(parsed_server["data"])
                for server in servers:
                    server = dict(server)
                    server_hostname = str(server['id'])
                    if server_hostname in parsed_data_dict.keys():
                        return_dict = dict()
                        return_dict["name"] = str(server_hostname)
                        return_dict["cluster_id"] = server['cluster_id']
                        return_dict[str(uve_name)] = self.filter_inventory_results(
                            parsed_data_dict[str(server_hostname)],
                            ret_data["type"])
                        list_return_dict.append(return_dict)
                    else:
                        self.log(self.ERROR, "Server Details missing in cache. ")
                        self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
                        pass
            else:
                self.log(self.ERROR, "Server Details missing in db. ")
                return {}
        except ServerMgrException as e:
            self.log(self.ERROR, "Get Inventory Info Exception: " + str(e.message))
            return json.dumps({})
        except Exception as e:
            self.log(self.ERROR, "Get Inventory Info Exception: " + str(e.message))
            return json.dumps({})
        #self.log("debug", "Exited get_inventory_info " + str(datetime.now()))
        return json.dumps(list_return_dict)
        # end get_inventory_info

    def run_inventory(self):
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        self.log(self.DEBUG, "run_inventory")
        try:
            entity = bottle.request
            ret_data = self._base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
            #ret_data = self._base_obj.validate_run_inv_params(entity, self.rev_tags_dict)
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
            if match_key == "tag":
                match_dict = self._base_obj.process_server_tags(self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            if match_dict.keys():
                servers = self._serverDb.get_server(
                    match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            self.handle_inventory_trigger("add", servers)
        except ServerMgrException as e:
            self.log("error", "Run Inventory Exception: " + e.message)
            raise e
        except Exception as e:
            self.log("error", "Run Inventory Exception: " + e.message)
            raise e
        inventory_status = dict()
        inventory_status['return_message'] = "server(s) run_inventory issued"
        return inventory_status

    def handle_inventory_trigger(self, action, servers):
        server_dict = self._base_obj.create_server_dict(servers)

        gevent_threads = []
        if len(server_dict.keys()) >= 1:
            for server_id in server_dict:
                server = dict(server_dict[str(server_id)])
                if action == "add":
                    if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
                            in server and 'ipmi_password' in server:
                        thread = gevent.spawn(self.gevent_runner_function, action, server['id'], server['ip_address'],
                                              server['ipmi_address'], server['ipmi_username'], server['ipmi_password'],
                                              self.ssh_access_method)
                        gevent_threads.append(thread)
                    else:
                        self.log(self.ERROR, "Missing fields in server dictionary - skipping inventory addition")
                elif action == "delete":
                    if 'id' in server:
                        thread = gevent.spawn(self.gevent_runner_function, action, server['id'], self.ssh_access_method)
                        gevent_threads.append(thread)
                    else:
                        self.log(self.ERROR, "Missing id field in server dictionary - skipping inventory deletion")
                time.sleep(1)
        self.log(self.DEBUG, "Finished Running Inventory")
Beispiel #3
0
class ServerMgrInventory():
    _serverDb = None
    _inventory_log = None
    _collectors_ip = None
    _default_ipmi_username = None
    _default_ipmi_password = None
    _base_obj = None
    DEBUG = "debug"
    INFO = "info"
    WARN = "warn"
    ERROR = "error"
    CRITICAL = "critical"

    def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
        ''' Constructor '''
        self._base_obj = ServerMgrMonBasePlugin()
        logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
        # create logger
        self._inventory_log = logging.getLogger('INVENTORY')
        self.smgr_ip = smgr_ip
        self.smgr_port = smgr_port
        self.introspect_port = introspect_port
        self.rev_tags_dict = rev_tags_dict
        self.ssh_access_method = "key"

    def set_serverdb(self, server_db):
        self._serverDb = server_db
        self._base_obj.set_serverdb(server_db=server_db)

    def set_ipmi_defaults(self, ipmi_username, ipmi_password):
        self._default_ipmi_username = ipmi_username
        self._default_ipmi_password = ipmi_password
        self._base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)

    def log(self, level, msg):
        frame, filename, line_number, function_name, lines, index = inspect.stack(
        )[1]
        log_dict = dict()
        log_dict['log_frame'] = frame
        log_dict['log_filename'] = os.path.basename(filename)
        log_dict['log_line_number'] = line_number
        log_dict['log_function_name'] = function_name
        log_dict['log_line'] = lines
        log_dict['log_index'] = index
        try:
            if level == self.DEBUG:
                self._inventory_log.debug(msg, extra=log_dict)
            elif level == self.INFO:
                self._inventory_log.info(msg, extra=log_dict)
            elif level == self.WARN:
                self._inventory_log.warn(msg, extra=log_dict)
            elif level == self.ERROR:
                self._inventory_log.error(msg, extra=log_dict)
            elif level == self.CRITICAL:
                self._inventory_log.critical(msg, extra=log_dict)
        except Exception as e:
            print "Error logging msg in Inv" + e.message

    # call_send function is the sending function of the sandesh object (send_inst)
    def call_send(self, send_inst):
        #self.log(self.INFO, "Sending UVE Info over Sandesh")
        #self.log("info", "UVE Info = " + str(send_inst.data))
        send_inst.send()

    @staticmethod
    def fru_dict_init(hostname):
        fru_dict = dict()
        fru_dict['id'] = hostname
        fru_dict['fru_description'] = "N/A"
        fru_dict['chassis_type'] = "N/A"
        fru_dict['chassis_serial_number'] = "N/A"
        fru_dict['board_mfg_date'] = "N/A"
        fru_dict['board_manufacturer'] = "N/A"
        fru_dict['board_product_name'] = "N/A"
        fru_dict['board_serial_number'] = "N/A"
        fru_dict['board_part_number'] = "N/A"
        fru_dict['product_manfacturer'] = "N/A"
        fru_dict['product_name'] = "N/A"
        fru_dict['product_part_number'] = "N/A"
        return fru_dict

    @staticmethod
    def fru_obj_init(hostname):
        fru_info_obj = fru_info()
        fru_info_obj.fru_description = "N/A"
        fru_info_obj.chassis_type = "N/A"
        fru_info_obj.chassis_serial_number = "N/A"
        fru_info_obj.board_mfg_date = "N/A"
        fru_info_obj.board_manufacturer = "N/A"
        fru_info_obj.board_product_name = "N/A"
        fru_info_obj.board_serial_number = "N/A"
        fru_info_obj.board_part_number = "N/A"
        fru_info_obj.product_manfacturer = "N/A"
        fru_info_obj.product_name = "N/A"
        fru_info_obj.product_part_number = "N/A"
        return fru_info_obj

    def get_fru_info(self, hostname, ip, username, password):
        cmd = 'ipmitool -H %s -U %s -P %s fru' % (ip, username, password)
        try:
            result = self._base_obj.call_subprocess(cmd)
            sensor = ""
            if result:
                inventory_info_obj = ServerInventoryInfo()
                inventory_info_obj.name = hostname
                fileoutput = cStringIO.StringIO(result)
                fru_obj_list = list()
                fru_dict = self.fru_dict_init(hostname)
                fru_info_obj = self.fru_obj_init(hostname)
                for line in fileoutput:
                    if ":" in line:
                        reading = line.split(":")
                        sensor = reading[0].strip()
                        reading_value = reading[1].strip()
                        if reading_value == "":
                            reading_value = "N/A"
                    else:
                        sensor = ""
                    if sensor == "FRU Device Description":
                        fru_dict = self.fru_dict_init(hostname)
                        fru_info_obj = self.fru_obj_init(hostname)
                        fru_info_obj.fru_description = reading_value
                        fru_dict['fru_description'] = str(
                            hostname) + " " + reading_value
                    elif sensor == "Chassis Type":
                        fru_info_obj.chassis_type = reading_value
                        fru_dict['chassis_type'] = reading_value
                    elif sensor == "Chassis Serial":
                        fru_info_obj.chassis_serial_number = reading_value
                        fru_dict['chassis_serial_number'] = reading_value
                    elif sensor == "Board Mfg Date":
                        fru_info_obj.board_mfg_date = reading_value
                        fru_dict['board_mfg_date'] = reading_value
                    elif sensor == "Board Mfg":
                        fru_info_obj.board_manufacturer = reading_value
                        fru_dict['board_manufacturer'] = reading_value
                    elif sensor == "Board Product":
                        fru_info_obj.board_product_name = reading_value
                        fru_dict['board_product_name'] = reading_value
                    elif sensor == "Board Serial":
                        fru_info_obj.board_serial_number = reading_value
                        fru_dict['board_serial_number'] = reading_value
                    elif sensor == "Board Part Number":
                        fru_info_obj.board_part_number = reading_value
                        fru_dict['board_part_number'] = reading_value
                    elif sensor == "Product Manufacturer":
                        fru_info_obj.product_manfacturer = reading_value
                        fru_dict['product_manfacturer'] = reading_value
                    elif sensor == "Product Name":
                        fru_info_obj.product_name = reading_value
                        fru_dict['product_name'] = reading_value
                    elif sensor == "Product Part Number":
                        fru_info_obj.product_part_number = reading_value
                        fru_dict['product_part_number'] = reading_value
                    elif fru_dict['fru_description'] != "N/A" and sensor == "":
                        fru_obj_list.append(fru_info_obj)
                        rc = self._serverDb.add_inventory(fru_dict)
                        if rc != 0:
                            self.log(
                                self.ERROR,
                                "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" %
                                rc)
                        fru_dict = self.fru_dict_init(hostname)
                        fru_info_obj = self.fru_obj_init(hostname)
                if fru_dict['fru_description'] != "N/A":
                    fru_obj_list.append(fru_info_obj)
                    rc = self._serverDb.add_inventory(fru_dict)
                    if rc != 0:
                        self.log(
                            self.ERROR,
                            "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
                inventory_info_obj.fru_infos = fru_obj_list
            else:
                self.log(self.INFO,
                         "Could not get the FRU info for IP: %s" % ip)
                inventory_info_obj = ServerInventoryInfo()
                inventory_info_obj.name = hostname
                inventory_info_obj.fru_infos = None
            self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
        except Exception as e:
            self.log(
                self.ERROR, "Could not get the FRU info for IP " + str(ip) +
                " Error: %s" + str(e))
            inventory_info_obj = ServerInventoryInfo()
            inventory_info_obj.name = hostname
            inventory_info_obj.fru_infos = None
            self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
            raise e

    @staticmethod
    def inventory_lookup(key):
        inv_dict = {
            'hostname': 'name',
            'boardproductname': 'board_product_name',
            'boardserialnumber': 'board_serial_number',
            'boardmanufacturer': 'board_manufacturer',
            'hardwaremodel': 'hardware_model',
            'interfaces': 'interface_name',
            'physicalprocessorcount': 'physical_processor_count',
            'processorcount': 'cpu_cores_count',
            'virtual': 'virtual_machine',
            'memorytotal': 'total_memory_mb',
            'operatingsystem': 'os',
            'operatingsystemrelease': 'os_version',
            'osfamily': 'os_family',
            'kernelversion': 'kernel_version',
            'uptime_seconds': 'uptime_seconds',
            'ipaddress': 'ip_addr',
            'netmask': 'netmask',
            'macaddress': 'macaddress'
        }
        if key in inv_dict:
            return inv_dict[key]
        else:
            return None

    def get_facter_info(self, hostname, ip, sshclient):
        server_inventory_info = ServerInventoryInfo()
        # Get the total number of disks
        numdisks = self.get_field_value(sshclient, ip,
                                        'lsblk | grep disk | wc -l')
        is_ethtool = sshclient.exec_command('which ethtool')
        server_inventory_info.total_numof_disks = int(numdisks)
        # Get the other inventory information from the facter tool
        try:
            filestr = sshclient.exec_command('facter')
            fileoutput = cStringIO.StringIO(filestr)
            if fileoutput is not None:
                interface_dict = {}
                intinfo_list = []
                for line in fileoutput:
                    inventory = line.split('=>')
                    try:
                        key = inventory[0].strip()
                        if len(inventory) > 1:
                            value = inventory[1].strip()
                        if key == 'interfaces':
                            interface_list = value.split(',')
                            for name in interface_list:
                                # Skip the loopback interface
                                if name.strip() == 'lo':
                                    continue
                                intinfo = interface_info()
                                intinfo.interface_name = name
                                intinfo.speed_Mb_per_sec = 0
                                intinfo.model = "N/A"
                                if not is_ethtool:
                                    self.log(
                                        self.DEBUG,
                                        "ethtool not installed on host : %s" %
                                        ip)
                                else:
                                    eth_cmd = 'ethtool ' + name + ' | grep Speed'
                                    driver_cmd = 'ethtool -i ' + name + ' | grep driver'
                                    intinfo.speed_Mb_per_sec = self.get_field_value(
                                        sshclient, ip, eth_cmd)
                                    if bool(
                                            re.search(
                                                r'\d',
                                                str(intinfo.speed_Mb_per_sec))
                                    ):
                                        temp_var = re.findall(
                                            '\d+|\D+',
                                            str(intinfo.speed_Mb_per_sec))
                                        intinfo.speed_Mb_per_sec = int(
                                            temp_var[0].strip())
                                    else:
                                        intinfo.speed_Mb_per_sec = 0
                                    intinfo.model = self.get_field_value(
                                        sshclient, ip, driver_cmd)

                                exp = '.*_' + name + '.*$'
                                # exp = '(^ipaddress_|^macaddress_|^netmask_).*'+name+'.*$'
                                res = re.findall(exp, filestr, re.MULTILINE)
                                for items in res:
                                    actualkey = items.split('=>')
                                    namekey = actualkey[0].split('_')
                                    objkey = self.inventory_lookup(
                                        key=namekey[0].strip())
                                    value = actualkey[1].strip()
                                    if objkey:
                                        setattr(intinfo, objkey, value)
                                if not getattr(intinfo, 'macaddress'):
                                    setattr(intinfo, 'macaddress', "N/A")
                                if not getattr(intinfo, 'ip_addr'):
                                    setattr(intinfo, 'ip_addr', "N/A")
                                if not getattr(intinfo, 'netmask'):
                                    setattr(intinfo, 'netmask', "N/A")
                                intinfo_list.append(intinfo)
                        else:
                            objkey = self.inventory_lookup(key)
                            if key == 'physicalprocessorcount' or key == 'processorcount' or key == 'uptime_seconds':
                                value = int(value)
                            elif key == 'memorytotal':
                                memval = value.split()
                                value = math.trunc(float(memval[0]))
                                if memval[1].strip() == 'GB':
                                    value *= 1024
                            if objkey:
                                setattr(server_inventory_info, objkey, value)
                    except Exception as KeyError:
                        self.log(
                            self.INFO, "keyerror: %s " + str(KeyError) +
                            " for IP: %s" % ip)
                        continue
                server_inventory_info.name = str(hostname)
                server_inventory_info.interface_infos = intinfo_list
                self.call_send(
                    ServerInventoryInfoUve(data=server_inventory_info))
            else:
                self.log(self.ERROR,
                         "Could not get the Facter info for IP: %s" % ip)
        except Exception as e:
            self.log(
                self.ERROR, "Could not get the Facter info for IP " + str(ip) +
                " Error: %s" + str(e))
            raise e

    def get_field_value(self, sshclient, ip, cmd):
        try:
            filestr = sshclient.exec_command(cmd)
            if not filestr:
                return None
            if cmd == "lsblk" or cmd == "facter" or "statistics" in cmd or cmd == "vmstat" or cmd == "lspci":
                return filestr
            else:
                fileoutput = cStringIO.StringIO(filestr)
                if fileoutput is not None:
                    for line in fileoutput:
                        if "=>" in line:
                            value = line.rstrip("\n").split("=>")[1].lstrip()
                            return value
                        elif ":" not in line:
                            return line
                        value = line.rstrip("\n").split(":")[1].lstrip()
                        return value
        except Exception as e:
            self.log(
                self.ERROR, "Error in get_field_value: " + str(ip) +
                " Command = " + str(cmd) + "Error: " + str(e))
            return None

    def get_cpu_info(self, hostname, ip, sshclient):
        try:
            # Get the CPU information from server
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.cpu_info_state = cpu_info()
            check_cmd = 'cat /proc/cpuinfo'
            lscpu__cmd = 'which lscpu'
            server_inventory_info.cpu_info_state.model = "N/A"
            server_inventory_info.cpu_info_state.core_count = 0
            server_inventory_info.cpu_info_state.clock_speed_MHz = 0.0
            server_inventory_info.cpu_info_state.num_of_threads = 0
            if self.get_field_value(sshclient, ip,
                                    check_cmd) and self.get_field_value(
                                        sshclient, ip, lscpu__cmd):
                model_cmd = 'cat /proc/cpuinfo | grep "model name" | head -n 1'
                core_cmd = 'cat /proc/cpuinfo | grep "cpu cores" | head -n 1'
                clock_cmd = 'cat /proc/cpuinfo | grep "cpu MHz" | head -n 1'
                thread_cmd = 'lscpu | grep "Thread"'
                server_inventory_info.cpu_info_state.model = self.get_field_value(
                    sshclient, ip, model_cmd)
                server_inventory_info.cpu_info_state.core_count = int(
                    self.get_field_value(sshclient, ip, core_cmd))
                server_inventory_info.cpu_info_state.clock_speed_MHz = float(
                    self.get_field_value(sshclient, ip, clock_cmd))
                server_inventory_info.cpu_info_state.num_of_threads = int(
                    self.get_field_value(sshclient, ip, thread_cmd))
            else:
                self.log(self.DEBUG, "lscpu not installed on host : %s" % ip)
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(self.ERROR,
                     "Error in get_cpu_info: " + str(ip) + "Error: " + str(e))
            raise e

    def get_ethernet_info(self, hostname, ip, sshclient):
        try:
            # Get the Ethernet information from server
            is_lspci = sshclient.exec_command('which lspci')
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.eth_controller_state = ethernet_controller()
            server_inventory_info.eth_controller_state.num_of_ports = 0
            if not is_lspci:
                self.log(self.DEBUG, "lspci not installed on host : %s" % ip)
            else:
                port_cmd = 'lspci | grep Net | wc -l'
                server_inventory_info.eth_controller_state.num_of_ports = int(
                    self.get_field_value(sshclient, ip, port_cmd))
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(
                self.ERROR,
                "Error in get_ethernet_info: " + str(ip) + "Error: " + str(e))
            raise e

    def get_memory_info(self, hostname, ip, sshclient):
        try:
            server_inventory_info = ServerInventoryInfo()
            server_inventory_info.name = str(hostname)
            server_inventory_info.mem_state = memory_info()
            server_inventory_info.mem_state.mem_type = "N/A"
            server_inventory_info.mem_state.mem_speed_MHz = 0
            server_inventory_info.mem_state.num_of_dimms = 0
            server_inventory_info.mem_state.dimm_size_mb = 0
            server_inventory_info.mem_state.total_mem__mb = 0
            server_inventory_info.mem_state.swap_size_mb = 0.0
            dmi_cmd = 'which dmidecode'
            if self.get_field_value(sshclient, ip, dmi_cmd):
                type_cmd = 'dmidecode -t memory |  grep "Type:" | grep -v "Unknown" | grep -v "Error" | head -n1'
                mem_cmd = 'dmidecode -t memory | grep "Speed" | grep -v "Unknown" | head -n1'
                dimm_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | head -n1'
                num_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | wc -l'
                swap_cmd = 'facter | egrep -w swapsize_mb'
                total_mem_cmd = 'vmstat -s | grep "total memory"'
                server_inventory_info.mem_state.mem_type = self.get_field_value(
                    sshclient, ip, type_cmd)
                mem_speed = self.get_field_value(sshclient, ip, mem_cmd)
                unit = mem_speed.split(" ")[1]
                if unit == "MHz":
                    server_inventory_info.mem_state.mem_speed_MHz = int(
                        mem_speed.split(" ")[0])
                dimm_size = self.get_field_value(sshclient, ip, dimm_cmd)
                unit = dimm_size.split(" ")[1]
                if unit == "MB":
                    server_inventory_info.mem_state.dimm_size_mb = int(
                        dimm_size.split(" ")[0])
                server_inventory_info.mem_state.num_of_dimms = int(
                    self.get_field_value(sshclient, ip, num_cmd))
                swap_size = self.get_field_value(sshclient, ip, swap_cmd)
                server_inventory_info.mem_state.swap_size_mb = float(
                    swap_size.split(" ")[0])
                total_mem = self.get_field_value(sshclient, ip, total_mem_cmd)
                server_inventory_info.mem_state.total_mem_mb = int(
                    int(total_mem.lstrip().split(" ")[0]) / 1024)
            else:
                self.log(self.INFO,
                         "Couldn't get the Memory info for IP: %s" % ip)
            self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
        except Exception as e:
            self.log(
                self.ERROR,
                "Error in get_memory_info: " + str(ip) + "Error: " + str(e))
            raise e

    def add_inventory(self):
        servers = self._serverDb.get_server(None, detail=True)
        gevent.spawn(self.handle_inventory_trigger, "add", servers)

    def delete_inventory_info(self, hostname):
        inventory_info_obj = ServerInventoryInfo()
        inventory_info_obj.name = str(hostname)
        inventory_info_obj.deleted = True
        inventory_info_obj.fru_infos = None
        inventory_info_obj.interface_infos = None
        inventory_info_obj.cpu_info_state = None
        inventory_info_obj.eth_controller_state = None
        self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))

    def gevent_runner_function(self,
                               action,
                               hostname,
                               ip=None,
                               ipmi=None,
                               username=None,
                               password=None,
                               option="key"):
        sshclient = None
        if action == "add" and ip and ipmi and username and password:
            try:
                sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
                sshclient.connect(ip, hostname, option)
                self.get_facter_info(hostname, ip, sshclient)
                self.get_cpu_info(hostname, ip, sshclient)
                self.get_ethernet_info(hostname, ip, sshclient)
                self.get_memory_info(hostname, ip, sshclient)
                sshclient.close()
            except Exception as e:
                if sshclient:
                    sshclient.close()
                self.log(
                    "error", "Gevent SSH Connect Exception for server id: " +
                    str(hostname) + " Error : " + str(e))
                pass
            try:
                self.get_fru_info(hostname, ipmi, username, password)
            except Exception as e:
                self.log(
                    "error",
                    "Error in getting Inventory Info through IPMI for server id: "
                    + str(hostname) + " Error : " + str(e))
                pass
        elif action == "delete":
            self.log(self.INFO, "Deleted info of server: %s" % hostname)
            self.delete_inventory_info(hostname)

    ######## INVENTORY GET INFO SECTION ###########

    def convert_type(self, field_dict):
        data_type = field_dict["@type"]
        if "#text" in field_dict:
            if data_type == "bool":
                return json.loads(field_dict["#text"])
            elif data_type == "double":
                return float(field_dict["#text"])
            elif data_type == "u64":
                return int(field_dict["#text"])
            else:
                return str(field_dict["#text"])
        else:
            return "N/A"

    def filter_inventory_results(self, xml_dict, type_list):
        return_dict = {}
        if "all" in type_list:
            return_dict = dict(xml_dict)
        else:
            selected_fields = set(xml_dict.keys()).intersection(type_list)
            for selected_field in selected_fields:
                return_dict[selected_field] = xml_dict[selected_field]
        return return_dict

    @staticmethod
    def get_inv_conf_details(self):
        return "Configuration for Inventory set correctly."

    def get_inventory_info(self):
        list_return_dict = list()
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        server_cluster_list = list()
        server_tag_dict_list = list()
        self.log(self.DEBUG, "get_inventory_info")
        uve_name = "ServerInventoryInfo"
        try:
            entity = bottle.request
            ret_data = self._base_obj.validate_rest_api_args(
                entity, self.rev_tags_dict)
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {
                    "msg": ret_data["msg"],
                    "type_msg": ret_data["type_msg"]
                }
            if match_key == "tag":
                match_dict = self._base_obj.process_server_tags(
                    self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            if match_dict.keys():
                servers = self._serverDb.get_server(match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            #self.log(self.DEBUG, "Getting inventory info of following servers: " + str(servers))
            if len(servers) == 1:
                url = self._base_obj.get_sandesh_url(self.smgr_ip,
                                                     self.introspect_port,
                                                     uve_name,
                                                     dict(servers[0])['id'])
            else:
                url = self._base_obj.get_sandesh_url(self.smgr_ip,
                                                     self.introspect_port,
                                                     uve_name)
            headers = {'content-type': 'application/json'}
            resp = requests.get(url, timeout=300, headers=headers)
            xml_data = resp.text
            data = xmltodict.parse(str(xml_data))
            data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
            parsed_data_list = self._base_obj.parse_sandesh_xml(
                data_dict, uve_name)
            parsed_data_dict = dict()
            if parsed_data_list and servers:
                for parsed_server in parsed_data_list:
                    parsed_server = dict(parsed_server)
                    parsed_data_dict[str(
                        parsed_server["data"]["name"])] = dict(
                            parsed_server["data"])
                for server in servers:
                    server = dict(server)
                    server_hostname = str(server['id'])
                    if server_hostname in parsed_data_dict.keys():
                        return_dict = dict()
                        return_dict["name"] = str(server_hostname)
                        return_dict["cluster_id"] = server['cluster_id']
                        return_dict[str(
                            uve_name)] = self.filter_inventory_results(
                                parsed_data_dict[str(server_hostname)],
                                ret_data["type"])
                        list_return_dict.append(return_dict)
                    else:
                        self.log(self.ERROR,
                                 "Server Details missing in cache. ")
                        self.log(self.ERROR,
                                 "Server Hostname = " + str(server_hostname))
                        pass
            else:
                self.log(self.ERROR, "Server Details missing in db. ")
                return {}
        except ServerMgrException as e:
            self.log(self.ERROR,
                     "Get Inventory Info Exception: " + str(e.message))
            return json.dumps({})
        except Exception as e:
            self.log(self.ERROR,
                     "Get Inventory Info Exception: " + str(e.message))
            return json.dumps({})
        #self.log("debug", "Exited get_inventory_info " + str(datetime.now()))
        return json.dumps(list_return_dict)
        # end get_inventory_info

    def run_inventory(self):
        return_dict = dict()
        match_dict = dict()
        server_hostname_list = list()
        self.log(self.DEBUG, "run_inventory")
        try:
            entity = bottle.request
            ret_data = self._base_obj.validate_rest_api_args(
                entity, self.rev_tags_dict)
            #ret_data = self._base_obj.validate_run_inv_params(entity, self.rev_tags_dict)
            if ret_data["status"]:
                match_key = ret_data["match_key"]
                match_value = ret_data["match_value"]
            else:
                return {
                    "msg": ret_data["msg"],
                    "type_msg": ret_data["type_msg"]
                }
            if match_key == "tag":
                match_dict = self._base_obj.process_server_tags(
                    self.rev_tags_dict, match_value)
            elif match_key:
                match_dict[match_key] = match_value
            if match_dict.keys():
                servers = self._serverDb.get_server(match_dict, detail=True)
            else:
                servers = self._serverDb.get_server(detail=True)
            self.handle_inventory_trigger("add", servers)
        except ServerMgrException as e:
            self.log("error", "Run Inventory Exception: " + e.message)
            raise e
        except Exception as e:
            self.log("error", "Run Inventory Exception: " + e.message)
            raise e
        inventory_status = dict()
        inventory_status['return_message'] = "server(s) run_inventory issued"
        return inventory_status

    def handle_inventory_trigger(self, action, servers):
        server_dict = self._base_obj.create_server_dict(servers)

        gevent_threads = []
        if len(server_dict.keys()) >= 1:
            for server_id in server_dict:
                server = dict(server_dict[str(server_id)])
                if action == "add":
                    if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
                            in server and 'ipmi_password' in server:
                        thread = gevent.spawn(
                            self.gevent_runner_function, action, server['id'],
                            server['ip_address'], server['ipmi_address'],
                            server['ipmi_username'], server['ipmi_password'],
                            self.ssh_access_method)
                        gevent_threads.append(thread)
                    else:
                        self.log(
                            self.ERROR,
                            "Missing fields in server dictionary - skipping inventory addition"
                        )
                elif action == "delete":
                    if 'id' in server:
                        thread = gevent.spawn(self.gevent_runner_function,
                                              action, server['id'],
                                              self.ssh_access_method)
                        gevent_threads.append(thread)
                    else:
                        self.log(
                            self.ERROR,
                            "Missing id field in server dictionary - skipping inventory deletion"
                        )
                time.sleep(1)
        self.log(self.DEBUG, "Finished Running Inventory")