def __init__(self, val, frequency, collectors_ip=None):
''' Constructor '''
ServerMgrMonBasePlugin.__init__(self)
self.base_obj = ServerMgrMonBasePlugin()
self.val = val
self.freq = float(frequency)
self._serverDb = None
self._collectors_ip = collectors_ip
def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
''' Constructor '''
self._base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
# create logger
self._inventory_log = logging.getLogger('INVENTORY')
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
def __init__(self, val, frequency, smgr_ip=None, smgr_port=None, collectors_ip=None, introspect_port=None,
rev_tags_dict=None):
''' Constructor '''
self.base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig(self.log_file)
# create logger
self._monitoring_log = logging.getLogger('MONITORING')
self.val = val
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.freq = float(frequency)
self._collectors_ip = collectors_ip
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
''' Constructor '''
self._base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
# create logger
self._inventory_log = logging.getLogger('INVENTORY')
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
class ServerMgrInventory():
_serverDb = None
_inventory_log = None
_collectors_ip = None
_default_ipmi_username = None
_default_ipmi_password = None
_base_obj = None
DEBUG = "debug"
INFO = "info"
WARN = "warn"
ERROR = "error"
CRITICAL = "critical"
def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
''' Constructor '''
self._base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
# create logger
self._inventory_log = logging.getLogger('INVENTORY')
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
def set_serverdb(self, server_db):
self._serverDb = server_db
self._base_obj.set_serverdb(server_db=server_db)
def set_ipmi_defaults(self, ipmi_username, ipmi_password):
self._default_ipmi_username = ipmi_username
self._default_ipmi_password = ipmi_password
self._base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)
def log(self, level, msg):
frame, filename, line_number, function_name, lines, index = inspect.stack()[1]
log_dict = dict()
log_dict['log_frame'] = frame
log_dict['log_filename'] = os.path.basename(filename)
log_dict['log_line_number'] = line_number
log_dict['log_function_name'] = function_name
log_dict['log_line'] = lines
log_dict['log_index'] = index
try:
if level == self.DEBUG:
self._inventory_log.debug(msg, extra=log_dict)
elif level == self.INFO:
self._inventory_log.info(msg, extra=log_dict)
elif level == self.WARN:
self._inventory_log.warn(msg, extra=log_dict)
elif level == self.ERROR:
self._inventory_log.error(msg, extra=log_dict)
elif level == self.CRITICAL:
self._inventory_log.critical(msg, extra=log_dict)
except Exception as e:
print "Error logging msg in Inv" + e.message
# call_send function is the sending function of the sandesh object (send_inst)
def call_send(self, send_inst):
#self.log(self.INFO, "Sending UVE Info over Sandesh")
#self.log("info", "UVE Info = " + str(send_inst.data))
send_inst.send()
@staticmethod
def fru_dict_init(hostname):
fru_dict = dict()
fru_dict['id'] = hostname
fru_dict['fru_description'] = "N/A"
fru_dict['chassis_type'] = "N/A"
fru_dict['chassis_serial_number'] = "N/A"
fru_dict['board_mfg_date'] = "N/A"
fru_dict['board_manufacturer'] = "N/A"
fru_dict['board_product_name'] = "N/A"
fru_dict['board_serial_number'] = "N/A"
fru_dict['board_part_number'] = "N/A"
fru_dict['product_manfacturer'] = "N/A"
fru_dict['product_name'] = "N/A"
fru_dict['product_part_number'] = "N/A"
return fru_dict
@staticmethod
def fru_obj_init(hostname):
fru_info_obj = fru_info()
fru_info_obj.fru_description = "N/A"
fru_info_obj.chassis_type = "N/A"
fru_info_obj.chassis_serial_number = "N/A"
fru_info_obj.board_mfg_date = "N/A"
fru_info_obj.board_manufacturer = "N/A"
fru_info_obj.board_product_name = "N/A"
fru_info_obj.board_serial_number = "N/A"
fru_info_obj.board_part_number = "N/A"
fru_info_obj.product_manfacturer = "N/A"
fru_info_obj.product_name = "N/A"
fru_info_obj.product_part_number = "N/A"
return fru_info_obj
def get_fru_info(self, hostname, ip, username, password):
cmd = 'ipmitool -H %s -U %s -P %s fru' % (ip, username, password)
try:
result = self._base_obj.call_subprocess(cmd)
sensor = ""
if result:
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
fileoutput = cStringIO.StringIO(result)
fru_obj_list = list()
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
for line in fileoutput:
if ":" in line:
reading = line.split(":")
sensor = reading[0].strip()
reading_value = reading[1].strip()
if reading_value == "":
reading_value = "N/A"
else:
sensor = ""
if sensor == "FRU Device Description":
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
fru_info_obj.fru_description = reading_value
fru_dict['fru_description'] = str(hostname) + " " + reading_value
elif sensor == "Chassis Type":
fru_info_obj.chassis_type = reading_value
fru_dict['chassis_type'] = reading_value
elif sensor == "Chassis Serial":
fru_info_obj.chassis_serial_number = reading_value
fru_dict['chassis_serial_number'] = reading_value
elif sensor == "Board Mfg Date":
fru_info_obj.board_mfg_date = reading_value
fru_dict['board_mfg_date'] = reading_value
elif sensor == "Board Mfg":
fru_info_obj.board_manufacturer = reading_value
fru_dict['board_manufacturer'] = reading_value
elif sensor == "Board Product":
fru_info_obj.board_product_name = reading_value
fru_dict['board_product_name'] = reading_value
elif sensor == "Board Serial":
fru_info_obj.board_serial_number = reading_value
fru_dict['board_serial_number'] = reading_value
elif sensor == "Board Part Number":
fru_info_obj.board_part_number = reading_value
fru_dict['board_part_number'] = reading_value
elif sensor == "Product Manufacturer":
fru_info_obj.product_manfacturer = reading_value
fru_dict['product_manfacturer'] = reading_value
elif sensor == "Product Name":
fru_info_obj.product_name = reading_value
fru_dict['product_name'] = reading_value
elif sensor == "Product Part Number":
fru_info_obj.product_part_number = reading_value
fru_dict['product_part_number'] = reading_value
elif fru_dict['fru_description'] != "N/A" and sensor == "":
fru_obj_list.append(fru_info_obj)
rc = self._serverDb.add_inventory(fru_dict)
if rc != 0:
self.log(self.ERROR, "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
if fru_dict['fru_description'] != "N/A":
fru_obj_list.append(fru_info_obj)
rc = self._serverDb.add_inventory(fru_dict)
if rc != 0:
self.log(self.ERROR, "ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
inventory_info_obj.fru_infos = fru_obj_list
else:
self.log(self.INFO, "Could not get the FRU info for IP: %s" % ip)
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
inventory_info_obj.fru_infos = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
except Exception as e:
self.log(self.ERROR, "Could not get the FRU info for IP " + str(ip) + " Error: %s" + str(e))
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
inventory_info_obj.fru_infos = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
raise e
@staticmethod
def inventory_lookup(key):
inv_dict = {
'hostname': 'name',
'boardproductname': 'board_product_name',
'boardserialnumber': 'board_serial_number',
'boardmanufacturer': 'board_manufacturer',
'hardwaremodel': 'hardware_model',
'interfaces': 'interface_name', 'physicalprocessorcount': 'physical_processor_count',
'processorcount': 'cpu_cores_count',
'virtual': 'virtual_machine',
'memorytotal' : 'total_memory_mb',
'operatingsystem' : 'os','operatingsystemrelease' : 'os_version',
'osfamily':'os_family',
'kernelversion':'kernel_version',
'uptime_seconds':'uptime_seconds',
'ipaddress' :'ip_addr',
'netmask' :'netmask',
'macaddress' : 'macaddress'
}
if key in inv_dict:
return inv_dict[key]
else:
return None
def get_facter_info(self, hostname, ip, sshclient):
server_inventory_info = ServerInventoryInfo()
# Get the total number of disks
numdisks = self.get_field_value(sshclient, ip, 'lsblk | grep disk | wc -l')
is_ethtool = sshclient.exec_command('which ethtool')
server_inventory_info.total_numof_disks = int(numdisks)
# Get the other inventory information from the facter tool
try:
filestr = sshclient.exec_command('facter')
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
interface_dict = {}
intinfo_list = []
for line in fileoutput:
inventory = line.split('=>')
try:
key = inventory[0].strip()
if len(inventory) > 1:
value = inventory[1].strip()
if key == 'interfaces':
interface_list = value.split(',')
for name in interface_list:
# Skip the loopback interface
if name.strip() == 'lo':
continue
intinfo = interface_info()
intinfo.interface_name = name
intinfo.speed_Mb_per_sec = 0
intinfo.model = "N/A"
if not is_ethtool:
self.log(self.DEBUG, "ethtool not installed on host : %s" % ip)
else:
eth_cmd = 'ethtool ' + name + ' | grep Speed'
driver_cmd = 'ethtool -i ' + name + ' | grep driver'
intinfo.speed_Mb_per_sec = self.get_field_value(sshclient, ip, eth_cmd)
if bool(re.search(r'\d', str(intinfo.speed_Mb_per_sec))):
temp_var = re.findall('\d+|\D+', str(intinfo.speed_Mb_per_sec))
intinfo.speed_Mb_per_sec = int(temp_var[0].strip())
else:
intinfo.speed_Mb_per_sec = 0
intinfo.model = self.get_field_value(sshclient, ip, driver_cmd)
exp = '.*_' + name + '.*$'
# exp = '(^ipaddress_|^macaddress_|^netmask_).*'+name+'.*$'
res = re.findall(exp, filestr, re.MULTILINE)
for items in res:
actualkey = items.split('=>')
namekey = actualkey[0].split('_')
objkey = self.inventory_lookup(key=namekey[0].strip())
value = actualkey[1].strip()
if objkey:
setattr(intinfo, objkey, value)
if not getattr(intinfo, 'macaddress'):
setattr(intinfo, 'macaddress', "N/A")
if not getattr(intinfo, 'ip_addr'):
setattr(intinfo, 'ip_addr', "N/A")
if not getattr(intinfo, 'netmask'):
setattr(intinfo, 'netmask', "N/A")
intinfo_list.append(intinfo)
else:
objkey = self.inventory_lookup(key)
if key == 'physicalprocessorcount' or key == 'processorcount' or key == 'uptime_seconds':
value = int(value)
elif key == 'memorytotal':
memval = value.split()
value = math.trunc(float(memval[0]))
if memval[1].strip() == 'GB':
value *= 1024
if objkey:
setattr(server_inventory_info, objkey, value)
except Exception as KeyError:
self.log(self.INFO, "keyerror: %s " + str(KeyError) + " for IP: %s" % ip)
continue
server_inventory_info.name = str(hostname)
server_inventory_info.interface_infos = intinfo_list
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
else:
self.log(self.ERROR, "Could not get the Facter info for IP: %s" % ip)
except Exception as e:
self.log(self.ERROR, "Could not get the Facter info for IP " + str(ip) + " Error: %s" + str(e))
raise e
def get_field_value(self, sshclient, ip, cmd):
try:
filestr = sshclient.exec_command(cmd)
if not filestr:
return None
if cmd == "lsblk" or cmd == "facter" or "statistics" in cmd or cmd == "vmstat" or cmd == "lspci":
return filestr
else:
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
for line in fileoutput:
if "=>" in line:
value = line.rstrip("\n").split("=>")[1].lstrip()
return value
elif ":" not in line:
return line
value = line.rstrip("\n").split(":")[1].lstrip()
return value
except Exception as e:
self.log(self.ERROR, "Error in get_field_value: " + str(ip) + " Command = " + str(cmd) + "Error: " + str(e))
return None
def get_cpu_info(self, hostname, ip, sshclient):
try:
# Get the CPU information from server
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.cpu_info_state = cpu_info()
check_cmd = 'cat /proc/cpuinfo'
lscpu__cmd = 'which lscpu'
server_inventory_info.cpu_info_state.model = "N/A"
server_inventory_info.cpu_info_state.core_count = 0
server_inventory_info.cpu_info_state.clock_speed_MHz = 0.0
server_inventory_info.cpu_info_state.num_of_threads = 0
if self.get_field_value(sshclient, ip, check_cmd) and self.get_field_value(sshclient, ip, lscpu__cmd):
model_cmd = 'cat /proc/cpuinfo | grep "model name" | head -n 1'
core_cmd = 'cat /proc/cpuinfo | grep "cpu cores" | head -n 1'
clock_cmd = 'cat /proc/cpuinfo | grep "cpu MHz" | head -n 1'
thread_cmd = 'lscpu | grep "Thread"'
server_inventory_info.cpu_info_state.model = self.get_field_value(sshclient, ip, model_cmd)
server_inventory_info.cpu_info_state.core_count = int(self.get_field_value(sshclient, ip, core_cmd))
server_inventory_info.cpu_info_state.clock_speed_MHz = float(
self.get_field_value(sshclient, ip, clock_cmd))
server_inventory_info.cpu_info_state.num_of_threads = int(
self.get_field_value(sshclient, ip, thread_cmd))
else:
self.log(self.DEBUG, "lscpu not installed on host : %s" % ip)
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(self.ERROR, "Error in get_cpu_info: " + str(ip) + "Error: " + str(e))
raise e
def get_ethernet_info(self, hostname, ip, sshclient):
try:
# Get the Ethernet information from server
is_lspci = sshclient.exec_command('which lspci')
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.eth_controller_state = ethernet_controller()
server_inventory_info.eth_controller_state.num_of_ports = 0
if not is_lspci:
self.log(self.DEBUG, "lspci not installed on host : %s" % ip)
else:
port_cmd = 'lspci | grep Net | wc -l'
server_inventory_info.eth_controller_state.num_of_ports = int(
self.get_field_value(sshclient, ip, port_cmd))
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(self.ERROR, "Error in get_ethernet_info: " + str(ip) + "Error: " + str(e))
raise e
def get_memory_info(self, hostname, ip, sshclient):
try:
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.mem_state = memory_info()
server_inventory_info.mem_state.mem_type = "N/A"
server_inventory_info.mem_state.mem_speed_MHz = 0
server_inventory_info.mem_state.num_of_dimms = 0
server_inventory_info.mem_state.dimm_size_mb = 0
server_inventory_info.mem_state.total_mem__mb = 0
server_inventory_info.mem_state.swap_size_mb = 0.0
dmi_cmd = 'which dmidecode'
if self.get_field_value(sshclient, ip, dmi_cmd):
type_cmd = 'dmidecode -t memory | grep "Type:" | grep -v "Unknown" | grep -v "Error" | head -n1'
mem_cmd = 'dmidecode -t memory | grep "Speed" | grep -v "Unknown" | head -n1'
dimm_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | head -n1'
num_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | wc -l'
swap_cmd = 'facter | egrep -w swapsize_mb'
total_mem_cmd = 'vmstat -s | grep "total memory"'
server_inventory_info.mem_state.mem_type = self.get_field_value(sshclient, ip, type_cmd)
mem_speed = self.get_field_value(sshclient, ip, mem_cmd)
unit = mem_speed.split(" ")[1]
if unit == "MHz":
server_inventory_info.mem_state.mem_speed_MHz = int(mem_speed.split(" ")[0])
dimm_size = self.get_field_value(sshclient, ip, dimm_cmd)
unit = dimm_size.split(" ")[1]
if unit == "MB":
server_inventory_info.mem_state.dimm_size_mb = int(dimm_size.split(" ")[0])
server_inventory_info.mem_state.num_of_dimms = int(self.get_field_value(sshclient, ip, num_cmd))
swap_size = self.get_field_value(sshclient, ip, swap_cmd)
server_inventory_info.mem_state.swap_size_mb = float(swap_size.split(" ")[0])
total_mem = self.get_field_value(sshclient, ip, total_mem_cmd)
server_inventory_info.mem_state.total_mem_mb = int(int(total_mem.lstrip().split(" ")[0]) / 1024)
else:
self.log(self.INFO, "Couldn't get the Memory info for IP: %s" % ip)
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(self.ERROR, "Error in get_memory_info: " + str(ip) + "Error: " + str(e))
raise e
def add_inventory(self):
servers = self._serverDb.get_server(None, detail=True)
gevent.spawn(self.handle_inventory_trigger, "add", servers)
def delete_inventory_info(self, hostname):
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = str(hostname)
inventory_info_obj.deleted = True
inventory_info_obj.fru_infos = None
inventory_info_obj.interface_infos = None
inventory_info_obj.cpu_info_state = None
inventory_info_obj.eth_controller_state = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
def gevent_runner_function(self, action, hostname, ip=None, ipmi=None, username=None, password=None, option="key"):
sshclient = None
if action == "add" and ip and ipmi and username and password:
try:
sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
sshclient.connect(ip, hostname, option)
self.get_facter_info(hostname, ip, sshclient)
self.get_cpu_info(hostname, ip, sshclient)
self.get_ethernet_info(hostname, ip, sshclient)
self.get_memory_info(hostname, ip, sshclient)
sshclient.close()
except Exception as e:
if sshclient:
sshclient.close()
self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e))
pass
try:
self.get_fru_info(hostname, ipmi, username, password)
except Exception as e:
self.log("error", "Error in getting Inventory Info through IPMI for server id: " + str(hostname) + " Error : " + str(e))
pass
elif action == "delete":
self.log(self.INFO, "Deleted info of server: %s" % hostname)
self.delete_inventory_info(hostname)
######## INVENTORY GET INFO SECTION ###########
def convert_type(self, field_dict):
data_type = field_dict["@type"]
if "#text" in field_dict:
if data_type == "bool":
return json.loads(field_dict["#text"])
elif data_type == "double":
return float(field_dict["#text"])
elif data_type == "u64":
return int(field_dict["#text"])
else:
return str(field_dict["#text"])
else:
return "N/A"
def filter_inventory_results(self, xml_dict, type_list):
return_dict = {}
if "all" in type_list:
return_dict = dict(xml_dict)
else:
selected_fields = set(xml_dict.keys()).intersection(type_list)
for selected_field in selected_fields:
return_dict[selected_field] = xml_dict[selected_field]
return return_dict
@staticmethod
def get_inv_conf_details(self):
return "Configuration for Inventory set correctly."
def get_inventory_info(self):
list_return_dict = list()
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
server_cluster_list = list()
server_tag_dict_list = list()
self.log(self.DEBUG, "get_inventory_info")
uve_name = "ServerInventoryInfo"
try:
entity = bottle.request
ret_data = self._base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
if match_key == "tag":
match_dict = self._base_obj.process_server_tags(self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
if match_dict.keys():
servers = self._serverDb.get_server(
match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
#self.log(self.DEBUG, "Getting inventory info of following servers: " + str(servers))
if len(servers) == 1:
url = self._base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name,
dict(servers[0])['id'])
else:
url = self._base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name)
headers = {'content-type': 'application/json'}
resp = requests.get(url, timeout=300, headers=headers)
xml_data = resp.text
data = xmltodict.parse(str(xml_data))
data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
parsed_data_list = self._base_obj.parse_sandesh_xml(data_dict, uve_name)
parsed_data_dict = dict()
if parsed_data_list and servers:
for parsed_server in parsed_data_list:
parsed_server = dict(parsed_server)
parsed_data_dict[str(parsed_server["data"]["name"])] = dict(parsed_server["data"])
for server in servers:
server = dict(server)
server_hostname = str(server['id'])
if server_hostname in parsed_data_dict.keys():
return_dict = dict()
return_dict["name"] = str(server_hostname)
return_dict["cluster_id"] = server['cluster_id']
return_dict[str(uve_name)] = self.filter_inventory_results(
parsed_data_dict[str(server_hostname)],
ret_data["type"])
list_return_dict.append(return_dict)
else:
self.log(self.ERROR, "Server Details missing in cache. ")
self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
pass
else:
self.log(self.ERROR, "Server Details missing in db. ")
return {}
except ServerMgrException as e:
self.log(self.ERROR, "Get Inventory Info Exception: " + str(e.message))
return json.dumps({})
except Exception as e:
self.log(self.ERROR, "Get Inventory Info Exception: " + str(e.message))
return json.dumps({})
#self.log("debug", "Exited get_inventory_info " + str(datetime.now()))
return json.dumps(list_return_dict)
# end get_inventory_info
def run_inventory(self):
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
self.log(self.DEBUG, "run_inventory")
try:
entity = bottle.request
ret_data = self._base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
#ret_data = self._base_obj.validate_run_inv_params(entity, self.rev_tags_dict)
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
if match_key == "tag":
match_dict = self._base_obj.process_server_tags(self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
if match_dict.keys():
servers = self._serverDb.get_server(
match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
self.handle_inventory_trigger("add", servers)
except ServerMgrException as e:
self.log("error", "Run Inventory Exception: " + e.message)
raise e
except Exception as e:
self.log("error", "Run Inventory Exception: " + e.message)
raise e
inventory_status = dict()
inventory_status['return_message'] = "server(s) run_inventory issued"
return inventory_status
def handle_inventory_trigger(self, action, servers):
server_dict = self._base_obj.create_server_dict(servers)
gevent_threads = []
if len(server_dict.keys()) >= 1:
for server_id in server_dict:
server = dict(server_dict[str(server_id)])
if action == "add":
if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
in server and 'ipmi_password' in server:
thread = gevent.spawn(self.gevent_runner_function, action, server['id'], server['ip_address'],
server['ipmi_address'], server['ipmi_username'], server['ipmi_password'],
self.ssh_access_method)
gevent_threads.append(thread)
else:
self.log(self.ERROR, "Missing fields in server dictionary - skipping inventory addition")
elif action == "delete":
if 'id' in server:
thread = gevent.spawn(self.gevent_runner_function, action, server['id'], self.ssh_access_method)
gevent_threads.append(thread)
else:
self.log(self.ERROR, "Missing id field in server dictionary - skipping inventory deletion")
time.sleep(1)
self.log(self.DEBUG, "Finished Running Inventory")
class ServerMgrIPMIMonitoring():
host_disklist = dict()
host_nw_info_list = dict()
_default_ipmi_username = None
_default_ipmi_password = None
DEBUG = "debug"
INFO = "info"
WARN = "warn"
ERROR = "error"
CRITICAL = "critical"
_serverDb = None
sleep_period = 10
ssh_access_method = ""
log_file = '/opt/contrail/server_manager/logger.conf'
def __init__(self, val, frequency, smgr_ip=None, smgr_port=None, collectors_ip=None, introspect_port=None,
rev_tags_dict=None):
''' Constructor '''
self.base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig(self.log_file)
# create logger
self._monitoring_log = logging.getLogger('MONITORING')
self.val = val
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.freq = float(frequency)
self._collectors_ip = collectors_ip
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
def log(self, level, msg):
frame, filename, line_number, function_name, lines, index = inspect.stack()[1]
log_dict = dict()
log_dict['log_frame'] = frame
log_dict['log_filename'] = os.path.basename(filename)
log_dict['log_line_number'] = line_number
log_dict['log_function_name'] = function_name
log_dict['log_line'] = lines
log_dict['log_index'] = index
try:
if level == self.DEBUG:
self._monitoring_log.debug(msg, extra=log_dict)
elif level == self.INFO:
self._monitoring_log.info(msg, extra=log_dict)
elif level == self.WARN:
self._monitoring_log.warn(msg, extra=log_dict)
elif level == self.ERROR:
self._monitoring_log.error(msg, extra=log_dict)
elif level == self.CRITICAL:
self._monitoring_log.critical(msg, extra=log_dict)
except Exception as e:
print "Error logging msg in Mon" + e.message
def set_serverdb(self, server_db):
self._serverDb = server_db
self.base_obj.set_serverdb(server_db=server_db)
def set_ipmi_defaults(self, ipmi_username, ipmi_password):
self._default_ipmi_username = ipmi_username
self._default_ipmi_password = ipmi_password
self.base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)
# call_send function is the sending function of the sandesh object (send_inst)
def call_send(self, send_inst):
#self.log("info", "Sending UVE Info over Sandesh")
send_inst.send()
#This function returns the boolean value needed in the Sandesh Payload
def str2bool(self,value):
return value.lower() in ("true")
# send_ipmi_stats function packages and sends the IPMI info gathered from server polling
# to the analytics node
def send_ipmi_stats(self, ip, ipmi_data, hostname, data_type):
base_send_type = base_mon_uve.UVE_NONE
if data_type == "ipmi_data":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.sensor_stats = []
for ipmidata in ipmi_data:
sm_ipmi_info.sensor_stats.append(ipmidata)
elif data_type == "ipmi_chassis_data":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.chassis_state = ipmi_data
elif data_type == "disk_list":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.disk_usage_stats = []
for data in ipmi_data:
sm_ipmi_info.disk_usage_stats.append(data)
elif data_type == "disk_list_tot":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.disk_usage_totals = []
for data in ipmi_data:
sm_ipmi_info.disk_usage_totals.append(data)
elif data_type == "resource_info_stats":
base_send_type = base_mon_uve.UVE_SUMMARY
sm_ipmi_info = ServerMonitoringSummary()
sm_ipmi_info.resource_info_stats = ipmi_data
elif data_type == "intinfo_list_tot":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.network_info_totals = []
for data in ipmi_data:
sm_ipmi_info.network_info_totals.append(data)
elif data_type == "intinfo_list":
base_send_type = base_mon_uve.UVE_SUMMARY
sm_ipmi_info = ServerMonitoringSummary()
sm_ipmi_info.network_info_stats = []
for data in ipmi_data:
sm_ipmi_info.network_info_stats.append(data)
elif data_type == "file_system_view_list":
base_send_type = base_mon_uve.UVE_INFO
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.file_system_view_stats = []
for data in ipmi_data:
sm_ipmi_info.file_system_view_stats.append(data)
# self.log("info", "Sending Monitoring UVE Info for: " + str(data_type))
# self.log("info", "UVE Interface Info = " + str(sm_ipmi_info))
# assign hostname
sm_ipmi_info.name = str(hostname)
sm_ipmi_info.deleted = False
# Send info based on base type
if base_send_type == base_mon_uve.UVE_INFO:
ipmi_stats_trace = ServerMonitoringInfoUve(data=sm_ipmi_info)
elif base_send_type == base_mon_uve.UVE_SUMMARY:
ipmi_stats_trace = ServerMonitoringSummaryUve(data=sm_ipmi_info)
elif base_send_type == base_mon_uve.UVE_NONE:
self.log("error", "Error base mon uve type for " + str(data_type))
# self.log("info", "UVE Info Sent= " + str(sm_ipmi_info))
self.call_send(ipmi_stats_trace)
# Packages and sends a REST API call to the ServerManager node
def send_run_inventory_request(self, ip, port, payload):
try:
url = "http://%s:%s/run_inventory" % (ip, port)
args_str = ''
match_key, match_value = payload.popitem()
if match_key and match_value:
args_str += urllib.quote_plus(str(match_key)) + "=" \
+ urllib.quote_plus(str(match_value))
if args_str != '':
url += "?" + args_str
headers = {'content-type': 'application/json'}
resp = requests.post(url, headers=headers, timeout=300)
return resp.text
except Exception as e:
self.log("error", "Error running inventory on " + str(payload) + " : " + str(e))
return None
def return_collector_ip(self):
return self._collectors_ip
def isfloat(self, num):
try:
float(num)
except ValueError:
return False
return True
def fetch_and_process_monitoring(self, hostname, ipmi, ip, username, password, supported_sensors):
try:
ipmi_data = []
cmd = 'ipmitool -H %s -U %s -P %s sdr list all' % (ipmi, username, password)
result = self.base_obj.call_subprocess(cmd)
if result is not None and "|" in result:
fileoutput = cStringIO.StringIO(result)
for line in fileoutput:
reading = line.split("|")
sensor = reading[0].strip()
reading_value = reading[1].strip()
status = reading[2].strip()
for i in supported_sensors:
if re.search(i, sensor, re.IGNORECASE) is not None:
sensor_type = 'unknown'
if 'FAN' in sensor or 'fan' in sensor or 'Fan' in sensor:
sensor_type = 'fan'
elif 'PWR' in sensor or 'Power' in sensor:
sensor_type = 'power'
elif 'Temp' in sensor or 'TEMP' in sensor or 'temp' in sensor:
sensor_type = 'temperature'
value = reading_value.split()
ipmidata = IpmiSensor()
ipmidata.sensor = sensor
ipmidata.status = status
if status == "ns":
pass
elif status == "ok" and self.isfloat(value[0].strip()):
ipmidata.reading = long(value[0].strip())
ipmidata.unit = value[len(value) - 1].strip()
ipmidata.sensor_type = sensor_type
ipmi_data.append(ipmidata)
self.send_ipmi_stats(ip, ipmi_data, hostname, "ipmi_data")
return True
else:
self.log("error", "Error getting Sesnsor info for: " + str(hostname) +
" Data returned was: " + str(result))
return False
except Exception as e:
self.log("error", "Error getting Sesnsor info for: " + str(hostname) + " Error is: " + str(e))
raise
def fetch_and_process_chassis(self, hostname, ipmi, ip, username, password):
ipmi_chassis_data = IpmiChassis_status_info()
cmd = 'ipmitool -H %s -U %s -P %s chassis status' % (ipmi, username, password)
try:
result = self.base_obj.call_subprocess(cmd)
if result is not None:
fileoutput = cStringIO.StringIO(result)
ipmichassisdata = IpmiChassis_status_info()
for line in fileoutput:
if len(line.split(":")) >= 2:
reading = line.split(":")
chassis_key = reading[0].strip()
chassis_value = reading[1].strip()
if chassis_key == "System Power":
ipmichassisdata.system_power = chassis_value
elif chassis_key == "Power Overload" and chassis_value:
ipmichassisdata.power_overload = self.str2bool(chassis_value)
elif chassis_key == "Power Interlock" and chassis_value:
ipmichassisdata.power_interlock = chassis_value
elif chassis_key == "Main Power Fault" and chassis_value:
ipmichassisdata.main_power_fault = self.str2bool(chassis_value)
elif chassis_key == "Power Control Fault" and chassis_value:
ipmichassisdata.power_control_fault = self.str2bool(chassis_value)
elif chassis_key == "Power Restore Policy" and chassis_value:
ipmichassisdata.power_restore_policy = chassis_value
elif chassis_key == "Last Power Event" and chassis_value:
ipmichassisdata.last_power_event = chassis_value
elif chassis_key == "Chassis Intrusion" and chassis_value:
ipmichassisdata.chassis_intrusion = chassis_value
elif chassis_key == "Front-Panel Lockout" and chassis_value:
ipmichassisdata.front_panel_lockout = chassis_value
elif chassis_key == "Drive Fault" and chassis_value:
ipmichassisdata.drive_fault = self.str2bool(chassis_value)
elif chassis_value:
ipmichassisdata.cooling_fan_fault = self.str2bool(chassis_value)
else:
pass
ipmi_chassis_data = ipmichassisdata
self.send_ipmi_stats(ip, ipmi_chassis_data, hostname, "ipmi_chassis_data")
except Exception as e:
self.log("error", "Error getting chassis data for " + str(hostname) + " : " + str(e.message))
raise
def fetch_and_process_disk_info(self, hostname, ip, sshclient):
disk_list = []
disk_list_tot = []
cmd = 'iostat -m'
is_sysstat = sshclient.exec_command('which iostat')
if not is_sysstat:
#self.log("info", "sysstat package not installed on " + str(ip))
disk_data = Disk()
disk_data_tot = Disk_totals()
disk_data.disk_name = "N/A"
disk_data.read_bytes = int(0)
disk_data.write_bytes = int(0)
disk_data_tot.total_read_bytes = int(0)
disk_data_tot.total_write_bytes = int(0)
disk_list.append(disk_data)
disk_list_tot.append(disk_data_tot)
self.send_ipmi_stats(ip, disk_list, hostname, "disk_list")
self.send_ipmi_stats(ip, disk_list_tot, hostname, "disk_list_tot")
else:
try:
filestr = sshclient.exec_command(cmd=cmd)
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
#lookup for host dictionary
if hostname not in self.host_disklist:
#if empty insert the disklist dictionary
dict_disk = dict()
self.host_disklist[hostname] = dict_disk
else:
# disklist entry found
dict_disk = self.host_disklist[hostname]
for line in fileoutput:
if line is not None:
if (line.find('sd') != -1 or line.find('dm') != -1) and line.find('Linux')!=0:
disk_data = Disk()
disk_data_tot = Disk_totals()
prev_disk_info = Disk_totals()
res = re.sub('\s+', ' ', line).strip()
arr = res.split()
disk_data.disk_name = arr[0]
disk_data_tot.disk_name = arr[0]
disk_data_tot.total_read_bytes = int(arr[4])
disk_data_tot.total_write_bytes = int(arr[5])
if len(dict_disk) != 0:
if disk_data_tot.disk_name in dict_disk:
disk_data.read_bytes = disk_data_tot.total_read_bytes - dict_disk[
disk_data_tot.disk_name].total_read_bytes
disk_data.write_bytes = disk_data_tot.total_write_bytes - dict_disk[
disk_data_tot.disk_name].total_write_bytes
disk_list.append(disk_data)
prev_disk_info.total_read_bytes = disk_data_tot.total_read_bytes
prev_disk_info.total_write_bytes = disk_data_tot.total_write_bytes
dict_disk[disk_data_tot.disk_name] = prev_disk_info
disk_list_tot.append(disk_data_tot)
if disk_list:
self.send_ipmi_stats(ip, disk_list, hostname, "disk_list")
#return True no return
if disk_list_tot:
self.send_ipmi_stats(ip, disk_list_tot, hostname, "disk_list_tot")
return True
else:
return False
except Exception as e:
self.log("error", "Error getting disk info for " + str(hostname) + " : " + str(e))
raise
#This function gets the mounted file system view. This will be the output of the df command
def fetch_and_process_file_system_view(self, hostname, ip, sshclient):
try:
fs_view = file_system_disk_view()
file_system_view_list = fs_view.get_file_system_view(sshclient)
self.send_ipmi_stats(ip, file_system_view_list, hostname, "file_system_view_list")
except Exception as e:
self.log("error", "Error getting file system view info for " + str(hostname) + " : " + str(e))
raise e
def fetch_and_process_resource_info(self, hostname, ip, sshclient):
try:
resource_info1 = resource_info()
is_mpstat = sshclient.exec_command('which mpstat')
if not is_mpstat:
resource_info1.cpu_usage_percentage = 0.0
else:
cmd = 'mpstat'
filestr = sshclient.exec_command(cmd=cmd)
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
idle_index = -1
for line in fileoutput:
res = re.sub('\s+', ' ', line).strip()
arr = res.split()
if idle_index != -1:
resource_info1.cpu_usage_percentage = (100.0 - float(arr[idle_index]))
break
if "%idle" not in arr:
continue
else:
idle_index = arr.index('%idle')
is_vmstat = sshclient.exec_command('which vmstat')
if not is_vmstat:
resource_info1.mem_usage_mb = 0
resource_info1.mem_usage_percent = 0
else:
cmd = 'vmstat -s | grep "used memory"'
filestr = sshclient.exec_command(cmd=cmd)
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
for line in fileoutput:
arr = line.split()
resource_info1.mem_usage_mb = int(int(arr[0])/1024)
cmd = 'vmstat -s | grep "total memory"'
filestr = sshclient.exec_command(cmd=cmd)
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
for line in fileoutput:
arr = line.split()
resource_info1.mem_usage_percent = float("{0:.2f}".format(
(resource_info1.mem_usage_mb/float(int(arr[0])/1024))*100))
self.send_ipmi_stats(ip, resource_info1, hostname, "resource_info_stats")
except Exception as e:
self.log("error", "Error in getting resource info for " + str(hostname) + str(e))
raise e
def fetch_and_process_network_info(self, hostname, ip, sshclient):
try:
intinfo_list = []
intinfo_list_tot = []
phys_intf_checker = sshclient.exec_command("ls -l /sys/class/net/")
phys_intf_list = list()
if phys_intf_checker:
checker_output = cStringIO.StringIO(phys_intf_checker)
for line in checker_output:
line = str(line)
if "virtual" not in line and "total" not in line and len(line.split(' ')) > 8:
line = line.split('/')
phys_intf_list.append(line[len(line) - 1].rstrip('\n'))
result = sshclient.exec_command("ls /sys/class/net/")
if result:
output = cStringIO.StringIO(result)
net_dictinfo = None
#lookup for host dictionary
if hostname not in self.host_nw_info_list:
#if empty insert the net dict info
net_dictinfo = dict()
self.host_nw_info_list[hostname] = net_dictinfo
else:
# disklist entry found
net_dictinfo = self.host_nw_info_list[hostname]
for line in output:
intinfo = network_info()
intinfo_tot = network_info_totals()
prev_nw = network_info_totals()
if line.rstrip() in phys_intf_list:
cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/tx_bytes"
tx_bytes = sshclient.exec_command(cmd=cmd)
cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/tx_packets"
tx_packets = sshclient.exec_command(cmd=cmd)
cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/rx_bytes"
rx_bytes = sshclient.exec_command(cmd=cmd)
cmd = "cat /sys/class/net/" + line.rstrip() + "/statistics/rx_packets"
rx_packets = sshclient.exec_command(cmd=cmd)
intinfo_tot.interface_name = line.rstrip()
intinfo.interface_name = line.rstrip()
intinfo_tot.total_tx_bytes = int(tx_bytes.rstrip())
intinfo_tot.total_tx_packets = int(tx_packets.rstrip())
intinfo_tot.total_rx_bytes = int(rx_bytes.rstrip())
intinfo_tot.total_rx_packets = int(rx_packets.rstrip())
if len(net_dictinfo) != 0:
if intinfo_tot.interface_name in net_dictinfo:
intinfo.tx_bytes = intinfo_tot.total_tx_bytes - net_dictinfo[
intinfo_tot.interface_name].total_tx_bytes
intinfo.rx_bytes = intinfo_tot.total_rx_bytes - net_dictinfo[
intinfo_tot.interface_name].total_rx_bytes
intinfo.tx_packets = intinfo_tot.total_tx_packets - net_dictinfo[
intinfo_tot.interface_name].total_tx_packets
intinfo.rx_packets = intinfo_tot.total_rx_packets - net_dictinfo[
intinfo_tot.interface_name].total_rx_packets
intinfo_list.append(intinfo)
prev_nw.total_tx_bytes = intinfo_tot.total_tx_bytes
prev_nw.total_rx_bytes = intinfo_tot.total_rx_bytes
prev_nw.total_tx_packets = intinfo_tot.total_tx_packets
prev_nw.total_rx_packets = intinfo_tot.total_rx_packets
net_dictinfo[intinfo_tot.interface_name] = prev_nw
intinfo_list_tot.append(intinfo_tot)
if intinfo_list:
self.send_ipmi_stats(ip, intinfo_list, hostname, "intinfo_list")
if intinfo_list_tot:
self.send_ipmi_stats(ip, intinfo_list_tot, hostname, "intinfo_list_tot")
return True
else:
return False
except Exception as e:
self.log("error", "Error in getting network info for " + str(hostname) + " Error: " + str(e))
raise e
def fetch_and_process_sel_logs(self, hostname, ip, username, password, sel_event_log_list):
sel_cmd = 'ipmitool -H %s -U %s -P %s sel elist' % (ip, username, password)
sel_result = self.base_obj.call_subprocess(sel_cmd)
try:
if sel_result is not None:
fileoutput = cStringIO.StringIO(sel_result)
for line in fileoutput:
sellog = IpmiSystemEventLog()
sellog.name = str(hostname)
col = line.split("|")
hex_event_id = col[0].strip()
if hex_event_id.isdigit():
event_id = int(hex_event_id, 16)
else:
event_id = None
if event_id and event_id not in sel_event_log_list:
sel_event_log_list.append(event_id)
sellog.event_id = event_id
sellog.ipmi_timestamp = str(col[1]) + " " + str(col[2])
sensor_data = str(col[3])
sensor_data = sensor_data.split(" ")
sellog.sensor_type = str(sensor_data[1]) + " " + str(sensor_data[2])
sellog.sensor_name = str(sensor_data[3])
if len(sensor_data) >= 5:
sellog.sensor_name += " " + str(sensor_data[4])
sellog.ipmi_message = str(col[4])
if len(col) >= 6:
sellog.ipmi_message += " " + str(col[5])
sellog.send()
else:
pass
return sel_event_log_list
except Exception as e:
self.log("error", "Error getting SEL Logs for " + str(hostname) + " : " + str(e.message))
raise e
def delete_monitoring_info(self, hostname_list):
for hostname in hostname_list:
sm_ipmi_info = ServerMonitoringInfo()
sm_ipmi_info.name = str(hostname)
sm_ipmi_info.deleted = True
sm_ipmi_info.chassis_state = None
ipmi_stats_trace = ServerMonitoringInfoUve(data=sm_ipmi_info)
self.call_send(ipmi_stats_trace)
def gevent_runner_func(self, hostname, ipmi, ip, username, password, supported_sensors, ipmi_state,
sel_event_log_list, option="key"):
return_dict = dict()
#self.log("info", "Gevent Thread created for %s" % hostname)
try:
sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
sshclient.connect(ip, hostname, option)
self.fetch_and_process_resource_info(hostname, ip, sshclient)
self.fetch_and_process_network_info(hostname, ip, sshclient)
self.fetch_and_process_disk_info(hostname, ip, sshclient)
self.fetch_and_process_file_system_view(hostname, ip, sshclient)
sshclient.close()
except Exception as e:
self.log("error", "Gevent SSH Connect Exception for server id: " + str(hostname) + " Error : " + str(e))
sshclient.close()
pass
try:
return_dict["ipmi_status"] = \
self.fetch_and_process_monitoring(hostname, ipmi, ip, username, password, supported_sensors)
self.fetch_and_process_chassis(hostname, ipmi, ip, username, password)
if sel_event_log_list:
return_dict["sel_log"] = \
self.fetch_and_process_sel_logs(hostname, ipmi, username, password, sel_event_log_list)
else:
return_dict["sel_log"] = self.fetch_and_process_sel_logs(hostname, ipmi, username, password, [])
if not ipmi_state and return_dict["ipmi_status"]:
# Trigger REST API CALL to inventory for Server Hostname
payload = dict()
payload["id"] = str(hostname)
self.send_run_inventory_request(self.smgr_ip, self.smgr_port, payload=payload)
# self.log("info", "Payload return dict:\n" + str(return_dict))
return return_dict
except Exception as e:
self.log("error", "Error in getting monitoring info through IPMI for server id: " + str(hostname) + " Error : " + str(e))
pass
# ####### MONITORING GET INFO SECTION ###########
def convert_type(self, field_dict):
data_type = field_dict["@type"]
if "#text" in field_dict:
if data_type == "bool":
return json.loads(field_dict["#text"])
elif data_type == "double":
return float(field_dict["#text"])
elif data_type == "u64":
return int(field_dict["#text"])
else:
return str(field_dict["#text"])
else:
return "N/A"
# Filters the data returned from REST API call for requested information
def filter_monitoring_results(self, xml_dict, type_list):
return_dict = {}
if "all" in type_list:
return_dict = dict(xml_dict)
else:
selected_fields = set(xml_dict.keys()).intersection(type_list)
for selected_field in selected_fields:
return_dict[selected_field] = xml_dict[selected_field]
return return_dict
def get_mon_conf_details(self):
list_return_dict = list()
main_return_dict = dict()
return_dict = dict()
return_dict["monitoring_frequency"] = self.freq
return_dict["http_introspect_port"] = self.introspect_port
return_dict["analytics_node_ips"] = list(self._collectors_ip)
main_return_dict["config"] = dict(return_dict)
list_return_dict.append(main_return_dict)
return json.dumps(list_return_dict)
def get_monitoring_info(self):
list_return_dict = list()
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
server_cluster_list = list()
server_tag_dict_list = list()
self.log("debug", "get_monitoring_info")
#self.log("debug", "Entered get_monitoring_info " + str(datetime.now()))
uve_name = "ServerMonitoringInfo"
summary_uve_name = "ServerMonitoringSummary"
try:
entity = bottle.request
ret_data = self.base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
#self.log("debug", "Validated rest api params " + str(datetime.now()))
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
if match_key == "tag":
match_dict = self.base_obj.process_server_tags(self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
#self.log("debug", "Before server db read " + str(datetime.now()))
if match_dict.keys():
servers = self._serverDb.get_server(
match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
#self.log("debug", "After server read " + str(datetime.now()))
#self.log("debug", "Getting monitoring info of following servers: " + str(server_hostname_list))
if len(servers) == 1:
url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name,
dict(servers[0])['id'])
summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name,
dict(servers[0])['id'])
else:
url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, uve_name)
summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name)
headers = {'content-type': 'application/json'}
#self.log("debug", "After get_sandesh_url, before REST API call " + str(datetime.now()))
#time_before = time.time()
resp = requests.get(url, timeout=300, headers=headers)
sum_resp = requests.get(summary_url, timeout=300, headers=headers)
xml_data = resp.text
sum_xml_data = sum_resp.text
#time_after = time.time()
#time_sec = time_after - time_before
#self.log("debug", "Sandesh REST API Call : Time taken = " + str(time_sec)
# + " Resp length = " + str(len(xml_data)))
#self.log("debug", "After REST API call " + str(datetime.now()))
time_before = time.time()
data = xmltodict.parse(str(xml_data))
sum_data = xmltodict.parse(str(sum_xml_data))
#self.log("debug", "After XMLtoDict" + str(datetime.now()))
time_after = time.time()
time_sec = time_after - time_before
#self.log("debug", "XMLtoDict Call : Time taken = " + str(time_sec))
data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
sum_data_dict = dict(sum_data["__" + str(summary_uve_name) + "Uve_list"])
#self.log("debug", "Before processing " + str(datetime.now()))
parsed_data_list = self.base_obj.parse_sandesh_xml(data_dict, uve_name)
sum_parsed_data_list = self.base_obj.parse_sandesh_xml(sum_data_dict, summary_uve_name)
parsed_data_dict = dict()
sum_parsed_data_dict = dict()
if parsed_data_list and sum_parsed_data_list and servers:
for parsed_server in parsed_data_list:
parsed_server = dict(parsed_server)
parsed_data_dict[str(parsed_server["data"]["name"])] = dict(parsed_server["data"])
for sum_parsed_server in sum_parsed_data_list:
sum_parsed_server = dict(sum_parsed_server)
sum_parsed_data_dict[str(sum_parsed_server["data"]["name"])] = dict(sum_parsed_server["data"])
for server in servers:
server = dict(server)
server_hostname = str(server['id'])
if server_hostname in parsed_data_dict.keys():
return_dict = dict()
return_dict["name"] = str(server['id'])
return_dict["cluster_id"] = server['cluster_id']
if str(server['id']) in parsed_data_dict.keys():
main_dict = self.filter_monitoring_results(
parsed_data_dict[str(server['id'])],
ret_data["type"])
else:
main_dict = {}
if str(server['id']) in sum_parsed_data_dict.keys():
summary_dict = self.filter_monitoring_results(
sum_parsed_data_dict[str(server['id'])],
ret_data["type"])
else:
summary_dict = {}
for summary_key in summary_dict:
main_dict[str(summary_key)] = summary_dict[summary_key]
return_dict[str(uve_name)] = main_dict
#self.log("info", "All Keys:" + str(return_dict.keys()))
#self.log("info", "All Smgr Keys:" + str(return_dict[str(uve_name)].keys()))
list_return_dict.append(return_dict)
else:
self.log(self.ERROR, "Server Details missing in cache. ")
self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
pass
else:
self.log(self.ERROR, "Server Details missing in db. ")
pass
except ServerMgrException as e:
self.log("error", "Get Monitoring Info Exception: " + str(e.message))
return_dict = {}
list_return_dict = list()
list_return_dict.append(return_dict)
return json.dumps(list_return_dict)
except Exception as e:
self.log("error", "Get Monitoring Info Exception: " + str(e.message))
return_dict = {}
list_return_dict = list()
list_return_dict.append(return_dict)
return json.dumps(list_return_dict)
#self.log("debug", "Exited get_monitoring_info " + str(datetime.now()))
return json.dumps(list_return_dict)
def get_monitoring_info_summary(self):
list_return_dict = list()
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
server_cluster_list = list()
server_tag_dict_list = list()
self.log("debug", "get_monitoring_info_summary")
#self.log("debug", "Entered get_monitoring_info " + str(datetime.now()))
uve_name = "ServerMonitoringInfo"
summary_uve_name = "ServerMonitoringSummary"
try:
entity = bottle.request
ret_data = self.base_obj.validate_rest_api_args(entity, self.rev_tags_dict)
#self.log("debug", "Validated rest api params " + str(datetime.now()))
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {"msg": ret_data["msg"], "type_msg": ret_data["type_msg"]}
if match_key == "tag":
match_dict = self.base_obj.process_server_tags(self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
#self.log("debug", "Before server db read " + str(datetime.now()))
if match_dict.keys():
servers = self._serverDb.get_server(
match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
#self.log("debug", "After server read " + str(datetime.now()))
#self.log("debug", "Getting monitoring info of following servers: " + str(server_hostname_list))
if len(servers) == 1:
summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name,
dict(servers[0])['id'])
else:
summary_url = self.base_obj.get_sandesh_url(self.smgr_ip, self.introspect_port, summary_uve_name)
headers = {'content-type': 'application/json'}
#self.log("debug", "After get_sandesh_url, before REST API call " + str(datetime.now()))
#time_before = time.time()
sum_resp = requests.get(summary_url, timeout=300, headers=headers)
sum_xml_data = sum_resp.text
#time_after = time.time()
#time_sec = time_after - time_before
#self.log("debug", "Sandesh REST API Call : Time taken = " + str(time_sec)
# + " Resp length = " + str(len(xml_data)))
#self.log("debug", "After REST API call " + str(datetime.now()))
time_before = time.time()
sum_data = xmltodict.parse(str(sum_xml_data))
#self.log("debug", "After XMLtoDict" + str(datetime.now()))
time_after = time.time()
time_sec = time_after - time_before
#self.log("debug", "XMLtoDict Call : Time taken = " + str(time_sec))
sum_data_dict = dict(sum_data["__" + str(summary_uve_name) + "Uve_list"])
#self.log("debug", "Before processing " + str(datetime.now()))
sum_parsed_data_list = self.base_obj.parse_sandesh_xml(sum_data_dict, summary_uve_name)
sum_parsed_data_dict = dict()
if sum_parsed_data_list and servers:
for sum_parsed_server in sum_parsed_data_list:
sum_parsed_server = dict(sum_parsed_server)
sum_parsed_data_dict[str(sum_parsed_server["data"]["name"])] = dict(sum_parsed_server["data"])
for server in servers:
server = dict(server)
server_hostname = str(server['id'])
if server_hostname in sum_parsed_data_dict.keys():
return_dict = dict()
return_dict["name"] = str(server['id'])
return_dict["cluster_id"] = server['cluster_id']
return_dict[str(summary_uve_name)] = self.filter_monitoring_results(
sum_parsed_data_dict[str(server['id'])],
ret_data["type"]
)
return_dict[str(uve_name)] = return_dict[str(summary_uve_name)]
return_dict.pop(str(summary_uve_name))
list_return_dict.append(return_dict)
else:
self.log(self.ERROR, "Server Details missing in cache. ")
self.log(self.ERROR, "Server Hostname = " + str(server_hostname))
pass
else:
self.log(self.ERROR, "Server Details missing in db. ")
pass
except ServerMgrException as e:
self.log("error", "Get Monitoring Info Summary Exception: " + str(e.message))
return_dict = {}
list_return_dict = list()
list_return_dict.append(return_dict)
return json.dumps(list_return_dict)
except Exception as e:
self.log("error", "Get Monitoring Info Summary Exception: " + str(e.message))
return_dict = {}
list_return_dict = list()
list_return_dict.append(return_dict)
return json.dumps(list_return_dict)
#self.log("debug", "Exited get_monitoring_info " + str(datetime.now()))
return json.dumps(list_return_dict)
def cleanup(self, obj):
if obj:
obj.kill()
# The Thread's run function continually checks the list of servers in the Server Mgr DB and polls them.
# It then calls other functions to send the information to the correct analytics server.
def run(self):
print "Starting monitoring thread"
#self.log("info", "Starting monitoring thread")
sel_log_dict = dict()
ipmi_list = list()
hostname_list = list()
server_ip_list = list()
ipmi_username_list = list()
ipmi_password_list = list()
ipmi_state = dict()
supported_sensors = ['FAN|.*_FAN', '^PWR', '.*Temp', '.*_Power']
while True:
servers = self._serverDb.get_server(
None, detail=True)
old_server_set = set(hostname_list)
del ipmi_list[:]
del hostname_list[:]
del server_ip_list[:]
del ipmi_username_list[:]
del ipmi_password_list[:]
#self.base_obj.populate_server_data_lists(servers, ipmi_list, hostname_list, server_ip_list,
# ipmi_username_list, ipmi_password_list, "monitoring")
server_dict = self.base_obj.create_server_dict(servers)
hostname_list = list(server_dict.keys())
new_server_set = set(hostname_list)
deleted_servers = set(old_server_set.difference(new_server_set))
if len(hostname_list) == 0:
time.sleep(self.freq)
continue
if len(deleted_servers) > 0:
#self.log("info", "Deleting monitoring info of certain servers that have been removed")
#self.log("info", "Deleted servers: " + str(list(deleted_servers)))
self.delete_monitoring_info(list(deleted_servers))
self.log("info", "Started IPMI Polling")
gevent_threads = dict()
gevent_priority_queue = gevent_queue.PriorityQueue()
for server_id in server_dict:
gevent_priority_queue.put(dict(server_dict[str(server_id)]))
total_no_of_servers = gevent_priority_queue.qsize()
sleep_period = self.sleep_period
servers_per_period = int(math.floor(float(total_no_of_servers / self.freq * sleep_period)))
#self.log("info", "Total number of servers this round: " + str(total_no_of_servers))
#self.log("info", "Servers per period this round: " + str(servers_per_period))
sleep_period = 1
time_set_success = False
while not time_set_success:
servers_per_period = int(math.floor(float(total_no_of_servers / self.freq * sleep_period)))
if servers_per_period >= 1:
time_set_success = True
else:
sleep_period += 1
try:
counter = servers_per_period
times_slept = 0
spawned = 0
total_spawned = 0
for server_id in server_dict:
counter -= 1
spawned += 1
total_spawned += 1
server = dict(server_dict[str(server_id)])
if server['id'] not in ipmi_state and server['id'] not in sel_log_dict:
ipmi_state[str(server['id'])] = True
sel_log_dict[str(server['id'])] = None
if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
in server and 'ipmi_password' in server and server['id'] and server['ip_address'] \
and server['ipmi_address'] and server['ipmi_username'] and server['ipmi_password']:
thread = gevent.spawn(
self.gevent_runner_func, server['id'], server['ipmi_address'], server['ip_address'],
server['ipmi_username'], server['ipmi_password'],
supported_sensors, ipmi_state[str(server['id'])], sel_log_dict[str(server['id'])],
self.ssh_access_method)
gevent_threads[str(server['id'])] = thread
else:
self.log("error", "Missing fields in server dictionary - skipping monitoring run")
if counter > 0:
pass
else:
#self.log("debug", "Round of Spawning completed. Sleeping for 10 secs. ")
#self.log("debug", "Number of gevents spawned this round: " + str(spawned))
#self.log("debug", "Total spawned: " + str(total_spawned))
time.sleep(sleep_period)
times_slept += 1
counter = servers_per_period
spawned = 0
#self.log("debug", "Slept for " + str(times_slept*sleep_period) + " s, sleeping for an additional " +
#str(self.freq - times_slept * sleep_period))
if (self.freq - times_slept*sleep_period) > 0:
time.sleep(max(self.freq-times_slept*sleep_period, 0))
else:
#self.log("debug", "No additional sleep. ")
pass
for hostname in gevent_threads:
thread = gevent_threads[str(hostname)]
if thread.successful() and thread.value:
return_dict = dict(thread.value)
ipmi_state[str(hostname)] = return_dict["ipmi_status"]
sel_log_dict[str(hostname)] = return_dict["sel_log"]
thread.kill()
else:
self.log("error", "Greenlet for server " + str(hostname) + " didn't return successfully: "
+ str(thread.get()))
thread.kill()
pass
except Exception as e:
self.log("error", "Exception occured while spawning gevents. Error = " + str(e))
pass
class ServerMgrInventory():
_serverDb = None
_inventory_log = None
_collectors_ip = None
_default_ipmi_username = None
_default_ipmi_password = None
_base_obj = None
DEBUG = "debug"
INFO = "info"
WARN = "warn"
ERROR = "error"
CRITICAL = "critical"
def __init__(self, smgr_ip, smgr_port, introspect_port, rev_tags_dict):
''' Constructor '''
self._base_obj = ServerMgrMonBasePlugin()
logging.config.fileConfig('/opt/contrail/server_manager/logger.conf')
# create logger
self._inventory_log = logging.getLogger('INVENTORY')
self.smgr_ip = smgr_ip
self.smgr_port = smgr_port
self.introspect_port = introspect_port
self.rev_tags_dict = rev_tags_dict
self.ssh_access_method = "key"
def set_serverdb(self, server_db):
self._serverDb = server_db
self._base_obj.set_serverdb(server_db=server_db)
def set_ipmi_defaults(self, ipmi_username, ipmi_password):
self._default_ipmi_username = ipmi_username
self._default_ipmi_password = ipmi_password
self._base_obj.set_ipmi_defaults(ipmi_username, ipmi_password)
def log(self, level, msg):
frame, filename, line_number, function_name, lines, index = inspect.stack(
)[1]
log_dict = dict()
log_dict['log_frame'] = frame
log_dict['log_filename'] = os.path.basename(filename)
log_dict['log_line_number'] = line_number
log_dict['log_function_name'] = function_name
log_dict['log_line'] = lines
log_dict['log_index'] = index
try:
if level == self.DEBUG:
self._inventory_log.debug(msg, extra=log_dict)
elif level == self.INFO:
self._inventory_log.info(msg, extra=log_dict)
elif level == self.WARN:
self._inventory_log.warn(msg, extra=log_dict)
elif level == self.ERROR:
self._inventory_log.error(msg, extra=log_dict)
elif level == self.CRITICAL:
self._inventory_log.critical(msg, extra=log_dict)
except Exception as e:
print "Error logging msg in Inv" + e.message
# call_send function is the sending function of the sandesh object (send_inst)
def call_send(self, send_inst):
#self.log(self.INFO, "Sending UVE Info over Sandesh")
#self.log("info", "UVE Info = " + str(send_inst.data))
send_inst.send()
@staticmethod
def fru_dict_init(hostname):
fru_dict = dict()
fru_dict['id'] = hostname
fru_dict['fru_description'] = "N/A"
fru_dict['chassis_type'] = "N/A"
fru_dict['chassis_serial_number'] = "N/A"
fru_dict['board_mfg_date'] = "N/A"
fru_dict['board_manufacturer'] = "N/A"
fru_dict['board_product_name'] = "N/A"
fru_dict['board_serial_number'] = "N/A"
fru_dict['board_part_number'] = "N/A"
fru_dict['product_manfacturer'] = "N/A"
fru_dict['product_name'] = "N/A"
fru_dict['product_part_number'] = "N/A"
return fru_dict
@staticmethod
def fru_obj_init(hostname):
fru_info_obj = fru_info()
fru_info_obj.fru_description = "N/A"
fru_info_obj.chassis_type = "N/A"
fru_info_obj.chassis_serial_number = "N/A"
fru_info_obj.board_mfg_date = "N/A"
fru_info_obj.board_manufacturer = "N/A"
fru_info_obj.board_product_name = "N/A"
fru_info_obj.board_serial_number = "N/A"
fru_info_obj.board_part_number = "N/A"
fru_info_obj.product_manfacturer = "N/A"
fru_info_obj.product_name = "N/A"
fru_info_obj.product_part_number = "N/A"
return fru_info_obj
def get_fru_info(self, hostname, ip, username, password):
cmd = 'ipmitool -H %s -U %s -P %s fru' % (ip, username, password)
try:
result = self._base_obj.call_subprocess(cmd)
sensor = ""
if result:
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
fileoutput = cStringIO.StringIO(result)
fru_obj_list = list()
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
for line in fileoutput:
if ":" in line:
reading = line.split(":")
sensor = reading[0].strip()
reading_value = reading[1].strip()
if reading_value == "":
reading_value = "N/A"
else:
sensor = ""
if sensor == "FRU Device Description":
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
fru_info_obj.fru_description = reading_value
fru_dict['fru_description'] = str(
hostname) + " " + reading_value
elif sensor == "Chassis Type":
fru_info_obj.chassis_type = reading_value
fru_dict['chassis_type'] = reading_value
elif sensor == "Chassis Serial":
fru_info_obj.chassis_serial_number = reading_value
fru_dict['chassis_serial_number'] = reading_value
elif sensor == "Board Mfg Date":
fru_info_obj.board_mfg_date = reading_value
fru_dict['board_mfg_date'] = reading_value
elif sensor == "Board Mfg":
fru_info_obj.board_manufacturer = reading_value
fru_dict['board_manufacturer'] = reading_value
elif sensor == "Board Product":
fru_info_obj.board_product_name = reading_value
fru_dict['board_product_name'] = reading_value
elif sensor == "Board Serial":
fru_info_obj.board_serial_number = reading_value
fru_dict['board_serial_number'] = reading_value
elif sensor == "Board Part Number":
fru_info_obj.board_part_number = reading_value
fru_dict['board_part_number'] = reading_value
elif sensor == "Product Manufacturer":
fru_info_obj.product_manfacturer = reading_value
fru_dict['product_manfacturer'] = reading_value
elif sensor == "Product Name":
fru_info_obj.product_name = reading_value
fru_dict['product_name'] = reading_value
elif sensor == "Product Part Number":
fru_info_obj.product_part_number = reading_value
fru_dict['product_part_number'] = reading_value
elif fru_dict['fru_description'] != "N/A" and sensor == "":
fru_obj_list.append(fru_info_obj)
rc = self._serverDb.add_inventory(fru_dict)
if rc != 0:
self.log(
self.ERROR,
"ERROR REPORTED BY INVENTORY ADD TO DICT: %s" %
rc)
fru_dict = self.fru_dict_init(hostname)
fru_info_obj = self.fru_obj_init(hostname)
if fru_dict['fru_description'] != "N/A":
fru_obj_list.append(fru_info_obj)
rc = self._serverDb.add_inventory(fru_dict)
if rc != 0:
self.log(
self.ERROR,
"ERROR REPORTED BY INVENTORY ADD TO DICT: %s" % rc)
inventory_info_obj.fru_infos = fru_obj_list
else:
self.log(self.INFO,
"Could not get the FRU info for IP: %s" % ip)
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
inventory_info_obj.fru_infos = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
except Exception as e:
self.log(
self.ERROR, "Could not get the FRU info for IP " + str(ip) +
" Error: %s" + str(e))
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = hostname
inventory_info_obj.fru_infos = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
raise e
@staticmethod
def inventory_lookup(key):
inv_dict = {
'hostname': 'name',
'boardproductname': 'board_product_name',
'boardserialnumber': 'board_serial_number',
'boardmanufacturer': 'board_manufacturer',
'hardwaremodel': 'hardware_model',
'interfaces': 'interface_name',
'physicalprocessorcount': 'physical_processor_count',
'processorcount': 'cpu_cores_count',
'virtual': 'virtual_machine',
'memorytotal': 'total_memory_mb',
'operatingsystem': 'os',
'operatingsystemrelease': 'os_version',
'osfamily': 'os_family',
'kernelversion': 'kernel_version',
'uptime_seconds': 'uptime_seconds',
'ipaddress': 'ip_addr',
'netmask': 'netmask',
'macaddress': 'macaddress'
}
if key in inv_dict:
return inv_dict[key]
else:
return None
def get_facter_info(self, hostname, ip, sshclient):
server_inventory_info = ServerInventoryInfo()
# Get the total number of disks
numdisks = self.get_field_value(sshclient, ip,
'lsblk | grep disk | wc -l')
is_ethtool = sshclient.exec_command('which ethtool')
server_inventory_info.total_numof_disks = int(numdisks)
# Get the other inventory information from the facter tool
try:
filestr = sshclient.exec_command('facter')
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
interface_dict = {}
intinfo_list = []
for line in fileoutput:
inventory = line.split('=>')
try:
key = inventory[0].strip()
if len(inventory) > 1:
value = inventory[1].strip()
if key == 'interfaces':
interface_list = value.split(',')
for name in interface_list:
# Skip the loopback interface
if name.strip() == 'lo':
continue
intinfo = interface_info()
intinfo.interface_name = name
intinfo.speed_Mb_per_sec = 0
intinfo.model = "N/A"
if not is_ethtool:
self.log(
self.DEBUG,
"ethtool not installed on host : %s" %
ip)
else:
eth_cmd = 'ethtool ' + name + ' | grep Speed'
driver_cmd = 'ethtool -i ' + name + ' | grep driver'
intinfo.speed_Mb_per_sec = self.get_field_value(
sshclient, ip, eth_cmd)
if bool(
re.search(
r'\d',
str(intinfo.speed_Mb_per_sec))
):
temp_var = re.findall(
'\d+|\D+',
str(intinfo.speed_Mb_per_sec))
intinfo.speed_Mb_per_sec = int(
temp_var[0].strip())
else:
intinfo.speed_Mb_per_sec = 0
intinfo.model = self.get_field_value(
sshclient, ip, driver_cmd)
exp = '.*_' + name + '.*$'
# exp = '(^ipaddress_|^macaddress_|^netmask_).*'+name+'.*$'
res = re.findall(exp, filestr, re.MULTILINE)
for items in res:
actualkey = items.split('=>')
namekey = actualkey[0].split('_')
objkey = self.inventory_lookup(
key=namekey[0].strip())
value = actualkey[1].strip()
if objkey:
setattr(intinfo, objkey, value)
if not getattr(intinfo, 'macaddress'):
setattr(intinfo, 'macaddress', "N/A")
if not getattr(intinfo, 'ip_addr'):
setattr(intinfo, 'ip_addr', "N/A")
if not getattr(intinfo, 'netmask'):
setattr(intinfo, 'netmask', "N/A")
intinfo_list.append(intinfo)
else:
objkey = self.inventory_lookup(key)
if key == 'physicalprocessorcount' or key == 'processorcount' or key == 'uptime_seconds':
value = int(value)
elif key == 'memorytotal':
memval = value.split()
value = math.trunc(float(memval[0]))
if memval[1].strip() == 'GB':
value *= 1024
if objkey:
setattr(server_inventory_info, objkey, value)
except Exception as KeyError:
self.log(
self.INFO, "keyerror: %s " + str(KeyError) +
" for IP: %s" % ip)
continue
server_inventory_info.name = str(hostname)
server_inventory_info.interface_infos = intinfo_list
self.call_send(
ServerInventoryInfoUve(data=server_inventory_info))
else:
self.log(self.ERROR,
"Could not get the Facter info for IP: %s" % ip)
except Exception as e:
self.log(
self.ERROR, "Could not get the Facter info for IP " + str(ip) +
" Error: %s" + str(e))
raise e
def get_field_value(self, sshclient, ip, cmd):
try:
filestr = sshclient.exec_command(cmd)
if not filestr:
return None
if cmd == "lsblk" or cmd == "facter" or "statistics" in cmd or cmd == "vmstat" or cmd == "lspci":
return filestr
else:
fileoutput = cStringIO.StringIO(filestr)
if fileoutput is not None:
for line in fileoutput:
if "=>" in line:
value = line.rstrip("\n").split("=>")[1].lstrip()
return value
elif ":" not in line:
return line
value = line.rstrip("\n").split(":")[1].lstrip()
return value
except Exception as e:
self.log(
self.ERROR, "Error in get_field_value: " + str(ip) +
" Command = " + str(cmd) + "Error: " + str(e))
return None
def get_cpu_info(self, hostname, ip, sshclient):
try:
# Get the CPU information from server
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.cpu_info_state = cpu_info()
check_cmd = 'cat /proc/cpuinfo'
lscpu__cmd = 'which lscpu'
server_inventory_info.cpu_info_state.model = "N/A"
server_inventory_info.cpu_info_state.core_count = 0
server_inventory_info.cpu_info_state.clock_speed_MHz = 0.0
server_inventory_info.cpu_info_state.num_of_threads = 0
if self.get_field_value(sshclient, ip,
check_cmd) and self.get_field_value(
sshclient, ip, lscpu__cmd):
model_cmd = 'cat /proc/cpuinfo | grep "model name" | head -n 1'
core_cmd = 'cat /proc/cpuinfo | grep "cpu cores" | head -n 1'
clock_cmd = 'cat /proc/cpuinfo | grep "cpu MHz" | head -n 1'
thread_cmd = 'lscpu | grep "Thread"'
server_inventory_info.cpu_info_state.model = self.get_field_value(
sshclient, ip, model_cmd)
server_inventory_info.cpu_info_state.core_count = int(
self.get_field_value(sshclient, ip, core_cmd))
server_inventory_info.cpu_info_state.clock_speed_MHz = float(
self.get_field_value(sshclient, ip, clock_cmd))
server_inventory_info.cpu_info_state.num_of_threads = int(
self.get_field_value(sshclient, ip, thread_cmd))
else:
self.log(self.DEBUG, "lscpu not installed on host : %s" % ip)
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(self.ERROR,
"Error in get_cpu_info: " + str(ip) + "Error: " + str(e))
raise e
def get_ethernet_info(self, hostname, ip, sshclient):
try:
# Get the Ethernet information from server
is_lspci = sshclient.exec_command('which lspci')
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.eth_controller_state = ethernet_controller()
server_inventory_info.eth_controller_state.num_of_ports = 0
if not is_lspci:
self.log(self.DEBUG, "lspci not installed on host : %s" % ip)
else:
port_cmd = 'lspci | grep Net | wc -l'
server_inventory_info.eth_controller_state.num_of_ports = int(
self.get_field_value(sshclient, ip, port_cmd))
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(
self.ERROR,
"Error in get_ethernet_info: " + str(ip) + "Error: " + str(e))
raise e
def get_memory_info(self, hostname, ip, sshclient):
try:
server_inventory_info = ServerInventoryInfo()
server_inventory_info.name = str(hostname)
server_inventory_info.mem_state = memory_info()
server_inventory_info.mem_state.mem_type = "N/A"
server_inventory_info.mem_state.mem_speed_MHz = 0
server_inventory_info.mem_state.num_of_dimms = 0
server_inventory_info.mem_state.dimm_size_mb = 0
server_inventory_info.mem_state.total_mem__mb = 0
server_inventory_info.mem_state.swap_size_mb = 0.0
dmi_cmd = 'which dmidecode'
if self.get_field_value(sshclient, ip, dmi_cmd):
type_cmd = 'dmidecode -t memory | grep "Type:" | grep -v "Unknown" | grep -v "Error" | head -n1'
mem_cmd = 'dmidecode -t memory | grep "Speed" | grep -v "Unknown" | head -n1'
dimm_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | head -n1'
num_cmd = 'dmidecode -t memory | grep "Size" | grep -v "No Module" | wc -l'
swap_cmd = 'facter | egrep -w swapsize_mb'
total_mem_cmd = 'vmstat -s | grep "total memory"'
server_inventory_info.mem_state.mem_type = self.get_field_value(
sshclient, ip, type_cmd)
mem_speed = self.get_field_value(sshclient, ip, mem_cmd)
unit = mem_speed.split(" ")[1]
if unit == "MHz":
server_inventory_info.mem_state.mem_speed_MHz = int(
mem_speed.split(" ")[0])
dimm_size = self.get_field_value(sshclient, ip, dimm_cmd)
unit = dimm_size.split(" ")[1]
if unit == "MB":
server_inventory_info.mem_state.dimm_size_mb = int(
dimm_size.split(" ")[0])
server_inventory_info.mem_state.num_of_dimms = int(
self.get_field_value(sshclient, ip, num_cmd))
swap_size = self.get_field_value(sshclient, ip, swap_cmd)
server_inventory_info.mem_state.swap_size_mb = float(
swap_size.split(" ")[0])
total_mem = self.get_field_value(sshclient, ip, total_mem_cmd)
server_inventory_info.mem_state.total_mem_mb = int(
int(total_mem.lstrip().split(" ")[0]) / 1024)
else:
self.log(self.INFO,
"Couldn't get the Memory info for IP: %s" % ip)
self.call_send(ServerInventoryInfoUve(data=server_inventory_info))
except Exception as e:
self.log(
self.ERROR,
"Error in get_memory_info: " + str(ip) + "Error: " + str(e))
raise e
def add_inventory(self):
servers = self._serverDb.get_server(None, detail=True)
gevent.spawn(self.handle_inventory_trigger, "add", servers)
def delete_inventory_info(self, hostname):
inventory_info_obj = ServerInventoryInfo()
inventory_info_obj.name = str(hostname)
inventory_info_obj.deleted = True
inventory_info_obj.fru_infos = None
inventory_info_obj.interface_infos = None
inventory_info_obj.cpu_info_state = None
inventory_info_obj.eth_controller_state = None
self.call_send(ServerInventoryInfoUve(data=inventory_info_obj))
def gevent_runner_function(self,
action,
hostname,
ip=None,
ipmi=None,
username=None,
password=None,
option="key"):
sshclient = None
if action == "add" and ip and ipmi and username and password:
try:
sshclient = ServerMgrSSHClient(serverdb=self._serverDb)
sshclient.connect(ip, hostname, option)
self.get_facter_info(hostname, ip, sshclient)
self.get_cpu_info(hostname, ip, sshclient)
self.get_ethernet_info(hostname, ip, sshclient)
self.get_memory_info(hostname, ip, sshclient)
sshclient.close()
except Exception as e:
if sshclient:
sshclient.close()
self.log(
"error", "Gevent SSH Connect Exception for server id: " +
str(hostname) + " Error : " + str(e))
pass
try:
self.get_fru_info(hostname, ipmi, username, password)
except Exception as e:
self.log(
"error",
"Error in getting Inventory Info through IPMI for server id: "
+ str(hostname) + " Error : " + str(e))
pass
elif action == "delete":
self.log(self.INFO, "Deleted info of server: %s" % hostname)
self.delete_inventory_info(hostname)
######## INVENTORY GET INFO SECTION ###########
def convert_type(self, field_dict):
data_type = field_dict["@type"]
if "#text" in field_dict:
if data_type == "bool":
return json.loads(field_dict["#text"])
elif data_type == "double":
return float(field_dict["#text"])
elif data_type == "u64":
return int(field_dict["#text"])
else:
return str(field_dict["#text"])
else:
return "N/A"
def filter_inventory_results(self, xml_dict, type_list):
return_dict = {}
if "all" in type_list:
return_dict = dict(xml_dict)
else:
selected_fields = set(xml_dict.keys()).intersection(type_list)
for selected_field in selected_fields:
return_dict[selected_field] = xml_dict[selected_field]
return return_dict
@staticmethod
def get_inv_conf_details(self):
return "Configuration for Inventory set correctly."
def get_inventory_info(self):
list_return_dict = list()
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
server_cluster_list = list()
server_tag_dict_list = list()
self.log(self.DEBUG, "get_inventory_info")
uve_name = "ServerInventoryInfo"
try:
entity = bottle.request
ret_data = self._base_obj.validate_rest_api_args(
entity, self.rev_tags_dict)
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {
"msg": ret_data["msg"],
"type_msg": ret_data["type_msg"]
}
if match_key == "tag":
match_dict = self._base_obj.process_server_tags(
self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
if match_dict.keys():
servers = self._serverDb.get_server(match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
#self.log(self.DEBUG, "Getting inventory info of following servers: " + str(servers))
if len(servers) == 1:
url = self._base_obj.get_sandesh_url(self.smgr_ip,
self.introspect_port,
uve_name,
dict(servers[0])['id'])
else:
url = self._base_obj.get_sandesh_url(self.smgr_ip,
self.introspect_port,
uve_name)
headers = {'content-type': 'application/json'}
resp = requests.get(url, timeout=300, headers=headers)
xml_data = resp.text
data = xmltodict.parse(str(xml_data))
data_dict = dict(data["__" + str(uve_name) + "Uve_list"])
parsed_data_list = self._base_obj.parse_sandesh_xml(
data_dict, uve_name)
parsed_data_dict = dict()
if parsed_data_list and servers:
for parsed_server in parsed_data_list:
parsed_server = dict(parsed_server)
parsed_data_dict[str(
parsed_server["data"]["name"])] = dict(
parsed_server["data"])
for server in servers:
server = dict(server)
server_hostname = str(server['id'])
if server_hostname in parsed_data_dict.keys():
return_dict = dict()
return_dict["name"] = str(server_hostname)
return_dict["cluster_id"] = server['cluster_id']
return_dict[str(
uve_name)] = self.filter_inventory_results(
parsed_data_dict[str(server_hostname)],
ret_data["type"])
list_return_dict.append(return_dict)
else:
self.log(self.ERROR,
"Server Details missing in cache. ")
self.log(self.ERROR,
"Server Hostname = " + str(server_hostname))
pass
else:
self.log(self.ERROR, "Server Details missing in db. ")
return {}
except ServerMgrException as e:
self.log(self.ERROR,
"Get Inventory Info Exception: " + str(e.message))
return json.dumps({})
except Exception as e:
self.log(self.ERROR,
"Get Inventory Info Exception: " + str(e.message))
return json.dumps({})
#self.log("debug", "Exited get_inventory_info " + str(datetime.now()))
return json.dumps(list_return_dict)
# end get_inventory_info
def run_inventory(self):
return_dict = dict()
match_dict = dict()
server_hostname_list = list()
self.log(self.DEBUG, "run_inventory")
try:
entity = bottle.request
ret_data = self._base_obj.validate_rest_api_args(
entity, self.rev_tags_dict)
#ret_data = self._base_obj.validate_run_inv_params(entity, self.rev_tags_dict)
if ret_data["status"]:
match_key = ret_data["match_key"]
match_value = ret_data["match_value"]
else:
return {
"msg": ret_data["msg"],
"type_msg": ret_data["type_msg"]
}
if match_key == "tag":
match_dict = self._base_obj.process_server_tags(
self.rev_tags_dict, match_value)
elif match_key:
match_dict[match_key] = match_value
if match_dict.keys():
servers = self._serverDb.get_server(match_dict, detail=True)
else:
servers = self._serverDb.get_server(detail=True)
self.handle_inventory_trigger("add", servers)
except ServerMgrException as e:
self.log("error", "Run Inventory Exception: " + e.message)
raise e
except Exception as e:
self.log("error", "Run Inventory Exception: " + e.message)
raise e
inventory_status = dict()
inventory_status['return_message'] = "server(s) run_inventory issued"
return inventory_status
def handle_inventory_trigger(self, action, servers):
server_dict = self._base_obj.create_server_dict(servers)
gevent_threads = []
if len(server_dict.keys()) >= 1:
for server_id in server_dict:
server = dict(server_dict[str(server_id)])
if action == "add":
if 'id' in server and 'ip_address' in server and 'ipmi_address' in server and 'ipmi_username' \
in server and 'ipmi_password' in server:
thread = gevent.spawn(
self.gevent_runner_function, action, server['id'],
server['ip_address'], server['ipmi_address'],
server['ipmi_username'], server['ipmi_password'],
self.ssh_access_method)
gevent_threads.append(thread)
else:
self.log(
self.ERROR,
"Missing fields in server dictionary - skipping inventory addition"
)
elif action == "delete":
if 'id' in server:
thread = gevent.spawn(self.gevent_runner_function,
action, server['id'],
self.ssh_access_method)
gevent_threads.append(thread)
else:
self.log(
self.ERROR,
"Missing id field in server dictionary - skipping inventory deletion"
)
time.sleep(1)
self.log(self.DEBUG, "Finished Running Inventory")
class ServerMgrIPMIMonitoring(ServerMgrMonBasePlugin):
def __init__(self, val, frequency, collectors_ip=None):
''' Constructor '''
ServerMgrMonBasePlugin.__init__(self)
self.base_obj = ServerMgrMonBasePlugin()
self.val = val
self.freq = float(frequency)
self._serverDb = None
self._collectors_ip = collectors_ip
# call_send function is the sending function of the sandesh object (send_inst)
def call_send(self, send_inst):
self.base_obj.log("info", "Sending UVE Info over Sandesh")
send_inst.send()
# send_ipmi_stats function packages and sends the IPMI info gathered from server polling
# to the analytics node
def send_ipmi_stats(self, ipmi_data, hostname):
sm_ipmi_info = SMIpmiInfo()
sm_ipmi_info.name = str(hostname)
sm_ipmi_info.sensor_stats = []
sm_ipmi_info.sensor_state = []
for ipmidata in ipmi_data:
ipmi_stats = IpmiSensor()
ipmi_stats.sensor = ipmidata.sensor
ipmi_stats.reading = ipmidata.reading
ipmi_stats.status = ipmidata.status
ipmi_stats.unit = ipmidata.unit
ipmi_stats.sensor_type = ipmidata.sensor_type
sm_ipmi_info.sensor_stats.append(ipmi_stats)
sm_ipmi_info.sensor_state.append(ipmi_stats)
ipmi_stats_trace = SMIpmiInfoTrace(data=sm_ipmi_info)
self.call_send(ipmi_stats_trace)
# sandesh_init function opens a sandesh connection to the analytics node's ip
# (this is recevied from Server Mgr's config or cluster config). The function is called only once.
# For this node, a list of collector ips is passed to the sandesh init_generator.
def sandesh_init(self, collectors_ip_list=None):
try:
self.base_obj.log("info", "Initializing sandesh")
# storage node module initialization part
module = Module.IPMI_STATS_MGR
module_name = ModuleNames[module]
node_type = Module2NodeType[module]
node_type_name = NodeTypeNames[node_type]
instance_id = INSTANCE_ID_DEFAULT
if collectors_ip_list:
self.base_obj.log("info", "Collector IPs from config: "+str(collectors_ip_list))
collectors_ip_list = eval(collectors_ip_list)
sandesh_global.init_generator(
module_name,
socket.gethostname(),
node_type_name,
instance_id,
collectors_ip_list,
module_name,
HttpPortIpmiStatsmgr,
['contrail_sm_monitoring.ipmi'])
else:
raise ServerMgrException("Error during Sandesh Init: No collector ips or Discovery server/port given")
except Exception as e:
raise ServerMgrException("Error during Sandesh Init: " + str(e))
def return_collector_ip(self):
return self._collectors_ip
# The Thread's run function continually checks the list of servers in the Server Mgr DB and polls them.
# It then calls other functions to send the information to the correct analytics server.
def run(self):
print "Starting monitoring thread"
self.base_obj.log("info", "Starting monitoring thread")
ipmi_data = []
supported_sensors = ['FAN|.*_FAN', '^PWR', '.*TEMP', '.*Temp', '.*_Power']
while True:
servers = self._serverDb.get_server(
None, detail=True)
ipmi_list = list()
hostname_list = list()
server_ip_list = list()
ipmi_username_list = list()
ipmi_password_list = list()
data = ""
sensor_type = None
if self._collectors_ip:
for server in servers:
server = dict(server)
if 'ipmi_address' in server:
if server['ipmi_address']:
ipmi_list.append(server['ipmi_address'])
else:
continue
else:
continue
if 'id' in server:
if server['id']:
hostname_list.append(server['id'])
else:
ipmi_list.pop()
continue
else:
ipmi_list.pop()
continue
if 'ip_address' in server:
if server['ip_address']:
server_ip_list.append(server['ip_address'])
else:
ipmi_list.pop()
hostname_list.pop()
continue
else:
ipmi_list.pop()
hostname_list.pop()
continue
if 'ipmi_username' in server and server['ipmi_username']:
ipmi_username_list.append(server['ipmi_username'])
else:
ipmi_username_list.append(self._default_ipmi_username)
if 'ipmi_password' in server and server['ipmi_password']:
ipmi_password_list.append(server['ipmi_password'])
else:
ipmi_password_list.append(self._default_ipmi_password)
self.base_obj.log("info", "Started IPMI Polling")
for ip, hostname, username, password in \
zip(ipmi_list, hostname_list, ipmi_username_list, ipmi_password_list):
ipmi_data = []
cmd = 'ipmitool -H %s -U %s -P %s sdr list all' % (ip, username, password)
result = super(ServerMgrIPMIMonitoring, self).call_subprocess(cmd)
if result is not None:
fileoutput = cStringIO.StringIO(result)
for line in fileoutput:
reading = line.split("|")
sensor = reading[0].strip()
reading_value = reading[1].strip()
status = reading[2].strip()
for i in supported_sensors:
if re.search(i, sensor) is not None:
if 'FAN' in sensor:
sensor_type = 'fan'
elif 'PWR' in sensor or 'Power' in sensor:
sensor_type = 'power'
elif 'Temp' in sensor:
sensor_type = 'temperature'
value = reading_value.split()
ipmidata = IpmiData()
ipmidata.sensor = sensor
ipmidata.status = status
if status == "ns":
pass
elif status == "ok":
ipmidata.reading = long(value[0].strip())
ipmidata.unit = value[len(value) - 1].strip()
ipmidata.sensor_type = sensor_type
ipmi_data.append(ipmidata)
else:
self.base_obj.log("info", "IPMI Polling failed for " + str(ip))
self.send_ipmi_stats(ipmi_data, hostname=hostname)
else:
self.base_obj.log("error", "IPMI Polling: No Analytics IP info found")
self.base_obj.log("info", "Monitoring thread is sleeping for " + str(self.freq) + " seconds")
time.sleep(self.freq)
self.base_obj.log("info", "Monitoring thread woke up")
