class Monitor:
DEFAULTS = {
'env': 'WebEX-Mirantis@Cisco',
'inventory': 'inventory',
'loglevel': 'WARNING'
}
def __init__(self):
self.args = self.get_args()
MongoAccess.set_config_file(self.args.mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
self.configuration = Configuration()
self.input_text = None
self.converter = SpecialCharConverter()
def get_args(self):
parser = argparse.ArgumentParser()
parser.add_argument("-m",
"--mongo_config",
nargs="?",
type=str,
default="",
help="name of config file with MongoDB server " +
"access details")
parser.add_argument("-e",
"--env",
nargs="?",
type=str,
default=self.DEFAULTS['env'],
help="name of environment to scan \n" +
"(default: {})".format(self.DEFAULTS['env']))
parser.add_argument("-y",
"--inventory",
nargs="?",
type=str,
default=self.DEFAULTS['inventory'],
help="name of inventory collection \n" +
"(default: {}".format(self.DEFAULTS['inventory']))
parser.add_argument('-i',
'--inputfile',
nargs='?',
type=str,
default='',
help="read input from the specifed file \n" +
"(default: from stdin)")
parser.add_argument("-l",
"--loglevel",
nargs="?",
type=str,
default=self.DEFAULTS["loglevel"],
help="logging level \n(default: '{}')".format(
self.DEFAULTS["loglevel"]))
args = parser.parse_args()
return args
def get_type_list(self, type_name) -> list:
types_list = []
docs = self.inv.find_items({'name': type_name}, collection='constants')
for types_list in docs:
types_list = [t['value'] for t in types_list['data']]
if not types_list:
raise ValueError('Unable to fetch {}'.format(
type_name.replace('_', ' ')))
return types_list
def match_object_types(self, check_name: str) -> list:
object_types = self.get_type_list('object_types')
matches = [t for t in object_types if check_name.startswith(t + '_')]
return matches
def match_link_types(self, check_name: str) -> list:
object_types = self.get_type_list('link_types')
matches = [
t for t in object_types if check_name.startswith('link_' + t + '_')
]
return matches
def find_object_type_and_id(self, check_name: str):
# if we have multiple matching host types, then take the longest
# of these. For example, if matches are ['host', 'host_pnic'],
# then take 'host_pnic'.
# To facilitate this, we sort the matches by reverse order.
is_link_check = check_name.startswith('link_')
check_type = 'link' if is_link_check else 'object'
if is_link_check:
matching_types = sorted(self.match_link_types(check_name),
reverse=True)
else:
matching_types = sorted(self.match_object_types(check_name),
reverse=True)
if not matching_types:
raise ValueError(
'Unable to match check name "{}" with {} type'.format(
check_name, check_type))
obj_type = matching_types[0]
postfix_len = len('link_') if is_link_check else 0
obj_id = (obj_type + '_' if is_link_check else '') + \
check_name[len(obj_type)+1+postfix_len:]
return check_type, obj_type, obj_id
def read_input(self):
if self.args.inputfile:
try:
with open(self.args.inputfile, 'r') as input_file:
self.input_text = input_file.read()
except Exception as e:
raise FileNotFoundError(
"failed to open input file {}: {}".format(
self.args.inputfile, str(e)))
else:
self.input_text = sys.stdin.read()
if not self.input_text:
raise ValueError("No input provided on stdin")
def get_handler_by_type(self, check_type, obj_type):
module_name = 'handle_link' if check_type == 'link' \
else 'handle_' + obj_type
package = 'monitoring.handlers'
handler = ClassResolver.get_instance_single_arg(
self.args, module_name=module_name, package_name=package)
return handler
def get_handler(self, check_type, obj_type):
basic_handling_types = ['instance', 'vedge', 'vservice', 'vconnector']
if obj_type not in basic_handling_types:
return self.get_handler_by_type(check_type, obj_type)
from monitoring.handlers.basic_check_handler \
import BasicCheckHandler
return BasicCheckHandler(self.args)
def check_link_interdependency_for(self,
object_id: str,
from_type: str = None,
to_type: str = None):
if from_type is not None and to_type is not None or \
from_type is None and to_type is None:
raise ValueError('check_link_interdependency: '
'supply one of from_type/to_type')
obj_id = self.converter.decode_special_characters(object_id)
obj = self.inv.get_by_id(environment=self.args.env, item_id=obj_id)
if not obj:
self.inv.log.error(
'check_link_interdependency: '
'failed to find object with ID: {}'.format(object_id))
return
if 'status' not in obj:
return
id_attr = 'source_id' if from_type is None else 'target_id'
link_type = '{}-{}'.format(
from_type if from_type is not None else obj['type'],
to_type if to_type is not None else obj['type'])
condition = {
'environment': self.args.env,
'link_type': link_type,
id_attr: obj_id
}
link = self.inv.find_one(search=condition, collection='links')
if not link:
self.inv.log.error('check_link_interdependency: '
'failed to find {} link with {}: {}'.format(
link_type, id_attr, obj_id))
return
other_id_attr = '{}_id' \
.format('source' if from_type is not None else 'target')
other_obj = self.inv.get_by_id(environment=self.args.env,
item_id=link[other_id_attr])
if not other_obj:
self.inv.log.error(
'check_link_interdependency: '
'failed to find {} with ID: {} (link type: {})'.format(
other_id_attr, link[other_id_attr], link_type))
return
if 'status' not in other_obj:
return
status = 'Warning'
if obj['status'] == 'OK' and other_obj['status'] == 'OK':
status = 'OK'
elif obj['status'] == 'OK' and other_obj['status'] == 'OK':
status = 'OK'
link['status'] = status
time_format = MonitoringCheckHandler.TIME_FORMAT
timestamp1 = obj['status_timestamp']
t1 = datetime.datetime.strptime(timestamp1, time_format)
timestamp2 = other_obj['status_timestamp']
t2 = datetime.datetime.strptime(timestamp2, time_format)
timestamp = max(t1, t2)
link['status_timestamp'] = datetime.datetime.strftime(
timestamp, time_format)
self.inv.set(link, self.inv.collections['links'])
def check_link_interdependency(self, object_id: str, object_type: str):
conf = self.configuration.get_env_config()
if 'OVS' in conf['mechanism_drivers']:
if object_type == 'vedge':
self.check_link_interdependency_for(object_id,
to_type='host_pnic')
if object_type == 'host_pnic':
self.check_link_interdependency_for(object_id,
from_type='vedge')
def process_input(self):
check_result_full = json.loads(self.input_text)
check_client = check_result_full['client']
check_result = check_result_full['check']
check_result['id'] = check_result_full['id']
name = check_result['name']
check_type, object_type, object_id = \
monitor.find_object_type_and_id(name)
if 'environment' in check_client:
self.args.env = check_client['environment']
else:
raise ValueError('Check client should contain environment name')
self.configuration.use_env(self.args.env)
check_handler = self.get_handler(check_type, object_type)
if check_handler:
check_handler.handle(object_id, check_result)
self.check_link_interdependency(object_id, object_type)
def process_check_result(self):
self.read_input()
self.process_input()
class CliFetchBondHostPnics(CliAccess):
BOND_DIR = '/proc/net/bonding/'
SLAVE_INTERFACE_HEADER = 'Slave Interface: '
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
def get(self, parent_id: str):
self.log.info(
'CliFetchBondHostPnics: checking under {}'.format(parent_id))
host_id = parent_id[:parent_id.rindex('-')]
cmd = 'ls -1 {} 2>&1'.format(self.BOND_DIR)
host = self.inv.get_by_id(self.get_env(), host_id)
if not host:
self.log.error('CliFetchBondHostPnics: host not found: ' + host_id)
return []
host_types = host['host_type']
if 'Network' not in host_types and 'Compute' not in host_types:
return []
lines = self.run_fetch_lines(cmd, host_id)
if lines and 'No such file or directory' in lines[0]:
return [] # no bonds so directory does not exist
bonds = []
for line in [l.strip() for l in lines]:
bond = self.get_bond_details(host_id, line)
if bond:
bonds.append(bond)
return bonds
def get_bond_details(self, host_id: str, interface_name: str) -> dict:
lines = self.run_fetch_lines(
'cat {}{}'.format(self.BOND_DIR, interface_name), host_id)
status, mac_address = \
self.get_bond_status_and_mac_address(host_id, interface_name)
interface_id = '{}-{}'.format(interface_name, mac_address)
interface = {
'host': host_id,
'name': interface_name,
'id': interface_id,
'local_name': interface_name,
'mac_address': mac_address,
'Link detected': 'yes' if status == 'up' else 'no',
'EtherChannel': True,
'EtherChannel Master': '',
'members': {}
}
# keep stack of info objects to support multi-level info
info_objects = deque([interface])
for line in [line for line in lines if line != '']:
if line.startswith(self.SLAVE_INTERFACE_HEADER):
name = line[line.index(':') + 1:].strip()
slave = {'name': name, 'EtherChannel Master': interface_id}
# remove any pending info objects, keep only interface
info_objects = deque([interface])
info_objects.append(slave)
interface['members'][name] = slave
elif line.rstrip(':').lower().endswith('info'):
# move to lower level info object
info_name = line.rstrip(':')
upper_info_obj = info_objects[-1]
info_obj = {}
upper_info_obj[info_name] = info_obj
info_objects.append(info_obj)
else:
self.get_attribute_from_line(info_objects[-1], line)
for slave in list(interface['members'].values()):
self.set_slave_host_pnic_bond_attributes(host_id, slave,
interface_id)
return interface
def get_bond_status_and_mac_address(self, host_id: str, name: str):
output = self.run_fetch_lines('ip link show {}'.format(name), host_id)
status_line = output[0]
status = status_line[status_line.index(' state ') + len(' state '):]
status = status[:status.index(' ')]
matches = [line.strip() for line in output if 'link/ether' in line]
if not matches:
self.log.error('Failed to find line with MAC address '
'for bond {} (host: {})'.format(name, host_id))
tokens = matches[0].split()
if len(tokens) < 2:
self.log.error('Failed to find MAC address in line: {}'.format(
matches[0]))
mac_address = tokens[1]
return status.lower(), mac_address
def get_attribute_from_line(self, obj: dict, line: str):
if ':' not in line:
self.log.error('object {}: failed to find ":" in line: {}'.format(
obj['name'], line))
return
attr = line[:line.index(':')]
value = line[len(attr) + 1:]
obj[attr.strip()] = value.strip()
def set_slave_host_pnic_bond_attributes(self, host, slave, interface_id):
pnic = self.inv.find_one({
'environment': self.get_env(),
'host': host,
'type': 'host_pnic',
'name': slave['name']
})
if not pnic:
self.log.error('unable to find slave pNIC {} under bond {}'.format(
slave['name'], interface_id))
return
mac_address = pnic['mac_address']
slave_id = '{}-{}'.format(slave.get('name', ''), mac_address)
slave['mac_address'] = mac_address
slave['id'] = slave_id
pnic['EtherChannel'] = True
pnic['EtherChannel Master'] = interface_id
self.inv.set(pnic)
class MonitoringCheckHandler(SpecialCharConverter):
status_labels = {}
TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
def __init__(self, args):
super().__init__()
self.log = FullLogger()
self.log.set_loglevel(args.loglevel)
self.env = args.env
try:
self.conf = Configuration(args.mongo_config)
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args.loglevel)
self.inv.set_collections(args.inventory)
self.status_labels = self.get_status_labels()
except FileNotFoundError:
sys.exit(1)
def get_status_labels(self):
statuses_name_search = {'name': 'monitoring_check_statuses'}
labels_data = self.inv.find_one(search=statuses_name_search,
collection='constants')
if not isinstance(labels_data, dict) or 'data' not in labels_data:
return ''
labels = {}
for status_data in labels_data['data']:
if not isinstance(status_data, dict):
continue
val = int(status_data['value'])
label = status_data['label']
labels[val] = label
return labels
def get_label_for_status(self, status: int) -> str:
if status not in self.status_labels.keys():
return ''
return self.status_labels.get(status, '')
def doc_by_id(self, object_id):
doc = self.inv.get_by_id(self.env, object_id)
if not doc:
self.log.warn('No matching object found with ID: ' + object_id)
return doc
def doc_by_db_id(self, db_id, coll_name=None):
coll = self.inv.collections[coll_name] if coll_name else None
doc = self.inv.find({'_id': ObjectId(db_id)},
get_single=True,
collection=coll)
if not doc:
self.log.warn('No matching object found with DB ID: ' + db_id)
return doc
def set_doc_status(self, doc, status, status_text, timestamp):
doc['status_value'] = status if isinstance(status, int) \
else status
doc['status'] = self.get_label_for_status(status) \
if isinstance(status, int) \
else status
if status_text:
doc['status_text'] = status_text
doc['status_timestamp'] = strftime(self.TIME_FORMAT, timestamp)
if 'link_type' in doc:
self.inv.write_link(doc)
else:
self.inv.set(doc)
@staticmethod
def check_ts(check_result):
return gmtime(check_result['executed'])
def keep_result(self, doc, check_result):
status = check_result['status']
ts = self.check_ts(check_result)
self.set_doc_status(doc, status, check_result['output'], ts)
self.keep_message(doc, check_result)
def keep_message(self, doc, check_result, error_level=None):
is_link = 'link_type' in doc
msg_id = check_result['id']
obj_id = 'link_{}_{}'.format(doc['source_id'], doc['target_id']) \
if is_link \
else doc['id']
obj_type = 'link_{}'.format(doc['link_type']) if is_link else \
doc['type']
display_context = obj_id if is_link \
else doc['network_id'] if doc['type'] == 'port' else doc['id']
level = error_level if error_level\
else ERROR_LEVEL[check_result['status']]
dt = datetime.datetime.utcfromtimestamp(check_result['executed'])
message = Message(msg_id=msg_id,
env=self.env,
source=SOURCE_SYSTEM,
object_id=obj_id,
object_type=obj_type,
display_context=display_context,
level=level,
msg=check_result,
ts=dt)
collection = self.inv.collections['messages']
collection.insert_one(message.get())
