class App:
ROUTE_DECLARATIONS = {
"/inventory": "resource.inventory.Inventory",
"/links": "resource.links.Links",
"/messages": "resource.messages.Messages",
"/cliques": "resource.cliques.Cliques",
"/clique_types": "resource.clique_types.CliqueTypes",
"/clique_constraints": "resource.clique_constraints.CliqueConstraints",
"/scans": "resource.scans.Scans",
"/scheduled_scans": "resource.scheduled_scans.ScheduledScans",
"/constants": "resource.constants.Constants",
"/monitoring_config_templates":
"resource.monitoring_config_templates.MonitoringConfigTemplates",
"/aggregates": "resource.aggregates.Aggregates",
"/environment_configs":
"resource.environment_configs.EnvironmentConfigs",
"/connection_tests": "resource.connection_tests.ConnectionTests",
"/auth/tokens": "auth.tokens.Tokens"
}
responders_path = "api.responders"
def __init__(self, mongo_config="", ldap_config="",
log_level="", inventory="", token_lifetime=86400):
MongoAccess.set_config_file(mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(inventory)
self.log = FullLogger()
self.log.set_loglevel(log_level)
self.ldap_access = LDAPAccess(ldap_config)
Token.set_token_lifetime(token_lifetime)
self.middleware = AuthenticationMiddleware()
self.app = falcon.API(middleware=[self.middleware])
self.app.add_error_handler(CalipsoApiException)
self.set_routes(self.app)
def get_app(self):
return self.app
def set_routes(self, app):
for url in self.ROUTE_DECLARATIONS.keys():
class_path = self.ROUTE_DECLARATIONS.get(url)
module = self.responders_path + "." + \
class_path[:class_path.rindex(".")]
class_name = class_path.split('.')[-1]
module = importlib.import_module(module)
class_ = getattr(module, class_name)
resource = class_()
app.add_route(url, resource)
def listen(args: dict = None):
args = setup_args(args, DefaultListener.DEFAULTS, get_args)
if 'process_vars' not in args:
args['process_vars'] = {}
env_name = args["env"]
inventory_collection = args["inventory"]
MongoAccess.set_config_file(args["mongo_config"])
inv = InventoryMgr()
inv.set_collections(inventory_collection)
conf = Configuration(args["environments_collection"])
conf.use_env(env_name)
event_handler = EventHandler(env_name, inventory_collection)
event_queues = []
env_config = conf.get_env_config()
common_metadata_file = os.path.join(env_config.get('app_path', '/etc/calipso'),
'config',
DefaultListener.COMMON_METADATA_FILE)
# import common metadata
import_metadata(event_handler, event_queues, common_metadata_file)
# import custom metadata if supplied
if args["metadata_file"]:
import_metadata(event_handler, event_queues, args["metadata_file"])
logger = FullLogger()
logger.set_loglevel(args["loglevel"])
amqp_config = conf.get("AMQP")
connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \
.format(user=amqp_config["user"],
pwd=amqp_config["pwd"],
host=amqp_config["host"],
port=amqp_config["port"])
with Connection(connect_url) as conn:
try:
print(conn)
conn.connect()
args['process_vars']['operational'] = OperationalStatus.RUNNING
terminator = SignalHandler()
worker = \
DefaultListener(connection=conn,
event_handler=event_handler,
event_queues=event_queues,
retry_limit=args["retry_limit"],
consume_all=args["consume_all"],
inventory_collection=inventory_collection,
env_name=env_name)
worker.run()
if terminator.terminated:
args.get('process_vars', {})['operational'] = \
OperationalStatus.STOPPED
except KeyboardInterrupt:
print('Stopped')
args['process_vars']['operational'] = OperationalStatus.STOPPED
except Exception as e:
logger.log.exception(e)
args['process_vars']['operational'] = OperationalStatus.ERROR
finally:
# This should enable safe saving of shared variables
time.sleep(0.1)
class MonitoringCheckHandler(SpecialCharConverter):
STATUS_LABEL = ['OK', 'Warning', 'Error']
def __init__(self, args):
super().__init__()
self.log = FullLogger()
self.log.set_loglevel(args.loglevel)
self.env = args.env
try:
self.conf = Configuration(args.mongo_config)
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args.loglevel)
self.inv.set_collections(args.inventory)
except FileNotFoundError:
sys.exit(1)
def doc_by_id(self, object_id):
doc = self.inv.get_by_id(self.env, object_id)
if not doc:
self.log.warn('No matching object found with ID: ' + object_id)
return doc
def doc_by_db_id(self, db_id, coll_name=None):
coll = self.inv.collections[coll_name] if coll_name else None
doc = self.inv.find({'_id': ObjectId(db_id)},
get_single=True,
collection=coll)
if not doc:
self.log.warn('No matching object found with DB ID: ' + db_id)
return doc
def set_doc_status(self, doc, status, status_text, timestamp):
doc['status'] = self.STATUS_LABEL[status] if isinstance(status, int) \
else status
if status_text:
doc['status_text'] = status_text
doc['status_timestamp'] = strftime(TIME_FORMAT, timestamp)
if 'link_type' in doc:
self.inv.write_link(doc)
else:
self.inv.set(doc)
@staticmethod
def check_ts(check_result):
return gmtime(check_result['executed'])
def keep_result(self, doc, check_result):
status = check_result['status']
ts = self.check_ts(check_result)
self.set_doc_status(doc, status, check_result['output'], ts)
self.keep_message(doc, check_result)
def keep_message(self, doc, check_result, error_level=None):
is_link = 'link_type' in doc
msg_id = check_result['id']
obj_id = 'link_{}_{}'.format(doc['source_id'], doc['target_id']) \
if is_link \
else doc['id']
obj_type = 'link_{}'.format(
doc['link_type']) if is_link else doc['type']
display_context = obj_id if is_link \
else doc['network_id'] if doc['type'] == 'port' else doc['id']
level = error_level if error_level\
else ERROR_LEVEL[check_result['status']]
dt = datetime.datetime.utcfromtimestamp(check_result['executed'])
message = Message(msg_id=msg_id,
env=self.env,
source=SOURCE_SYSTEM,
object_id=obj_id,
object_type=obj_type,
display_context=display_context,
level=level,
msg=check_result,
ts=dt)
collection = self.inv.collections['messages']
collection.insert_one(message.get())
class StatsConsumer(MongoAccess):
default_env = "WebEX-Mirantis@Cisco"
def __init__(self):
self.get_args()
MongoAccess.set_config_file(self.args.mongo_config)
MongoAccess.__init__(self)
self.log = FullLogger()
self.log.set_loglevel(self.args.loglevel)
self.conf = Configuration()
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
stats_coll = self.inv.get_coll_name('statistics')
self.stats = self.db[stats_coll]
# consume messages from topic
self.consumer = KafkaConsumer('VPP.stats',
group_id='calipso_test',
auto_offset_reset=self.args.offset,
bootstrap_servers=['localhost:9092'])
def get_args(self):
# try to read scan plan from command line parameters
parser = argparse.ArgumentParser()
parser.add_argument("-m", "--mongo_config", nargs="?", type=str,
default="",
help="name of config file " +
"with MongoDB servr access details")
parser.add_argument("-e", "--env", nargs="?", type=str,
default=self.default_env,
help="name of environment to scan \n" +
"(default: " + self.default_env + ")")
parser.add_argument("-y", "--inventory", nargs="?", type=str,
default="inventory",
help="name of inventory collection \n" +
"(default: 'inventory')")
parser.add_argument("-l", "--loglevel", nargs="?", type=str,
default="INFO",
help="logging level \n(default: 'INFO')")
parser.add_argument("-o", "--offset", nargs="?", type=str,
default="largest",
help="where to start reading" +
" - use 'smallest' for start \n" +
"(default: 'largest')")
self.args = parser.parse_args()
def read(self):
for kafka_msg in self.consumer:
msg = json.loads(kafka_msg.value.decode())
self.add_stats(msg)
def add_stats(self, msg):
host_ip = msg['hostIp']
search = {
'environment': self.args.env,
'type': 'host',
'ip_address': host_ip
}
host = self.inv.find_items(search, get_single=True)
if not host:
self.log.error('could not find host with ip address=' + host_ip)
return
host_id = host['id']
search = {
'environment': self.args.env,
'type': 'vedge',
'host': host_id
}
vedge = self.inv.find_items(search, get_single=True)
if not vedge:
self.log.error('could not find vEdge for host: ' + host_id)
return
self.log.info('setting VPP stats for vEdge of host: ' + host_id)
self.add_stats_for_object(vedge, msg)
def add_stats_for_object(self, o, msg):
msg['type'] = 'vedge_flows'
msg['environment'] = self.args.env
msg['object_type'] = o['type']
msg['object_id'] = o['id']
time_seconds = int(msg['averageArrivalNanoSeconds'] / 1000000000)
sample_time = time.gmtime(time_seconds)
msg['sample_time'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", sample_time)
# find instances between which the flow happens
# to find the instance, find the related vNIC first
msg['source'] = self.find_instance_for_stat('source', msg)
msg['destination'] = self.find_instance_for_stat('destination', msg)
self.stats.insert_one(msg)
def find_instance_for_stat(self, direction, msg):
search_by_mac_address = 'sourceMacAddress' in msg
value_attr = 'MacAddress' if search_by_mac_address else 'IpAddress'
value_to_search = msg[direction + value_attr]
attr = 'mac_address' if search_by_mac_address else 'ip_address'
search = {
'environment': self.args.env,
'type': 'vnic',
attr: value_to_search
}
vnic = self.inv.find_items(search, get_single=True)
if not vnic:
self.log.error('failed to find vNIC for ' +
attr + '=' + value_to_search)
return 'Unknown'
# now find the instance name from the vnic name
name_path = vnic['name_path'].split('/')
instance_name = name_path[8]
return instance_name
class MonitoringCheckHandler(SpecialCharConverter):
status_labels = {}
TIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
def __init__(self, args):
super().__init__()
self.log = FullLogger()
self.log.set_loglevel(args.loglevel)
self.env = args.env
try:
self.conf = Configuration(args.mongo_config)
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args.loglevel)
self.inv.set_collections(args.inventory)
self.status_labels = self.get_status_labels()
except FileNotFoundError:
sys.exit(1)
def get_status_labels(self):
statuses_name_search = {'name': 'monitoring_check_statuses'}
labels_data = self.inv.find_one(search=statuses_name_search,
collection='constants')
if not isinstance(labels_data, dict) or 'data' not in labels_data:
return ''
labels = {}
for status_data in labels_data['data']:
if not isinstance(status_data, dict):
continue
val = int(status_data['value'])
label = status_data['label']
labels[val] = label
return labels
def get_label_for_status(self, status: int) -> str:
if status not in self.status_labels.keys():
return ''
return self.status_labels.get(status, '')
def doc_by_id(self, object_id):
doc = self.inv.get_by_id(self.env, object_id)
if not doc:
self.log.warn('No matching object found with ID: ' + object_id)
return doc
def doc_by_db_id(self, db_id, coll_name=None):
coll = self.inv.collections[coll_name] if coll_name else None
doc = self.inv.find({'_id': ObjectId(db_id)},
get_single=True,
collection=coll)
if not doc:
self.log.warn('No matching object found with DB ID: ' + db_id)
return doc
def set_doc_status(self, doc, status, status_text, timestamp):
doc['status_value'] = status if isinstance(status, int) \
else status
doc['status'] = self.get_label_for_status(status) \
if isinstance(status, int) \
else status
if status_text:
doc['status_text'] = status_text
doc['status_timestamp'] = strftime(self.TIME_FORMAT, timestamp)
if 'link_type' in doc:
self.inv.write_link(doc)
else:
self.inv.set(doc)
@staticmethod
def check_ts(check_result):
return gmtime(check_result['executed'])
def keep_result(self, doc, check_result):
status = check_result['status']
ts = self.check_ts(check_result)
self.set_doc_status(doc, status, check_result['output'], ts)
self.keep_message(doc, check_result)
def keep_message(self, doc, check_result, error_level=None):
is_link = 'link_type' in doc
msg_id = check_result['id']
obj_id = 'link_{}_{}'.format(doc['source_id'], doc['target_id']) \
if is_link \
else doc['id']
obj_type = 'link_{}'.format(doc['link_type']) if is_link else \
doc['type']
display_context = obj_id if is_link \
else doc['network_id'] if doc['type'] == 'port' else doc['id']
level = error_level if error_level\
else ERROR_LEVEL[check_result['status']]
dt = datetime.datetime.utcfromtimestamp(check_result['executed'])
message = Message(msg_id=msg_id,
env=self.env,
source=SOURCE_SYSTEM,
object_id=obj_id,
object_type=obj_type,
display_context=display_context,
level=level,
msg=check_result,
ts=dt)
collection = self.inv.collections['messages']
collection.insert_one(message.get())