class EventHandler: def __init__(self, env: str, inventory_collection: str): super().__init__() self.inv = InventoryMgr() self.inv.set_collections(inventory_collection) self.env = env self.log = FullLogger(env=env) self.handlers = {} def discover_handlers(self, handlers_package: str, event_handlers: dict): if not event_handlers: raise TypeError("Event handlers list is empty") for event_name, handler_name in event_handlers.items(): handler = ClassResolver.get_instance_of_class(handler_name, handlers_package) if not issubclass(handler.__class__, EventBase): raise TypeError("Event handler '{}' is not a subclass of EventBase" .format(handler_name)) if event_name in self.handlers: self.log.warning("A handler is already registered for event type '{}'. Overwriting" .format(event_name)) self.handlers[event_name] = handler def handle(self, event_name: str, notification: dict) -> EventResult: if event_name not in self.handlers: self.log.info("No handler is able to process event of type '{}'" .format(event_name)) return self.handlers[event_name].handle(self.env, notification)
class TestScan(unittest.TestCase): def configure_environment(self): self.env = ENV_CONFIG self.inventory_collection = COLLECTION_CONFIG # mock the mongo access MongoAccess.mongo_connect = MagicMock() MongoAccess.db = MagicMock() # mock log FullLogger.info = MagicMock() self.conf = Configuration() self.conf.use_env = MagicMock() self.conf.environment = CONFIGURATIONS self.conf.configuration = CONFIGURATIONS["configuration"] self.inv = InventoryMgr() self.inv.clear = MagicMock() self.inv.set_collections(self.inventory_collection) MonitoringSetupManager.server_setup = MagicMock() DbAccess.get_neutron_db_name = MagicMock() DbAccess.get_neutron_db_name.return_value = "neutron" def setUp(self): self.configure_environment()
def __init__(self, env: str, inventory_collection: str): super().__init__() self.inv = InventoryMgr() self.inv.set_collections(inventory_collection) self.env = env self.log = FullLogger(env=env) self.handlers = {}
def __init__(self, env): super().__init__() self.log = FullLogger() self.configuration = Configuration() self.mechanism_drivers = \ self.configuration.environment['mechanism_drivers'] self.env = env self.had_errors = False self.monitoring_config = self.db.monitoring_config_templates try: self.env_monitoring_config = self.configuration.get('Monitoring') except IndexError: self.env_monitoring_config = {} self.local_host = self.env_monitoring_config.get('server_ip', '') self.scripts_prepared_for_host = {} self.replacements = self.env_monitoring_config self.inv = InventoryMgr() self.config_db = self.db[self.inv.get_coll_name('monitoring_config')] self.provision = self.provision_levels['none'] if self.env_monitoring_config: provision = self.env_monitoring_config.get('provision', 'none') provision = str.lower(provision) self.provision =\ self.provision_levels.get(provision, self.provision_levels['none'])
def __init__(self): self.args = self.get_args() MongoAccess.set_config_file(self.args.mongo_config) self.inv = InventoryMgr() self.inv.set_collections(self.args.inventory) self.configuration = Configuration() self.input_text = None self.converter = SpecialCharConverter()
class TestFetch(unittest.TestCase): def setUp(self): self._mongo_connect = MongoAccess.mongo_connect self._mongo_db = MongoAccess.db self._db_access_conn = DbAccess.conn self._ssh_connect = SshConnection.connect self._ssh_conn_check_defs = SshConnection.check_definitions self._ssh_check_defs = SshConn.check_definitions self.req_patcher = patch("discover.fetchers.api.api_access.requests") self.requests = self.req_patcher.start() self.response = MagicMock() self.response.codes.ok = 200 self.response.json = Mock(return_value=CORRECT_AUTH_CONTENT) self.response.status_code = self.requests.codes.ok self.requests.get.return_value = self.response self.requests.post.return_value = self.response self.ssh_patcher = patch("discover.fetchers.cli.cli_access.SshConn") self.ssh_conn = self.ssh_patcher.start().return_value def configure_environment(self): self.env = ENV_CONFIG self.inventory_collection = COLLECTION_CONFIG # mock the Mongo Access MongoAccess.mongo_connect = MagicMock() MongoAccess.db = MagicMock() self.conf = Configuration() self.conf.use_env = MagicMock() self.conf.environment = CONFIGURATIONS self.conf.configuration = CONFIGURATIONS["configuration"] self.inv = InventoryMgr() self.inv.set_collections(self.inventory_collection) DbAccess.conn = MagicMock() DbAccess.get_neutron_db_name = MagicMock() DbAccess.get_neutron_db_name.return_value = "neutron" SshConnection.connect = MagicMock() SshConnection.check_definitions = MagicMock() SshConn.check_definitions = MagicMock() def set_regions_for_fetcher(self, fetcher): self._regions = fetcher.regions fetcher.regions = REGIONS def reset_regions_for_fetcher(self, fetcher): fetcher.regions = self._regions def tearDown(self): MongoAccess.mongo_connect = self._mongo_connect MongoAccess.db = self._mongo_db DbAccess.conn = self._db_access_conn SshConnection.connect = self._ssh_connect SshConnection.check_definitions = self._ssh_conn_check_defs SshConn.check_definitions = self._ssh_check_defs self.req_patcher.stop() self.ssh_patcher.stop()
def __init__(self, env: str, host_id: str): super().__init__(env) self.cli_ssh = self.get_ssh(host_id) self.inv = InventoryMgr() self.host = self.inv.get_by_id(env, host_id) self.server = self.env_monitoring_config.get('server_ip') self.server_cli_ssh = self.get_ssh(self.server) self.ubuntu_dist = None self.required_package = None
def __init__(self, args): super().__init__() self.log = FullLogger() self.log.set_loglevel(args.loglevel) self.env = args.env try: self.conf = Configuration(args.mongo_config) self.inv = InventoryMgr() self.inv.log.set_loglevel(args.loglevel) self.inv.set_collections(args.inventory) except FileNotFoundError: sys.exit(1)
def __init__(self): """ Scanner is the base class for scanners. """ super().__init__() self.config = Configuration() self.inv = InventoryMgr() self.scanners_package = None self.scanners = {} self.link_finders = [] self.load_scanners_metadata() self.load_link_finders_metadata()
class CliFetchInstanceVnicsBase(CliAccess): def __init__(self): super().__init__() self.inv = InventoryMgr() def get(self, id): instance_uuid = id[:id.rindex('-')] instance = self.inv.get_by_id(self.get_env(), instance_uuid) if not instance: return [] host = self.inv.get_by_id(self.get_env(), instance["host"]) if not host or "Compute" not in host["host_type"]: return [] lines = self.run_fetch_lines("virsh list", instance["host"]) del lines[:2] # remove header virsh_ids = [l.split()[0] for l in lines if l > ""] results = [] # Note: there are 2 ids here of instances with local names, which are # not connected to the data we have thus far for the instance # therefore, we will decide whether the instance is the correct one # based on comparison of the uuid in the dumpxml output for id in virsh_ids: results.extend(self.get_vnics_from_dumpxml(id, instance)) return results def get_vnics_from_dumpxml(self, id, instance): xml_string = self.run("virsh dumpxml " + id, instance["host"]) if not xml_string.strip(): return [] response = xmltodict.parse(xml_string) if instance["uuid"] != response["domain"]["uuid"]: # this is the wrong instance - skip it return [] try: vnics = response["domain"]["devices"]["interface"] except KeyError: return [] if isinstance(vnics, dict): vnics = [vnics] for v in vnics: self.set_vnic_properties(v, instance) return vnics def set_vnic_properties(self, v, instance): v["name"] = self.get_vnic_name(v, instance) v["id"] = "{}-{}".format(instance["host"], v["name"]) v["vnic_type"] = "instance_vnic" v["host"] = instance["host"] v["instance_id"] = instance["id"] v["instance_db_id"] = instance["_id"] v["mac_address"] = v["mac"]["@address"] instance["mac_address"] = v["mac_address"] self.inv.set(instance)
def __init__(self, mongo_config="", ldap_config="", log_level="", inventory="", token_lifetime=86400): MongoAccess.set_config_file(mongo_config) self.inv = InventoryMgr() self.inv.set_collections(inventory) self.log = FullLogger() self.log.set_loglevel(log_level) self.ldap_access = LDAPAccess(ldap_config) Token.set_token_lifetime(token_lifetime) self.middleware = AuthenticationMiddleware() self.app = falcon.API(middleware=[self.middleware]) self.app.add_error_handler(CalipsoApiException) self.set_routes(self.app)
def configure(self): self.db_client = MongoAccess() self.inv = InventoryMgr() self.inv.set_collections(self.args.inventory) self.collection = self.db_client.db[self.args.collection] self.interval = max(self.MIN_INTERVAL, self.args.interval) self.log.set_loglevel(self.args.loglevel) self.log.info("Started EventManager with following configuration:\n" "Mongo config file path: {0}\n" "Collection: {1}\n" "Polling interval: {2} second(s)".format( self.args.mongo_config, self.collection.name, self.interval))
class App: ROUTE_DECLARATIONS = { "/inventory": "resource.inventory.Inventory", "/links": "resource.links.Links", "/messages": "resource.messages.Messages", "/cliques": "resource.cliques.Cliques", "/clique_types": "resource.clique_types.CliqueTypes", "/clique_constraints": "resource.clique_constraints.CliqueConstraints", "/scans": "resource.scans.Scans", "/scheduled_scans": "resource.scheduled_scans.ScheduledScans", "/constants": "resource.constants.Constants", "/monitoring_config_templates": "resource.monitoring_config_templates.MonitoringConfigTemplates", "/aggregates": "resource.aggregates.Aggregates", "/environment_configs": "resource.environment_configs.EnvironmentConfigs", "/connection_tests": "resource.connection_tests.ConnectionTests", "/auth/tokens": "auth.tokens.Tokens" } responders_path = "api.responders" def __init__(self, mongo_config="", ldap_config="", log_level="", inventory="", token_lifetime=86400): MongoAccess.set_config_file(mongo_config) self.inv = InventoryMgr() self.inv.set_collections(inventory) self.log = FullLogger() self.log.set_loglevel(log_level) self.ldap_access = LDAPAccess(ldap_config) Token.set_token_lifetime(token_lifetime) self.middleware = AuthenticationMiddleware() self.app = falcon.API(middleware=[self.middleware]) self.app.add_error_handler(CalipsoApiException) self.set_routes(self.app) def get_app(self): return self.app def set_routes(self, app): for url in self.ROUTE_DECLARATIONS.keys(): class_path = self.ROUTE_DECLARATIONS.get(url) module = self.responders_path + "." + \ class_path[:class_path.rindex(".")] class_name = class_path.split('.')[-1] module = importlib.import_module(module) class_ = getattr(module, class_name) resource = class_() app.add_route(url, resource)
def __init__(self): self.get_args() MongoAccess.set_config_file(self.args.mongo_config) MongoAccess.__init__(self) self.log = FullLogger() self.log.set_loglevel(self.args.loglevel) self.conf = Configuration() self.inv = InventoryMgr() self.inv.set_collections(self.args.inventory) stats_coll = self.inv.get_coll_name('statistics') self.stats = self.db[stats_coll] # consume messages from topic self.consumer = KafkaConsumer('VPP.stats', group_id='calipso_test', auto_offset_reset=self.args.offset, bootstrap_servers=['localhost:9092'])
class FindLinks(Fetcher): def __init__(self): super().__init__() self.inv = InventoryMgr() def create_link(self, env, source, source_id, target, target_id, link_type, link_name, state, link_weight, host=None, switch=None, implicit=False, extra_attributes=None): if extra_attributes is None: extra_attributes = {} source_label = extra_attributes.get('source_label', '') target_label = extra_attributes.get('target_label', '') link = self.inv.create_link(env, source, source_id, target, target_id, link_type, link_name, state, link_weight, implicit=implicit, source_label=source_label, target_label=target_label, host=host, switch=switch, extra_attributes=extra_attributes) if self.inv.monitoring_setup_manager: self.inv.monitoring_setup_manager.create_setup(link) return link
class ApiFetchPort(ApiAccess): def __init__(self): super(ApiFetchPort, self).__init__() self.inv = InventoryMgr() def get(self, project_id): if not project_id: self.log.info("Get method needs ID parameter") return [] # use project admin credentials, to be able to fetch all ports token = self.v2_auth_pwd(self.admin_project) if not token: return [] ret = [] for region in self.regions: ret.append(self.get_port(region, token, project_id)) if ret == []: self.log.info("ApiFetchPort: Port not found.") return ret def get_port(self, region, token, id): endpoint = self.get_region_url_nover(region, "neutron") req_url = endpoint + "/v2.0/ports/" + id headers = { "X-Auth-Project-Id": self.admin_project, "X-Auth-Token": token["id"] } response = self.get_url(req_url, headers) if not "port" in response: return [] doc = response["port"] doc["master_parent_type"] = "network" doc["master_parent_id"] = doc["network_id"] doc["parent_type"] = "ports_folder" doc["parent_id"] = doc["network_id"] + "-ports" doc["parent_text"] = "Ports" # get the project name net = self.inv.get_by_id(self.get_env(), doc["network_id"]) if net: doc["name"] = doc["mac_address"] else: doc["name"] = doc["id"] project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) if project: doc["project"] = project["name"] return doc
def __init__(self): super().__init__() self.inv = InventoryMgr() self.if_header = re.compile('^\d+: ([^:]+): (.+)') self.regexps = [{ 'name': 'mac_address', 're': '^.*\slink/ether\s(\S+)\s' }, { 'name': 'IP Address', 're': '^\s*inet ([0-9.]+)/' }, { 'name': 'netmask', 're': '^\s*inet [0-9.]+/([0-9]+)' }, { 'name': 'IPv6 Address', 're': '^\s*inet6 ([^/]+)/.* global ' }]
def __init__(self): super().__init__() self.inv = InventoryMgr() self.ethtool_attr = re.compile('^\s+([^:]+):\s(.*)$') self.regexps = [{ 'name': 'mac_address', 're': '^.*\slink/ether\s(\S+)\s', 'description': 'MAC address' }, { 'name': 'IP Address', 're': '^\s*inet ([0-9.]+)/', 'description': 'IP Address v4' }, { 'name': 'IPv6 Address', 're': '^\s*inet6 (\S+) .* global ', 'description': 'IPv6 Address' }]
class ApiFetchHostInstances(ApiAccess, DbAccess, metaclass=Singleton): def __init__(self): super(ApiFetchHostInstances, self).__init__() self.inv = InventoryMgr() self.endpoint = ApiAccess.base_url.replace(":5000", ":8774") self.projects = None self.db_fetcher = DbFetchInstances() def get_projects(self): if not self.projects: projects_list = self.inv.get(self.get_env(), "project", None) self.projects = [p["name"] for p in projects_list] def get(self, id): self.get_projects() host_id = id[:id.rindex("-")] host = self.inv.get_by_id(self.get_env(), host_id) if not host or "Compute" not in host.get("host_type", ""): return [] instances_found = self.get_instances_from_api(host_id) self.db_fetcher.get_instance_data(instances_found) return instances_found def get_instances_from_api(self, host_name): token = self.v2_auth_pwd(self.admin_project) if not token: return [] tenant_id = token["tenant"]["id"] req_url = self.endpoint + "/v2/" + tenant_id + \ "/os-hypervisors/" + host_name + "/servers" response = self.get_url(req_url, {"X-Auth-Token": token["id"]}) ret = [] if not "hypervisors" in response: return [] if not "servers" in response["hypervisors"][0]: return [] for doc in response["hypervisors"][0]["servers"]: doc["id"] = doc["uuid"] doc["host"] = host_name doc["local_name"] = doc.pop("name") ret.append(doc) self.log.info("found %s instances for host: %s", str(len(ret)), host_name) return ret
def __init__(self): super().__init__() self.env_config = None self.inv = InventoryMgr() self.inventory = self.inv.inventory_collection self.links = self.inv.collections["links"] self.clique_types = self.inv.collections["clique_types"] self.clique_types_by_type = {} self.clique_constraints = self.inv.collections["clique_constraints"] self.cliques = self.inv.collections["cliques"]
def __init__(self, environments_collection="environments_config"): super().__init__() self.db_client = MongoAccess() self.db = MongoAccess.db self.inv = InventoryMgr() self.collection = self.inv.collections.get(environments_collection) self.env_name = None self.environment = None self.configuration = None self.log = FullLogger()
class ApiFetchPorts(ApiAccess): def __init__(self): super(ApiFetchPorts, self).__init__() self.inv = InventoryMgr() def get(self, project_id): # use project admin credentials, to be able to fetch all ports token = self.v2_auth_pwd(self.admin_project) if not token: return [] ret = [] for region in self.regions: ret.extend(self.get_ports_for_region(region, token)) return ret def get_ports_for_region(self, region, token): endpoint = self.get_region_url_nover(region, "neutron") req_url = endpoint + "/v2.0/ports" headers = { "X-Auth-Project-Id": self.admin_project, "X-Auth-Token": token["id"] } response = self.get_url(req_url, headers) if not "ports" in response: return [] ports = response["ports"] for doc in ports: doc["master_parent_type"] = "network" doc["master_parent_id"] = doc["network_id"] doc["parent_type"] = "ports_folder" doc["parent_id"] = doc["network_id"] + "-ports" doc["parent_text"] = "Ports" # get the project name net = self.inv.get_by_id(self.get_env(), doc["network_id"]) if net: doc["name"] = doc["mac_address"] else: doc["name"] = doc["id"] project = self.inv.get_by_id(self.get_env(), doc["tenant_id"]) if project: doc["project"] = project["name"] return ports
def get(self, id): query = """ SELECT CONCAT('aggregate-', a.name, '-', host) AS id, host AS name FROM nova.aggregate_hosts ah JOIN nova.aggregates a ON a.id = ah.aggregate_id WHERE ah.deleted = 0 AND aggregate_id = %s """ hosts = self.get_objects_list_for_id(query, "host", id) if hosts: inv = InventoryMgr() for host_rec in hosts: host_id = host_rec['name'] host = inv.get_by_id(self.get_env(), host_id) if not host: self.log.error('unable to find host {} ' 'from aggregate {} in inventory'.format( host_id, id)) continue host_rec['ref_id'] = bson.ObjectId(host['_id']) return hosts
def configure_environment(self): self.env = ENV_CONFIG self.inventory_collection = COLLECTION_CONFIG # mock the Mongo Access MongoAccess.mongo_connect = MagicMock() MongoAccess.db = MagicMock() self.conf = Configuration() self.conf.use_env = MagicMock() self.conf.environment = CONFIGURATIONS self.conf.configuration = CONFIGURATIONS["configuration"] self.inv = InventoryMgr() self.inv.set_collections(self.inventory_collection) DbAccess.conn = MagicMock() DbAccess.get_neutron_db_name = MagicMock() DbAccess.get_neutron_db_name.return_value = "neutron" SshConnection.connect = MagicMock() SshConnection.check_definitions = MagicMock() SshConn.check_definitions = MagicMock()
def configure(self): self.db_client = MongoAccess() self.inv = InventoryMgr() self.inv.set_collections() self.scans_collection = self.db_client.db[self.args.scans_collection] self.scheduled_scans_collection = \ self.db_client.db[self.args.scheduled_scans_collection] self.environments_collection = \ self.db_client.db[self.args.environments_collection] self._update_document = \ partial(MongoAccess.update_document, self.scans_collection) self.interval = max(self.MIN_INTERVAL, self.args.interval) self.log.set_loglevel(self.args.loglevel) self.log.info("Started ScanManager with following configuration:\n" "Mongo config file path: {0.args.mongo_config}\n" "Scans collection: {0.scans_collection.name}\n" "Environments collection: " "{0.environments_collection.name}\n" "Polling interval: {0.interval} second(s)".format(self))
def __init__(self, connection: Connection, event_handler: EventHandler, event_queues: List, env_name: str = DEFAULTS["env"], inventory_collection: str = DEFAULTS["inventory"], retry_limit: int = DEFAULTS["retry_limit"], consume_all: bool = DEFAULTS["consume_all"]): super().__init__() self.connection = connection self.retry_limit = retry_limit self.env_name = env_name self.consume_all = consume_all self.handler = event_handler self.event_queues = event_queues self.failing_messages = defaultdict(int) self.inv = InventoryMgr() self.inv.set_collections(inventory_collection) if self.inv.is_feature_supported(self.env_name, EnvironmentFeatures.MONITORING): self.inv.monitoring_setup_manager = \ MonitoringSetupManager(self.env_name)
class MongoLoggingHandler(logging.Handler): """ Logging handler for MongoDB """ SOURCE_SYSTEM = 'Calipso' def __init__(self, env: str, level: str, origin: Origin = None): super().__init__(Logger.get_numeric_level(level)) self.str_level = level self.env = env self.inv = None self.origin = origin def emit(self, record): # Try to invoke InventoryMgr for logging if not self.inv: try: self.inv = InventoryMgr() except: return # make sure we do not try to log to DB when DB is not ready if not (self.inv.is_db_ready() and 'messages' in self.inv.collections): return # make ID from current timestamp now = datetime.datetime.utcnow() d = now - datetime.datetime(1970, 1, 1) timestamp_id = '{}.{}.{}'.format(d.days, d.seconds, d.microseconds) source = self.SOURCE_SYSTEM message = Message(msg_id=timestamp_id, env=self.env, source=source, msg=Logger.formatter.format(record), ts=now, level=record.levelname) if self.origin: message.extra['origin_id'] = ( str(self.origin.origin_id) if self.origin.origin_id else None ) message.extra['origin_type'] = ( self.origin.origin_type.value if self.origin.origin_type else None ) for extra_field in self.origin.extra: message.extra[extra_field] = getattr(self.origin, extra_field) self.inv.collections['messages'].insert_one(message.get())
def emit(self, record): # Try to invoke InventoryMgr for logging if not self.inv: try: self.inv = InventoryMgr() except: return # make sure we do not try to log to DB when DB is not ready if not (self.inv.is_db_ready() and 'messages' in self.inv.collections): return # make ID from current timestamp now = datetime.datetime.utcnow() d = now - datetime.datetime(1970, 1, 1) timestamp_id = '{}.{}.{}'.format(d.days, d.seconds, d.microseconds) source = self.SOURCE_SYSTEM message = Message(msg_id=timestamp_id, env=self.env, source=source, msg=Logger.formatter.format(record), ts=now, level=record.levelname) if self.origin: message.extra['origin_id'] = ( str(self.origin.origin_id) if self.origin.origin_id else None ) message.extra['origin_type'] = ( self.origin.origin_type.value if self.origin.origin_type else None ) for extra_field in self.origin.extra: message.extra[extra_field] = getattr(self.origin, extra_field) self.inv.collections['messages'].insert_one(message.get())
class CliFetchVconnectors(CliAccess, metaclass=ABCSingleton): def __init__(self): super().__init__() self.inv = InventoryMgr() @abstractmethod def get_vconnectors(self, host): raise NotImplementedError("Subclass must override get_vconnectors()") def get(self, id): host_id = id[:id.rindex('-')] host = self.inv.get_by_id(self.get_env(), host_id) if not host: self.log.error("CliFetchVconnectors: host not found: " + host_id) return [] if "host_type" not in host: self.log.error("host does not have host_type: " + host_id + \ ", host: " + str(host)) return [] return self.get_vconnectors(host)
class DbFetchVedgesVpp(DbAccess, CliAccess, metaclass=Singleton): def __init__(self): super().__init__() self.inv = InventoryMgr() def get(self, id): host_id = id[:id.rindex('-')] vedge = { 'host': host_id, 'id': host_id + '-VPP', 'name': 'VPP-' + host_id, 'agent_type': 'VPP' } ver = self.run_fetch_lines('vppctl show ver', host_id) if ver: ver = ver[0] vedge['binary'] = ver[:ver.index(' ', ver.index(' ') + 1)] host = self.inv.get_by_id(self.get_env(), host_id) if not host: self.log.error("unable to find host in inventory: %s", host_id) return [] host_types = host["host_type"] if "Network" not in host_types and "Compute" not in host_types: return [] interfaces = self.run_fetch_lines('vppctl show int', host_id) vedge['ports'] = self.fetch_ports(interfaces) return [vedge] def fetch_ports(self, interfaces): ports = {} for i in interfaces: if not i or i.startswith(' '): continue parts = i.split() port = { 'id': parts[1], 'state': parts[2], 'name': parts[0] } ports[port['name']] = port return ports