def __init__(self, env: str, inventory_collection: str):
super().__init__()
self.inv = InventoryMgr()
self.inv.set_collections(inventory_collection)
self.env = env
self.log = FullLogger(env=env)
self.handlers = {}
def __init__(self, env):
super().__init__()
self.log = FullLogger()
self.configuration = Configuration()
self.mechanism_drivers = \
self.configuration.environment['mechanism_drivers']
self.env = env
self.had_errors = False
self.monitoring_config = self.db.monitoring_config_templates
try:
self.env_monitoring_config = self.configuration.get('Monitoring')
except IndexError:
self.env_monitoring_config = {}
self.local_host = self.env_monitoring_config.get('server_ip', '')
self.scripts_prepared_for_host = {}
self.replacements = self.env_monitoring_config
self.inv = InventoryMgr()
self.config_db = self.db[self.inv.get_coll_name('monitoring_config')]
self.provision = self.provision_levels['none']
if self.env_monitoring_config:
provision = self.env_monitoring_config.get('provision', 'none')
provision = str.lower(provision)
self.provision =\
self.provision_levels.get(provision,
self.provision_levels['none'])
def __init__(self):
self.args = self.get_args()
MongoAccess.set_config_file(self.args.mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
self.configuration = Configuration()
self.input_text = None
self.converter = SpecialCharConverter()
def __init__(self, env: str, host_id: str):
super().__init__(env)
self.cli_ssh = self.get_ssh(host_id)
self.inv = InventoryMgr()
self.host = self.inv.get_by_id(env, host_id)
self.server = self.env_monitoring_config.get('server_ip')
self.server_cli_ssh = self.get_ssh(self.server)
self.ubuntu_dist = None
self.required_package = None
def __init__(self, environments_collection="environments_config"):
super().__init__()
self.db_client = MongoAccess()
self.db = MongoAccess.db
self.inv = InventoryMgr()
self.collection = self.inv.collections.get(environments_collection)
self.env_name = None
self.environment = None
self.configuration = None
self.log = FullLogger()
def __init__(self):
super().__init__()
self.env_config = None
self.inv = InventoryMgr()
self.inventory = self.inv.inventory_collection
self.links = self.inv.collections["links"]
self.clique_types = self.inv.collections["clique_types"]
self.clique_types_by_type = {}
self.clique_constraints = self.inv.collections["clique_constraints"]
self.cliques = self.inv.collections["cliques"]
def __init__(self):
"""
Scanner is the base class for scanners.
"""
super().__init__()
self.config = Configuration()
self.inv = InventoryMgr()
self.scanners_package = None
self.scanners = {}
self.link_finders = []
self.load_scanners_metadata()
self.load_link_finders_metadata()
def __init__(self, args):
super().__init__()
self.log = FullLogger()
self.log.set_loglevel(args.loglevel)
self.env = args.env
try:
self.conf = Configuration(args.mongo_config)
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args.loglevel)
self.inv.set_collections(args.inventory)
except FileNotFoundError:
sys.exit(1)
def __init__(self, mongo_config="", ldap_config="",
log_level="", inventory="", token_lifetime=86400):
MongoAccess.set_config_file(mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(inventory)
self.log = FullLogger()
self.log.set_loglevel(log_level)
self.ldap_access = LDAPAccess(ldap_config)
Token.set_token_lifetime(token_lifetime)
self.middleware = AuthenticationMiddleware()
self.app = falcon.API(middleware=[self.middleware])
self.app.add_error_handler(CalipsoApiException)
self.set_routes(self.app)
def configure(self):
self.db_client = MongoAccess()
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
self.collection = self.db_client.db[self.args.collection]
self.interval = max(self.MIN_INTERVAL, self.args.interval)
self.log.set_loglevel(self.args.loglevel)
self.log.info("Started EventManager with following configuration:\n"
"Mongo config file path: {0}\n"
"Collection: {1}\n"
"Polling interval: {2} second(s)".format(
self.args.mongo_config, self.collection.name,
self.interval))
def __init__(self):
self.get_args()
MongoAccess.set_config_file(self.args.mongo_config)
MongoAccess.__init__(self)
self.log = FullLogger()
self.log.set_loglevel(self.args.loglevel)
self.conf = Configuration()
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
stats_coll = self.inv.get_coll_name('statistics')
self.stats = self.db[stats_coll]
# consume messages from topic
self.consumer = KafkaConsumer('VPP.stats',
group_id='calipso_test',
auto_offset_reset=self.args.offset,
bootstrap_servers=['localhost:9092'])
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.if_header = re.compile('^\d+: ([^:]+): (.+)')
self.regexps = [{
'name': 'mac_address',
're': '^.*\slink/ether\s(\S+)\s'
}, {
'name': 'IP Address',
're': '^\s*inet ([0-9.]+)/'
}, {
'name': 'netmask',
're': '^\s*inet [0-9.]+/([0-9]+)'
}, {
'name': 'IPv6 Address',
're': '^\s*inet6 ([^/]+)/.* global '
}]
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.ethtool_attr = re.compile('^\s+([^:]+):\s(.*)$')
self.regexps = [{
'name': 'mac_address',
're': '^.*\slink/ether\s(\S+)\s',
'description': 'MAC address'
}, {
'name': 'IP Address',
're': '^\s*inet ([0-9.]+)/',
'description': 'IP Address v4'
}, {
'name': 'IPv6 Address',
're': '^\s*inet6 (\S+) .* global ',
'description': 'IPv6 Address'
}]
def configure(self):
self.db_client = MongoAccess()
self.inv = InventoryMgr()
self.inv.set_collections()
self.scans_collection = self.db_client.db[self.args.scans_collection]
self.scheduled_scans_collection = \
self.db_client.db[self.args.scheduled_scans_collection]
self.environments_collection = \
self.db_client.db[self.args.environments_collection]
self._update_document = \
partial(MongoAccess.update_document, self.scans_collection)
self.interval = max(self.MIN_INTERVAL, self.args.interval)
self.log.set_loglevel(self.args.loglevel)
self.log.info("Started ScanManager with following configuration:\n"
"Mongo config file path: {0.args.mongo_config}\n"
"Scans collection: {0.scans_collection.name}\n"
"Environments collection: "
"{0.environments_collection.name}\n"
"Polling interval: {0.interval} second(s)".format(self))
def get(self, id):
query = """
SELECT CONCAT('aggregate-', a.name, '-', host) AS id, host AS name
FROM nova.aggregate_hosts ah
JOIN nova.aggregates a ON a.id = ah.aggregate_id
WHERE ah.deleted = 0 AND aggregate_id = %s
"""
hosts = self.get_objects_list_for_id(query, "host", id)
if hosts:
inv = InventoryMgr()
for host_rec in hosts:
host_id = host_rec['name']
host = inv.get_by_id(self.get_env(), host_id)
if not host:
self.log.error('unable to find host {} '
'from aggregate {} in inventory'.format(
host_id, id))
continue
host_rec['ref_id'] = bson.ObjectId(host['_id'])
return hosts
def configure_environment(self):
self.env = ENV_CONFIG
self.inventory_collection = COLLECTION_CONFIG
# mock the Mongo Access
MongoAccess.mongo_connect = MagicMock()
MongoAccess.db = MagicMock()
self.conf = Configuration()
self.conf.use_env = MagicMock()
self.conf.environment = CONFIGURATIONS
self.conf.configuration = CONFIGURATIONS["configuration"]
self.inv = InventoryMgr()
self.inv.set_collections(self.inventory_collection)
DbAccess.conn = MagicMock()
DbAccess.get_neutron_db_name = MagicMock()
DbAccess.get_neutron_db_name.return_value = "neutron"
SshConnection.connect = MagicMock()
SshConnection.check_definitions = MagicMock()
SshConn.check_definitions = MagicMock()
def __init__(self, connection: Connection,
event_handler: EventHandler,
event_queues: List,
env_name: str = DEFAULTS["env"],
inventory_collection: str = DEFAULTS["inventory"],
retry_limit: int = DEFAULTS["retry_limit"],
consume_all: bool = DEFAULTS["consume_all"]):
super().__init__()
self.connection = connection
self.retry_limit = retry_limit
self.env_name = env_name
self.consume_all = consume_all
self.handler = event_handler
self.event_queues = event_queues
self.failing_messages = defaultdict(int)
self.inv = InventoryMgr()
self.inv.set_collections(inventory_collection)
if self.inv.is_feature_supported(self.env_name, EnvironmentFeatures.MONITORING):
self.inv.monitoring_setup_manager = \
MonitoringSetupManager(self.env_name)
def configure_environment(self):
self.env = ENV_CONFIG
self.inventory_collection = COLLECTION_CONFIG
# mock the mongo access
MongoAccess.mongo_connect = MagicMock()
MongoAccess.db = MagicMock()
# mock log
FullLogger.info = MagicMock()
self.conf = Configuration()
self.conf.use_env = MagicMock()
self.conf.environment = CONFIGURATIONS
self.conf.configuration = CONFIGURATIONS["configuration"]
self.inv = InventoryMgr()
self.inv.clear = MagicMock()
self.inv.set_collections(self.inventory_collection)
MonitoringSetupManager.server_setup = MagicMock()
DbAccess.get_neutron_db_name = MagicMock()
DbAccess.get_neutron_db_name.return_value = "neutron"
def emit(self, record):
# Try to invoke InventoryMgr for logging
if not self.inv:
try:
self.inv = InventoryMgr()
except:
return
# make sure we do not try to log to DB when DB is not ready
if not (self.inv.is_db_ready()
and 'messages' in self.inv.collections):
return
# make ID from current timestamp
now = datetime.datetime.utcnow()
d = now - datetime.datetime(1970, 1, 1)
timestamp_id = '{}.{}.{}'.format(d.days, d.seconds, d.microseconds)
source = self.SOURCE_SYSTEM
message = Message(msg_id=timestamp_id, env=self.env, source=source,
msg=Logger.formatter.format(record), ts=now,
level=record.levelname)
if self.origin:
message.extra['origin_id'] = (
str(self.origin.origin_id)
if self.origin.origin_id
else None
)
message.extra['origin_type'] = (
self.origin.origin_type.value
if self.origin.origin_type
else None
)
for extra_field in self.origin.extra:
message.extra[extra_field] = getattr(self.origin, extra_field)
self.inv.collections['messages'].insert_one(message.get())
def __init__(self):
super(ApiFetchNetworks, self).__init__()
self.inv = InventoryMgr()
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$")
self.port_line_header_prefix = " " * 8 + "Port "
def listen(args: dict = None):
args = setup_args(args, DefaultListener.DEFAULTS, get_args)
if 'process_vars' not in args:
args['process_vars'] = {}
env_name = args["env"]
inventory_collection = args["inventory"]
MongoAccess.set_config_file(args["mongo_config"])
inv = InventoryMgr()
inv.set_collections(inventory_collection)
conf = Configuration(args["environments_collection"])
conf.use_env(env_name)
event_handler = EventHandler(env_name, inventory_collection)
event_queues = []
env_config = conf.get_env_config()
common_metadata_file = os.path.join(env_config.get('app_path', '/etc/calipso'),
'config',
DefaultListener.COMMON_METADATA_FILE)
# import common metadata
import_metadata(event_handler, event_queues, common_metadata_file)
# import custom metadata if supplied
if args["metadata_file"]:
import_metadata(event_handler, event_queues, args["metadata_file"])
logger = FullLogger()
logger.set_loglevel(args["loglevel"])
amqp_config = conf.get("AMQP")
connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \
.format(user=amqp_config["user"],
pwd=amqp_config["pwd"],
host=amqp_config["host"],
port=amqp_config["port"])
with Connection(connect_url) as conn:
try:
print(conn)
conn.connect()
args['process_vars']['operational'] = OperationalStatus.RUNNING
terminator = SignalHandler()
worker = \
DefaultListener(connection=conn,
event_handler=event_handler,
event_queues=event_queues,
retry_limit=args["retry_limit"],
consume_all=args["consume_all"],
inventory_collection=inventory_collection,
env_name=env_name)
worker.run()
if terminator.terminated:
args.get('process_vars', {})['operational'] = \
OperationalStatus.STOPPED
except KeyboardInterrupt:
print('Stopped')
args['process_vars']['operational'] = OperationalStatus.STOPPED
except Exception as e:
logger.log.exception(e)
args['process_vars']['operational'] = OperationalStatus.ERROR
finally:
# This should enable safe saving of shared variables
time.sleep(0.1)
def __init__(self):
super(CliFetchHostVservice, self).__init__()
# match only DHCP agent and router (L3 agent)
self.type_re = re.compile("^q(dhcp|router)-")
self.inv = InventoryMgr()
self.agents_list = NetworkAgentsList()
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.env_config = self.config.get_env_config()
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.port_re = re.compile("^\s*port (\d+): ([^(]+)( \(internal\))?$")
def run(self, args: dict = None):
args = setup_args(args, self.DEFAULTS, self.get_args)
# After this setup we assume args dictionary has all keys
# defined in self.DEFAULTS
self.log.set_loglevel(args['loglevel'])
try:
MongoAccess.set_config_file(args['mongo_config'])
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args['loglevel'])
self.inv.set_collections(args['inventory'])
self.conf = Configuration()
except FileNotFoundError as e:
return False, 'Mongo configuration file not found: {}'\
.format(str(e))
scan_plan = self.get_scan_plan(args)
if scan_plan.clear or scan_plan.clear_all:
self.inv.clear(scan_plan)
self.conf.log.set_loglevel(scan_plan.loglevel)
env_name = scan_plan.env
self.conf.use_env(env_name)
# generate ScanObject Class and instance.
scanner = Scanner()
scanner.log.set_loglevel(args['loglevel'])
scanner.set_env(env_name)
scanner.found_errors[env_name] = False
# decide what scanning operations to do
inventory_only = scan_plan.inventory_only
links_only = scan_plan.links_only
cliques_only = scan_plan.cliques_only
monitoring_setup_only = scan_plan.monitoring_setup_only
run_all = False if inventory_only or links_only or cliques_only \
or monitoring_setup_only else True
# setup monitoring server
monitoring = \
self.inv.is_feature_supported(env_name,
EnvironmentFeatures.MONITORING)
if monitoring:
self.inv.monitoring_setup_manager = \
MonitoringSetupManager(env_name)
self.inv.monitoring_setup_manager.server_setup()
# do the actual scanning
try:
if inventory_only or run_all:
scanner.run_scan(scan_plan.scanner_type, scan_plan.obj,
scan_plan.id_field, scan_plan.child_id,
scan_plan.child_type)
if links_only or run_all:
scanner.scan_links()
if cliques_only or run_all:
scanner.scan_cliques()
if monitoring:
if monitoring_setup_only:
self.inv.monitoring_setup_manager.simulate_track_changes()
if not (inventory_only or links_only or cliques_only):
scanner.deploy_monitoring_setup()
except ScanError as e:
return False, "scan error: " + str(e)
SshConnection.disconnect_all()
status = 'ok' if not scanner.found_errors.get(env_name, False) \
else 'errors detected'
if status == 'ok' and scan_plan.object_type == "environment":
self.mark_env_scanned(scan_plan.env)
self.log.info('Scan completed, status: {}'.format(status))
return True, status
def __init__(self):
super().__init__()
self.log = FullLogger()
self.inv = InventoryMgr()
def __init__(self):
super().__init__()
self.inv = InventoryMgr()
self.log = FullLogger()
self.tokens_coll = self.inv.collections['api_tokens']
self.ldap_access = LDAPAccess()
def __init__(self):
self.args = self.get_args()
MongoAccess.set_config_file(self.args.mongo_config)
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
self.input_text = None
