class ScanController(Fetcher):
DEFAULTS = {
"env": "",
"mongo_config": "",
"type": "",
"inventory": "inventory",
"scan_self": False,
"parent_id": "",
"parent_type": "",
"id_field": "id",
"loglevel": "INFO",
"inventory_only": False,
"links_only": False,
"cliques_only": False,
"monitoring_setup_only": False,
"clear": False,
"clear_all": False
}
def __init__(self):
super().__init__()
self.conf = None
self.inv = None
def get_args(self):
# try to read scan plan from command line parameters
parser = argparse.ArgumentParser()
parser.add_argument("-m",
"--mongo_config",
nargs="?",
type=str,
default=self.DEFAULTS["mongo_config"],
help="name of config file " +
"with MongoDB server access details")
parser.add_argument("-e",
"--env",
nargs="?",
type=str,
default=self.DEFAULTS["env"],
help="name of environment to scan \n"
"(default: " + self.DEFAULTS["env"] + ")")
parser.add_argument("-t",
"--type",
nargs="?",
type=str,
default=self.DEFAULTS["type"],
help="type of object to scan \n"
"(default: environment)")
parser.add_argument("-y",
"--inventory",
nargs="?",
type=str,
default=self.DEFAULTS["inventory"],
help="name of inventory collection \n"
"(default: 'inventory')")
parser.add_argument("-s",
"--scan_self",
action="store_true",
help="scan changes to a specific object \n"
"(default: False)")
parser.add_argument("-i",
"--id",
nargs="?",
type=str,
default=self.DEFAULTS["env"],
help="ID of object to scan (when scan_self=true)")
parser.add_argument("-p",
"--parent_id",
nargs="?",
type=str,
default=self.DEFAULTS["parent_id"],
help="ID of parent object (when scan_self=true)")
parser.add_argument("-a",
"--parent_type",
nargs="?",
type=str,
default=self.DEFAULTS["parent_type"],
help="type of parent object (when scan_self=true)")
parser.add_argument("-f",
"--id_field",
nargs="?",
type=str,
default=self.DEFAULTS["id_field"],
help="name of ID field (when scan_self=true) \n"
"(default: 'id', use 'name' for projects)")
parser.add_argument("-l",
"--loglevel",
nargs="?",
type=str,
default=self.DEFAULTS["loglevel"],
help="logging level \n(default: '{}')".format(
self.DEFAULTS["loglevel"]))
parser.add_argument("--clear",
action="store_true",
help="clear all data related to "
"the specified environment prior to scanning\n"
"(default: False)")
parser.add_argument("--clear_all",
action="store_true",
help="clear all data prior to scanning\n"
"(default: False)")
parser.add_argument("--monitoring_setup_only",
action="store_true",
help="do only monitoring setup deployment \n"
"(default: False)")
# At most one of these arguments may be present
scan_only_group = parser.add_mutually_exclusive_group()
scan_only_group.add_argument("--inventory_only",
action="store_true",
help="do only scan to inventory\n" +
"(default: False)")
scan_only_group.add_argument("--links_only",
action="store_true",
help="do only links creation \n" +
"(default: False)")
scan_only_group.add_argument("--cliques_only",
action="store_true",
help="do only cliques creation \n" +
"(default: False)")
return parser.parse_args()
def get_scan_plan(self, args):
# PyCharm type checker can't reliably check types of document
# noinspection PyTypeChecker
return self.prepare_scan_plan(ScanPlan(args))
def prepare_scan_plan(self, plan):
# Find out object type if not specified in arguments
if not plan.object_type:
if not plan.object_id:
plan.object_type = "environment"
else:
# If we scan a specific object, it has to exist in db
scanned_object = self.inv.get_by_id(plan.env, plan.object_id)
if not scanned_object:
exc_msg = "No object found with specified id: '{}'" \
.format(plan.object_id)
raise ScanArgumentsError(exc_msg)
plan.object_type = scanned_object["type"]
plan.parent_id = scanned_object["parent_id"]
plan.type_to_scan = scanned_object["parent_type"]
class_module = plan.object_type
if not plan.scan_self:
plan.scan_self = plan.object_type != "environment"
plan.object_type = plan.object_type.title().replace("_", "")
if not plan.scan_self:
plan.child_type = None
else:
plan.child_id = plan.object_id
plan.object_id = plan.parent_id
if plan.type_to_scan.endswith("_folder"):
class_module = plan.child_type + "s_root"
else:
class_module = plan.type_to_scan
plan.object_type = class_module.title().replace("_", "")
if class_module == "environment":
plan.obj = {"id": plan.env}
else:
# fetch object from inventory
obj = self.inv.get_by_id(plan.env, plan.object_id)
if not obj:
raise ValueError("No match for object ID: {}".format(
plan.object_id))
plan.obj = obj
plan.scanner_type = "Scan" + plan.object_type
return plan
def run(self, args: dict = None):
args = setup_args(args, self.DEFAULTS, self.get_args)
# After this setup we assume args dictionary has all keys
# defined in self.DEFAULTS
self.log.set_loglevel(args['loglevel'])
try:
MongoAccess.set_config_file(args['mongo_config'])
self.inv = InventoryMgr()
self.inv.log.set_loglevel(args['loglevel'])
self.inv.set_collections(args['inventory'])
self.conf = Configuration()
except FileNotFoundError as e:
return False, 'Mongo configuration file not found: {}'\
.format(str(e))
scan_plan = self.get_scan_plan(args)
if scan_plan.clear or scan_plan.clear_all:
self.inv.clear(scan_plan)
self.conf.log.set_loglevel(scan_plan.loglevel)
env_name = scan_plan.env
self.conf.use_env(env_name)
# generate ScanObject Class and instance.
scanner = Scanner()
scanner.log.set_loglevel(args['loglevel'])
scanner.set_env(env_name)
scanner.found_errors[env_name] = False
# decide what scanning operations to do
inventory_only = scan_plan.inventory_only
links_only = scan_plan.links_only
cliques_only = scan_plan.cliques_only
monitoring_setup_only = scan_plan.monitoring_setup_only
run_all = False if inventory_only or links_only or cliques_only \
or monitoring_setup_only else True
# setup monitoring server
monitoring = \
self.inv.is_feature_supported(env_name,
EnvironmentFeatures.MONITORING)
if monitoring:
self.inv.monitoring_setup_manager = \
MonitoringSetupManager(env_name)
self.inv.monitoring_setup_manager.server_setup()
# do the actual scanning
try:
if inventory_only or run_all:
scanner.run_scan(scan_plan.scanner_type, scan_plan.obj,
scan_plan.id_field, scan_plan.child_id,
scan_plan.child_type)
if links_only or run_all:
scanner.scan_links()
if cliques_only or run_all:
scanner.scan_cliques()
if monitoring:
if monitoring_setup_only:
self.inv.monitoring_setup_manager.simulate_track_changes()
if not (inventory_only or links_only or cliques_only):
scanner.deploy_monitoring_setup()
except ScanError as e:
return False, "scan error: " + str(e)
SshConnection.disconnect_all()
status = 'ok' if not scanner.found_errors.get(env_name, False) \
else 'errors detected'
if status == 'ok' and scan_plan.object_type == "environment":
self.mark_env_scanned(scan_plan.env)
self.log.info('Scan completed, status: {}'.format(status))
return True, status
def mark_env_scanned(self, env):
environments_collection = self.inv.collection['environments_config']
environments_collection \
.update_one(filter={'name': env},
update={'$set': {'scanned': True}})
class DefaultListener(ListenerBase, ConsumerMixin):
SOURCE_SYSTEM = "OpenStack"
COMMON_METADATA_FILE = "events.json"
LOG_FILENAME = "default_listener.log"
LOG_LEVEL = Logger.INFO
DEFAULTS = {
"env": "Mirantis-Liberty",
"mongo_config": "",
"metadata_file": "",
"inventory": "inventory",
"loglevel": "INFO",
"environments_collection": "environments_config",
"retry_limit": 10,
"consume_all": False
}
def __init__(self, connection: Connection,
event_handler: EventHandler,
event_queues: List,
env_name: str = DEFAULTS["env"],
inventory_collection: str = DEFAULTS["inventory"],
retry_limit: int = DEFAULTS["retry_limit"],
consume_all: bool = DEFAULTS["consume_all"]):
super().__init__()
self.connection = connection
self.retry_limit = retry_limit
self.env_name = env_name
self.consume_all = consume_all
self.handler = event_handler
self.event_queues = event_queues
self.failing_messages = defaultdict(int)
self.inv = InventoryMgr()
self.inv.set_collections(inventory_collection)
if self.inv.is_feature_supported(self.env_name, EnvironmentFeatures.MONITORING):
self.inv.monitoring_setup_manager = \
MonitoringSetupManager(self.env_name)
def get_consumers(self, consumer, channel):
return [consumer(queues=self.event_queues,
accept=['json'],
callbacks=[self.process_task])]
# Determines if message should be processed by a handler
# and extracts message body if yes.
@staticmethod
def _extract_event_data(body):
if "event_type" in body:
return True, body
elif "event_type" in body.get("oslo.message", ""):
return True, json.loads(body["oslo.message"])
else:
return False, None
def process_task(self, body, message):
received_timestamp = datetime.datetime.now()
processable, event_data = self._extract_event_data(body)
# If env listener can't process the message
# or it's not intended for env listener to handle,
# leave the message in the queue unless "consume_all" flag is set
if processable and event_data["event_type"] in self.handler.handlers:
event_result = self.handle_event(event_data["event_type"],
event_data)
finished_timestamp = datetime.datetime.now()
self.save_message(message_body=event_data,
result=event_result,
started=received_timestamp,
finished=finished_timestamp)
# Check whether the event was fully handled
# and, if not, whether it should be retried later
if event_result.result:
message.ack()
elif event_result.retry:
if 'message_id' not in event_data:
message.reject()
else:
# Track message retry count
message_id = event_data['message_id']
self.failing_messages[message_id] += 1
# Retry handling the message
if self.failing_messages[message_id] <= self.retry_limit:
self.inv.log.info("Retrying handling message " +
"with id '{}'".format(message_id))
message.requeue()
# Discard the message if it's not accepted
# after specified number of trials
else:
self.inv.log.warn("Discarding message with id '{}' ".
format(message_id) +
"as it's exceeded the retry limit")
message.reject()
del self.failing_messages[message_id]
else:
message.reject()
elif self.consume_all:
message.reject()
# This method passes the event to its handler.
# Returns a (result, retry) tuple:
# 'Result' flag is True if handler has finished successfully,
# False otherwise
# 'Retry' flag specifies if the error is recoverable or not
# 'Retry' flag is checked only is 'result' is False
def handle_event(self, event_type: str, notification: dict) -> EventResult:
self.log.error("Got notification.\nEvent_type: {}\nNotification:\n{}".
format(event_type, notification))
try:
result = self.handler.handle(event_name=event_type,
notification=notification)
return result if result else EventResult(result=False, retry=False)
except Exception as e:
self.inv.log.exception(e)
return EventResult(result=False, retry=False)
def save_message(self, message_body: dict, result: EventResult,
started: datetime, finished: datetime):
try:
message = Message(
msg_id=message_body.get('message_id'),
env=self.env_name,
source=self.SOURCE_SYSTEM,
object_id=result.related_object,
display_context=result.display_context,
level=message_body.get('priority'),
msg=message_body,
ts=message_body.get('timestamp'),
received_ts=started,
finished_ts=finished
)
self.inv.collections['messages'].insert_one(message.get())
return True
except Exception as e:
self.inv.log.error("Failed to save message")
self.inv.log.exception(e)
return False
@staticmethod
def listen(args: dict = None):
args = setup_args(args, DefaultListener.DEFAULTS, get_args)
if 'process_vars' not in args:
args['process_vars'] = {}
env_name = args["env"]
inventory_collection = args["inventory"]
MongoAccess.set_config_file(args["mongo_config"])
inv = InventoryMgr()
inv.set_collections(inventory_collection)
conf = Configuration(args["environments_collection"])
conf.use_env(env_name)
event_handler = EventHandler(env_name, inventory_collection)
event_queues = []
env_config = conf.get_env_config()
common_metadata_file = os.path.join(env_config.get('app_path', '/etc/calipso'),
'config',
DefaultListener.COMMON_METADATA_FILE)
# import common metadata
import_metadata(event_handler, event_queues, common_metadata_file)
# import custom metadata if supplied
if args["metadata_file"]:
import_metadata(event_handler, event_queues, args["metadata_file"])
logger = FullLogger()
logger.set_loglevel(args["loglevel"])
amqp_config = conf.get("AMQP")
connect_url = 'amqp://{user}:{pwd}@{host}:{port}//' \
.format(user=amqp_config["user"],
pwd=amqp_config["pwd"],
host=amqp_config["host"],
port=amqp_config["port"])
with Connection(connect_url) as conn:
try:
print(conn)
conn.connect()
args['process_vars']['operational'] = OperationalStatus.RUNNING
terminator = SignalHandler()
worker = \
DefaultListener(connection=conn,
event_handler=event_handler,
event_queues=event_queues,
retry_limit=args["retry_limit"],
consume_all=args["consume_all"],
inventory_collection=inventory_collection,
env_name=env_name)
worker.run()
if terminator.terminated:
args.get('process_vars', {})['operational'] = \
OperationalStatus.STOPPED
except KeyboardInterrupt:
print('Stopped')
args['process_vars']['operational'] = OperationalStatus.STOPPED
except Exception as e:
logger.log.exception(e)
args['process_vars']['operational'] = OperationalStatus.ERROR
finally:
# This should enable safe saving of shared variables
time.sleep(0.1)
class ScanManager(Manager):
DEFAULTS = {
"mongo_config": "",
"scans": "scans",
"scheduled_scans": "scheduled_scans",
"environments": "environments_config",
"interval": 1,
"loglevel": "INFO"
}
def __init__(self):
self.args = self.get_args()
super().__init__(log_directory=self.args.log_directory,
mongo_config_file=self.args.mongo_config)
self.db_client = None
self.environments_collection = None
self.scans_collection = None
self.scheduled_scans_collection = None
@staticmethod
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument("-m",
"--mongo_config",
nargs="?",
type=str,
default=ScanManager.DEFAULTS["mongo_config"],
help="Name of config file " +
"with MongoDB server access details")
parser.add_argument("-c",
"--scans_collection",
nargs="?",
type=str,
default=ScanManager.DEFAULTS["scans"],
help="Scans collection to read from")
parser.add_argument("-s",
"--scheduled_scans_collection",
nargs="?",
type=str,
default=ScanManager.DEFAULTS["scheduled_scans"],
help="Scans collection to read from")
parser.add_argument("-e",
"--environments_collection",
nargs="?",
type=str,
default=ScanManager.DEFAULTS["environments"],
help="Environments collection to update "
"after scans")
parser.add_argument("-i",
"--interval",
nargs="?",
type=float,
default=ScanManager.DEFAULTS["interval"],
help="Interval between collection polls"
"(must be more than {} seconds)".format(
ScanManager.MIN_INTERVAL))
parser.add_argument("-l",
"--loglevel",
nargs="?",
type=str,
default=ScanManager.DEFAULTS["loglevel"],
help="Logging level \n(default: '{}')".format(
ScanManager.DEFAULTS["loglevel"]))
parser.add_argument(
"-d",
"--log_directory",
nargs="?",
type=str,
default=FileLogger.LOG_DIRECTORY,
help="File logger directory \n(default: '{}')".format(
FileLogger.LOG_DIRECTORY))
args = parser.parse_args()
return args
def configure(self):
self.db_client = MongoAccess()
self.inv = InventoryMgr()
self.inv.set_collections()
self.scans_collection = self.db_client.db[self.args.scans_collection]
self.scheduled_scans_collection = \
self.db_client.db[self.args.scheduled_scans_collection]
self.environments_collection = \
self.db_client.db[self.args.environments_collection]
self._update_document = \
partial(MongoAccess.update_document, self.scans_collection)
self.interval = max(self.MIN_INTERVAL, self.args.interval)
self.log.set_loglevel(self.args.loglevel)
self.log.info("Started ScanManager with following configuration:\n"
"Mongo config file path: {0.args.mongo_config}\n"
"Scans collection: {0.scans_collection.name}\n"
"Environments collection: "
"{0.environments_collection.name}\n"
"Polling interval: {0.interval} second(s)".format(self))
def _build_scan_args(self, scan_request: dict):
args = {
'mongo_config': self.args.mongo_config,
'scheduled': True if scan_request.get('interval') else False
}
def set_arg(name_from: str, name_to: str = None):
if name_to is None:
name_to = name_from
val = scan_request.get(name_from)
if val:
args[name_to] = val
set_arg("_id")
set_arg("object_id", "id")
set_arg("log_level", "loglevel")
set_arg("environment", "env")
set_arg("scan_only_inventory", "inventory_only")
set_arg("scan_only_links", "links_only")
set_arg("scan_only_cliques", "cliques_only")
set_arg("inventory")
set_arg("clear")
set_arg("clear_all")
return args
def _finalize_scan(self, scan_request: dict, status: ScanStatus,
scanned: bool):
scan_request['status'] = status.value
self._update_document(scan_request)
# If no object id is present, it's a full env scan.
# We need to update environments collection
# to reflect the scan results.
if not scan_request.get('id'):
self.environments_collection\
.update_one(filter={'name': scan_request.get('environment')},
update={'$set': {'scanned': scanned}})
def _fail_scan(self, scan_request: dict):
self._finalize_scan(scan_request, ScanStatus.FAILED, False)
def _complete_scan(self, scan_request: dict, result_message: str):
status = ScanStatus.COMPLETED if result_message == 'ok' \
else ScanStatus.COMPLETED_WITH_ERRORS
self._finalize_scan(scan_request, status, True)
# PyCharm type checker can't reliably check types of document
# noinspection PyTypeChecker
def _clean_up(self):
# Find and fail all running scans
running_scans = list(
self.scans_collection.find(
filter={'status': ScanStatus.RUNNING.value}))
self.scans_collection \
.update_many(filter={'_id': {'$in': [scan['_id']
for scan
in running_scans]}},
update={'$set': {'status': ScanStatus.FAILED.value}})
# Find all environments connected to failed full env scans
env_scans = [
scan['environment'] for scan in running_scans
if not scan.get('object_id') and scan.get('environment')
]
# Set 'scanned' flag in those envs to false
if env_scans:
self.environments_collection\
.update_many(filter={'name': {'$in': env_scans}},
update={'$set': {'scanned': False}})
def _submit_scan_request_for_schedule(self, scheduled_scan, interval, ts):
scans = self.scans_collection
new_scan = {
'status': 'submitted',
'log_level': scheduled_scan['log_level'],
'clear': scheduled_scan['clear'],
'scan_only_inventory': scheduled_scan['scan_only_inventory'],
'scan_only_links': scheduled_scan['scan_only_links'],
'scan_only_cliques': scheduled_scan['scan_only_cliques'],
'submit_timestamp': ts,
'interval': interval,
'environment': scheduled_scan['environment'],
'inventory': 'inventory'
}
scans.insert_one(new_scan)
def _set_scheduled_requests_next_run(self, scheduled_scan, interval, ts):
scheduled_scan['scheduled_timestamp'] = ts + self.INTERVALS[interval]
doc_id = scheduled_scan.pop('_id')
self.scheduled_scans_collection.update({'_id': doc_id}, scheduled_scan)
def _prepare_scheduled_requests_for_interval(self, interval):
now = datetime.datetime.utcnow()
# first, submit a scan request where the scheduled time has come
condition = {
'$and': [{
'freq': interval
}, {
'scheduled_timestamp': {
'$lte': now
}
}]
}
matches = self.scheduled_scans_collection.find(condition) \
.sort('scheduled_timestamp', pymongo.ASCENDING)
for match in matches:
self._submit_scan_request_for_schedule(match, interval, now)
self._set_scheduled_requests_next_run(match, interval, now)
# now set scheduled time where it was not set yet (new scheduled scans)
condition = {
'$and': [{
'freq': interval
}, {
'scheduled_timestamp': {
'$exists': False
}
}]
}
matches = self.scheduled_scans_collection.find(condition)
for match in matches:
self._set_scheduled_requests_next_run(match, interval, now)
def _prepare_scheduled_requests(self):
# see if any scheduled request is waiting to be submitted
for interval in self.INTERVALS.keys():
self._prepare_scheduled_requests_for_interval(interval)
def handle_scans(self):
self._prepare_scheduled_requests()
# Find a pending request that is waiting the longest time
results = self.scans_collection \
.find({'status': ScanStatus.PENDING.value,
'submit_timestamp': {'$ne': None}}) \
.sort("submit_timestamp", pymongo.ASCENDING) \
.limit(1)
# If no scans are pending, sleep for some time
if results.count() == 0:
time.sleep(self.interval)
else:
scan_request = results[0]
env = scan_request.get('environment')
scan_feature = EnvironmentFeatures.SCANNING
if not self.inv.is_feature_supported(env, scan_feature):
self.log.error("Scanning is not supported for env '{}'".format(
scan_request.get('environment')))
self._fail_scan(scan_request)
return
scan_request['start_timestamp'] = datetime.datetime.utcnow()
scan_request['status'] = ScanStatus.RUNNING.value
self._update_document(scan_request)
# Prepare scan arguments and run the scan with them
try:
scan_args = self._build_scan_args(scan_request)
self.log.info("Starting scan for '{}' environment".format(
scan_args.get('env')))
self.log.debug("Scan arguments: {}".format(scan_args))
result, message = ScanController().run(scan_args)
except ScanArgumentsError as e:
self.log.error("Scan request '{id}' "
"has invalid arguments. "
"Errors:\n{errors}".format(
id=scan_request['_id'], errors=e))
self._fail_scan(scan_request)
except Exception as e:
self.log.exception(e)
self.log.error("Scan request '{}' has failed.".format(
scan_request['_id']))
self._fail_scan(scan_request)
else:
# Check is scan returned success
if not result:
self.log.error(message)
self.log.error("Scan request '{}' has failed.".format(
scan_request['_id']))
self._fail_scan(scan_request)
return
# update the status and timestamps.
self.log.info("Request '{}' has been scanned. ({})".format(
scan_request['_id'], message))
end_time = datetime.datetime.utcnow()
scan_request['end_timestamp'] = end_time
self._complete_scan(scan_request, message)
def do_action(self):
self._clean_up()
try:
while True:
self.handle_scans()
finally:
self._clean_up()
class EventManager(Manager):
# After EventManager receives a SIGTERM,
# it will try to terminate all listeners.
# After this delay, a SIGKILL will be sent
# to each listener that is still alive.
SIGKILL_DELAY = 5 # in seconds
DEFAULTS = {
"mongo_config": "",
"collection": "environments_config",
"inventory": "inventory",
"interval": 5,
"loglevel": "INFO"
}
LISTENERS = {
'Mirantis': {
'6.0': DefaultListener,
'7.0': DefaultListener,
'8.0': DefaultListener,
'9.0': DefaultListener
},
'RDO': {
'Mitaka': DefaultListener,
'Liberty': DefaultListener,
},
'Apex': {
'Euphrates': DefaultListener,
},
}
def __init__(self):
self.args = self.get_args()
super().__init__(log_directory=self.args.log_directory,
mongo_config_file=self.args.mongo_config)
self.db_client = None
self.interval = None
self.processes = []
@staticmethod
def get_args():
parser = argparse.ArgumentParser()
parser.add_argument(
"-m",
"--mongo_config",
nargs="?",
type=str,
default=EventManager.DEFAULTS["mongo_config"],
help="Name of config file with MongoDB server access details")
parser.add_argument("-c",
"--collection",
nargs="?",
type=str,
default=EventManager.DEFAULTS["collection"],
help="Environments collection to read from "
"(default: '{}')".format(
EventManager.DEFAULTS["collection"]))
parser.add_argument("-y",
"--inventory",
nargs="?",
type=str,
default=EventManager.DEFAULTS["inventory"],
help="name of inventory collection "
"(default: '{}')".format(
EventManager.DEFAULTS["inventory"]))
parser.add_argument(
"-i",
"--interval",
nargs="?",
type=float,
default=EventManager.DEFAULTS["interval"],
help="Interval between collection polls "
"(must be more than {} seconds. Default: {})".format(
EventManager.MIN_INTERVAL, EventManager.DEFAULTS["interval"]))
parser.add_argument("-l",
"--loglevel",
nargs="?",
type=str,
default=EventManager.DEFAULTS["loglevel"],
help="Logging level \n(default: '{}')".format(
EventManager.DEFAULTS["loglevel"]))
parser.add_argument(
"-d",
"--log_directory",
nargs="?",
type=str,
default=FileLogger.LOG_DIRECTORY,
help="File logger directory \n(default: '{}')".format(
FileLogger.LOG_DIRECTORY))
args = parser.parse_args()
return args
def configure(self):
self.db_client = MongoAccess()
self.inv = InventoryMgr()
self.inv.set_collections(self.args.inventory)
self.collection = self.db_client.db[self.args.collection]
self.interval = max(self.MIN_INTERVAL, self.args.interval)
self.log.set_loglevel(self.args.loglevel)
self.log.info("Started EventManager with following configuration:\n"
"Mongo config file path: {0}\n"
"Collection: {1}\n"
"Polling interval: {2} second(s)".format(
self.args.mongo_config, self.collection.name,
self.interval))
def get_listener(self, env: str):
env_config = self.inv.get_env_config(env)
return (self.LISTENERS.get(env_config.get('distribution'), {}).get(
env_config.get('distribution_version'), DefaultListener))
def listen_to_events(self, listener: ListenerBase, env_name: str,
process_vars: dict):
listener.listen({
'env': env_name,
'mongo_config': self.args.mongo_config,
'inventory': self.args.inventory,
'loglevel': self.args.loglevel,
'environments_collection': self.args.collection,
'process_vars': process_vars
})
def _get_alive_processes(self):
return [p for p in self.processes if p['process'].is_alive()]
# Get all processes that should be terminated
def _get_stuck_processes(self, stopped_processes: list):
return [
p for p in self._get_alive_processes()
if p.get("name") in map(lambda p: p.get("name"), stopped_processes)
]
# Give processes time to finish and kill them if they are stuck
def _kill_stuck_processes(self, process_list: list):
if self._get_stuck_processes(process_list):
time.sleep(self.SIGKILL_DELAY)
for process in self._get_stuck_processes(process_list):
self.log.info("Killing event listener '{0}'".format(
process.get("name")))
os.kill(process.get("process").pid, signal.SIGKILL)
def _get_operational(self, process: dict) -> OperationalStatus:
try:
return process.get("vars", {})\
.get("operational")
except:
self.log.error("Event listener '{0}' is unreachable".format(
process.get("name")))
return OperationalStatus.STOPPED
def _update_operational_status(self, status: OperationalStatus):
self.collection.update_many(
{
"name": {
"$in": [
process.get("name") for process in self.processes
if self._get_operational(process) == status
]
}
}, {"$set": {
"operational": status.value
}})
def update_operational_statuses(self):
self._update_operational_status(OperationalStatus.RUNNING)
self._update_operational_status(OperationalStatus.ERROR)
self._update_operational_status(OperationalStatus.STOPPED)
def cleanup_processes(self):
# Query for envs that are no longer eligible for listening
# (scanned == false and/or listen == false)
dropped_envs = [
env['name'] for env in self.collection.find(
filter={'$or': [{
'scanned': False
}, {
'listen': False
}]},
projection=['name'])
]
live_processes = []
stopped_processes = []
# Drop already terminated processes
# and for all others perform filtering
for process in self._get_alive_processes():
# If env no longer qualifies for listening,
# stop the listener.
# Otherwise, keep the process
if process['name'] in dropped_envs:
self.log.info("Stopping event listener '{0}'".format(
process.get("name")))
process['process'].terminate()
stopped_processes.append(process)
else:
live_processes.append(process)
self._kill_stuck_processes(stopped_processes)
# Update all 'operational' statuses
# for processes stopped on the previous step
self.collection.update_many(
{
"name": {
"$in":
[process.get("name") for process in stopped_processes]
}
}, {"$set": {
"operational": OperationalStatus.STOPPED.value
}})
# Keep the living processes
self.processes = live_processes
def do_action(self):
try:
while True:
# Update "operational" field in db before removing dead processes
# so that we keep last statuses of env listeners before they were terminated
self.update_operational_statuses()
# Perform a cleanup that filters out all processes
# that are no longer eligible for listening
self.cleanup_processes()
envs = self.collection.find({'scanned': True, 'listen': True})
# Iterate over environments that don't have an event listener attached
for env in filter(
lambda e: e['name'] not in map(
lambda process: process["name"], self.processes),
envs):
env_name = env['name']
if not self.inv.is_feature_supported(
env_name, EnvironmentFeatures.LISTENING):
self.log.error(
"Listening is not supported for env '{}'".format(
env_name))
self.collection.update({"name": env_name}, {
"$set": {
"operational": OperationalStatus.ERROR.value
}
})
continue
listener = self.get_listener(env_name)
if not listener:
self.log.error(
"No listener is defined for env '{}'".format(
env_name))
self.collection.update({"name": env_name}, {
"$set": {
"operational": OperationalStatus.ERROR.value
}
})
continue
# A dict that is shared between event manager and newly created env listener
process_vars = SharedManager().dict()
p = Process(target=self.listen_to_events,
args=(
listener,
env_name,
process_vars,
),
name=env_name)
self.processes.append({
"process": p,
"name": env_name,
"vars": process_vars
})
self.log.info(
"Starting event listener '{0}'".format(env_name))
p.start()
# Make sure statuses are up-to-date before event manager goes to sleep
self.update_operational_statuses()
time.sleep(self.interval)
finally:
# Fetch operational statuses before terminating listeners.
# Shared variables won't be available after termination.
stopping_processes = [
process.get("name") for process in self.processes
if self._get_operational(process) != OperationalStatus.ERROR
]
self._update_operational_status(OperationalStatus.ERROR)
# Gracefully stop processes
for process in self._get_alive_processes():
self.log.info("Stopping event listener '{0}'".format(
process.get("name")))
process.get("process").terminate()
# Kill all remaining processes
self._kill_stuck_processes(self.processes)
# Updating operational statuses for stopped processes
self.collection.update_many({"name": {
"$in": stopping_processes
}}, {"$set": {
"operational": OperationalStatus.STOPPED.value
}})
