def _read_db_graph(self):
db = storage.get_connection_from_config(self._conf)
graph_snapshot = db.graph_snapshots.query()
NXGraph.read_gpickle(graph_snapshot.graph_snapshot, self._entity_graph)
GraphPersistency.do_replay_events(db, self._entity_graph,
graph_snapshot.event_id)
self._entity_graph.ready = True
def __init__(self, conf, workers):
self.conf = conf
self.graph = get_graph_driver(conf)('Entity Graph')
self.db = db_connection = storage.get_connection_from_config(conf)
self.workers = workers
self.events_coordination = EventsCoordination(conf, self.process_event)
self.persist = GraphPersistency(conf, db_connection, self.graph)
self.driver_exec = driver_exec.DriverExec(
self.conf, self.events_coordination.handle_multiple_low_priority,
self.persist)
self.scheduler = Scheduler(conf, self.graph, self.driver_exec,
self.persist)
self.processor = Processor(conf, self.graph)
def __init__(self, conf, graph, db_connection):
self.conf = conf
self.graph = graph
self.db = db_connection
self.workers = GraphWorkersManager(conf, graph, db_connection)
self.events_coordination = EventsCoordination(
conf,
self.process_event,
conf.datasources.notification_topic_collector,
EVALUATOR_TOPIC)
self.persist = GraphPersistency(conf, db_connection, graph)
self.scheduler = Scheduler(conf, graph, self.events_coordination,
self.persist)
self.processor = Processor(conf, graph)
class VitrageGraphInit(object):
def __init__(self, conf, graph, db_connection):
self.conf = conf
self.graph = graph
self.db = db_connection
self.workers = GraphWorkersManager(conf, graph, db_connection)
self.events_coordination = EventsCoordination(
conf,
self.process_event,
conf.datasources.notification_topic_collector,
EVALUATOR_TOPIC)
self.persist = GraphPersistency(conf, db_connection, graph)
self.scheduler = Scheduler(conf, graph, self.events_coordination,
self.persist)
self.processor = Processor(conf, graph)
def run(self):
LOG.info('Init Started')
graph_snapshot = self.persist.query_recent_snapshot()
if graph_snapshot:
self._restart_from_stored_graph(graph_snapshot)
else:
self._start_from_scratch()
self.workers.run()
def _restart_from_stored_graph(self, graph_snapshot):
LOG.info('Initializing graph from database snapshot (%sKb)',
len(graph_snapshot.graph_snapshot) / 1024)
NXGraph.read_gpickle(graph_snapshot.graph_snapshot, self.graph)
self.persist.replay_events(self.graph, graph_snapshot.event_id)
self._recreate_transformers_id_cache()
LOG.info("%s vertices loaded", self.graph.num_vertices())
self.subscribe_presist_notifier()
spawn(self._start_all_workers, is_snapshot=True)
def _start_from_scratch(self):
LOG.info('Starting for the first time')
LOG.info('Clearing database active_actions')
self.db.active_actions.delete()
LOG.info('Disabling previously active alarms')
self.db.history_facade.disable_alarms_in_history()
self.subscribe_presist_notifier()
ds_rpc.get_all(
ds_rpc.create_rpc_client_instance(self.conf),
self.events_coordination,
self.conf.datasources.types,
action=DatasourceAction.INIT_SNAPSHOT,
retry_on_fault=True)
LOG.info("%s vertices loaded", self.graph.num_vertices())
self.persist.store_graph()
spawn(self._start_all_workers, is_snapshot=False)
def _start_all_workers(self, is_snapshot):
if is_snapshot:
self.workers.submit_enable_evaluations()
else:
self.workers.submit_start_evaluations()
self._add_graph_subscriptions()
self.scheduler.start_periodic_tasks()
LOG.info('Init Finished')
self.events_coordination.start()
def process_event(self, event):
if event.get('template_action'):
self.workers.submit_template_event(event)
self.workers.submit_evaluators_reload_templates()
else:
self.processor.process_event(event)
def _recreate_transformers_id_cache(self):
for v in self.graph.get_vertices():
if not v.get(VProps.VITRAGE_CACHED_ID):
LOG.warning("Missing vitrage_cached_id in the vertex. "
"Vertex is not added to the ID cache %s", str(v))
else:
TransformerBase.key_to_uuid_cache[v[VProps.VITRAGE_CACHED_ID]]\
= v.vertex_id
def _add_graph_subscriptions(self):
self.graph.subscribe(self.workers.submit_graph_update)
vitrage_notifier = GraphNotifier(self.conf)
if vitrage_notifier.enabled:
self.graph.subscribe(vitrage_notifier.notify_when_applicable)
LOG.info('Subscribed vitrage notifier to graph changes')
self.graph.subscribe(self.persist.persist_event,
finalization=True)
def subscribe_presist_notifier(self):
self.graph.subscribe(PersistNotifier(self.conf).notify_when_applicable)
class VitrageGraphInit(object):
def __init__(self, workers):
self.graph = get_graph_driver()('Entity Graph')
self.db = db_connection = storage.get_connection_from_config()
self.workers = workers
self.events_coordination = EventsCoordination(self.process_event)
self.persist = GraphPersistency(db_connection, self.graph)
self.driver_exec = driver_exec.DriverExec(
self.events_coordination.handle_multiple_low_priority,
self.persist, self.graph)
consistency = ConsistencyEnforcer(
self.graph, self.events_coordination.handle_multiple_high_priority)
self.scheduler = Scheduler(self.graph, self.driver_exec, self.persist,
consistency)
self.processor = Processor(self.graph)
def run(self):
LOG.info('Init Started')
graph_snapshot = self.persist.query_recent_snapshot()
if graph_snapshot:
t = spawn(self.workers.submit_read_db_graph)
self._restart_from_stored_graph(graph_snapshot)
t.join()
self.workers.submit_enable_evaluations()
else:
self._start_from_scratch()
self.workers.submit_read_db_graph()
self.workers.submit_start_evaluations()
self._init_finale(immediate_get_all=True if graph_snapshot else False)
def _restart_from_stored_graph(self, graph_snapshot):
LOG.info('Main process - loading graph from database snapshot (%sKb)',
len(graph_snapshot.graph_snapshot) / 1024)
NXGraph.read_gpickle(graph_snapshot.graph_snapshot, self.graph)
self.persist.replay_events(self.graph, graph_snapshot.event_id)
self._recreate_transformers_id_cache()
LOG.info("%s vertices loaded", self.graph.num_vertices())
self.subscribe_presist_notifier()
def _start_from_scratch(self):
LOG.info('Starting for the first time')
LOG.info('Clearing database active_actions')
self.db.active_actions.delete()
LOG.info('Disabling previously active alarms')
self.db.history_facade.disable_alarms_in_history()
self.subscribe_presist_notifier()
self.driver_exec.snapshot_get_all()
LOG.info("%s vertices loaded", self.graph.num_vertices())
def _init_finale(self, immediate_get_all):
self._add_graph_subscriptions()
self.scheduler.start_periodic_tasks(immediate_get_all)
LOG.info('Init Finished')
self.events_coordination.start()
def process_event(self, event):
if isinstance(event, list):
for e in event:
self.processor.process_event(e)
elif event.get('template_action'):
self.workers.submit_template_event(event)
self.workers.submit_evaluators_reload_templates()
else:
self.processor.process_event(event)
self.persist.flush_events()
def _recreate_transformers_id_cache(self):
for v in self.graph.get_vertices():
if not v.get(VProps.VITRAGE_CACHED_ID):
LOG.warning(
"Missing vitrage_cached_id in the vertex. "
"Vertex is not added to the ID cache %s", v)
else:
TransformerBase.key_to_uuid_cache[v[VProps.VITRAGE_CACHED_ID]]\
= v.vertex_id
def _add_graph_subscriptions(self):
self.graph.subscribe(self.workers.submit_graph_update)
vitrage_notifier = GraphNotifier()
if vitrage_notifier.enabled:
self.graph.subscribe(vitrage_notifier.notify_when_applicable)
LOG.info('Subscribed vitrage notifier to graph changes')
self.graph.subscribe(self.persist.persist_event, finalization=True)
def subscribe_presist_notifier(self):
self.graph.subscribe(PersistNotifier().notify_when_applicable)