def test_execute_add_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host = host_vertices[0] targets = {TFields.TARGET: host} props = { TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE', TFields.SEVERITY: 'CRITICAL', VProps.STATE: AlarmProps.ACTIVE_STATE } # Raise alarm action adds new vertex with type vitrage to the graph action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props) alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE} before_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) event_queue = queue.Queue() action_executor = ActionExecutor(event_queue) expected_alarm_id = 'ALARM:vitrage:%s:%s' % (props[TFields.ALARM_NAME], host.vertex_id) # Test Action action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) after_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) # Assertions self.assertEqual(len(before_alarms) + 1, len(after_alarms)) self.assert_is_not_empty(after_alarms) alarms = [ alarm for alarm in after_alarms if alarm.vertex_id == expected_alarm_id ] # Expected exactly one alarm with expected id self.assertEqual(1, len(alarms)) alarm = alarms[0] self.assertEqual(alarm.properties[VProps.CATEGORY], EntityCategory.ALARM) self.assertEqual(alarm.properties[VProps.TYPE], VITRAGE_TYPE) self.assertEqual(alarm.properties[VProps.SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.OPERATIONAL_SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.STATE], AlarmProps.ACTIVE_STATE)
def test_execute_add_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host = host_vertices[0] targets = {TFields.TARGET: host} props = { TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE', TFields.SEVERITY: 'CRITICAL', VProps.STATE: AlarmProps.ACTIVE_STATE } # Raise alarm action adds new vertex with type vitrage to the graph action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props) alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE} before_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) event_queue = queue.Queue() action_executor = ActionExecutor(event_queue) expected_alarm_id = 'ALARM:vitrage:%s:%s' % (props[TFields.ALARM_NAME], host.vertex_id) # Test Action action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) after_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) # Assertions self.assertEqual(len(before_alarms) + 1, len(after_alarms)) self.assert_is_not_empty(after_alarms) alarms = [alarm for alarm in after_alarms if alarm.vertex_id == expected_alarm_id] # Expected exactly one alarm with expected id self.assertEqual(1, len(alarms)) alarm = alarms[0] self.assertEqual(alarm.properties[VProps.CATEGORY], EntityCategory.ALARM) self.assertEqual(alarm.properties[VProps.TYPE], VITRAGE_TYPE) self.assertEqual(alarm.properties[VProps.SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.OPERATIONAL_SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.STATE], AlarmProps.ACTIVE_STATE)
def test_execute_add_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host = host_vertices[0] targets = {TFields.TARGET: host} props = { TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE', TFields.SEVERITY: OperationalAlarmSeverity.CRITICAL, VProps.STATE: AlarmProps.ACTIVE_STATE, VProps.RESOURCE_ID: host[VProps.ID], VProps.VITRAGE_ID: 'DUMMY_ID' } # Raise alarm action adds new vertex with type vitrage to the graph action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props) alarm_vertex_attrs = {VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE} before_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) event_queue = queue.Queue() action_executor = ActionExecutor(self.conf, event_queue) # Test Action action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) after_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) # Assertions self.assertEqual(len(before_alarms) + 1, len(after_alarms)) self.assert_is_not_empty(after_alarms) alarm = after_alarms[0] self.assertEqual(alarm.properties[VProps.VITRAGE_CATEGORY], EntityCategory.ALARM) self.assertEqual(alarm.properties[VProps.VITRAGE_TYPE], VITRAGE_DATASOURCE) self.assertEqual(alarm.properties[VProps.SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.VITRAGE_OPERATIONAL_SEVERITY], props[TFields.SEVERITY]) self.assertEqual(alarm.properties[VProps.STATE], AlarmProps.ACTIVE_STATE) self.assertEqual( alarm.properties[VProps.VITRAGE_RESOURCE_ID], action_spec.targets[TTFields.TARGET][VProps.VITRAGE_ID]), self.assertEqual(alarm.properties[VProps.VITRAGE_RESOURCE_TYPE], NOVA_HOST_DATASOURCE)
def __init__(self, e_graph, scenario_repo, actions_callback, enabled=False): self._entity_graph = e_graph self._db = storage.get_connection_from_config() self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(actions_callback) self._entity_graph.subscribe(self.process_event) self.enabled = enabled self.connected_component_cache = defaultdict(dict)
def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._entity_graph = entity_graph self._graph_algs = create_algorithm(entity_graph) self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self.enabled = enabled
def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._scenario_repo = scenario_repo self._entity_graph = entity_graph self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf)) self.enabled = enabled self.connected_component_cache = defaultdict(dict)
def test_execute_add_edge(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host_1 = host_vertices[0] nagios_event1 = TestActionExecutor._get_nagios_event( host_1.get(VProps.ID), NOVA_HOST_DATASOURCE) processor.process_event(nagios_event1) host_2 = host_vertices[1] nagios_event2 = TestActionExecutor._get_nagios_event( host_2.get(VProps.ID), NOVA_HOST_DATASOURCE) processor.process_event(nagios_event2) alarms_attrs = {VProps.TYPE: NAGIOS_DATASOURCE} alarms_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=alarms_attrs) alarm1 = alarms_vertices[0] alarm2 = alarms_vertices[1] targets = { TFields.TARGET: alarm1.vertex_id, TFields.SOURCE: alarm2.vertex_id } action_spec = ActionSpecs(ActionType.ADD_CAUSAL_RELATIONSHIP, targets, {}) event_queue = queue.Queue() action_executor = ActionExecutor(event_queue) before_edge = processor.entity_graph.get_edge(alarm2.vertex_id, alarm1.vertex_id, EdgeLabels.CAUSES) # Test Action - do action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) new_edge = processor.entity_graph.get_edge(alarm2.vertex_id, alarm1.vertex_id, EdgeLabels.CAUSES) # Test Assertions self.assertIsNone(before_edge) self.assertIsNotNone(new_edge)
def __init__(self, conf, e_graph, scenario_repo, actions_callback, enabled=False): super(ScenarioEvaluator, self).__init__(conf, e_graph) self._db_connection = storage.get_connection_from_config(self._conf) self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(self._conf, actions_callback) self._entity_graph.subscribe(self.process_event) self._active_actions_tracker = ActiveActionsTracker( self._conf, self._db_connection) self.enabled = enabled self.connected_component_cache = defaultdict(dict)
def _init_executer(self): event_queue = queue.Queue() def actions_callback(event_type, data): event_queue.put(data) return event_queue, ActionExecutor(actions_callback)
def test_execute_update_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host_vertex_before = host_vertices[0] targets = {TFields.TARGET: host_vertex_before} props = {TFields.STATE: OperationalResourceState.SUBOPTIMAL} action_spec = ActionSpecs(ActionType.SET_STATE, targets, props) event_queue = queue.Queue() action_executor = ActionExecutor(event_queue) # Test Action - do action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) host_vertex_after = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions agg_state_before = host_vertex_before.get(VProps.AGGREGATED_STATE) self.assertTrue(agg_state_before != OperationalResourceState.SUBOPTIMAL) self.assertFalse(VProps.VITRAGE_STATE in host_vertex_before.properties) agg_state_after = host_vertex_after.get(VProps.AGGREGATED_STATE) self.assertEqual(agg_state_after, OperationalResourceState.SUBOPTIMAL) v_state_after = host_vertex_after.get(VProps.VITRAGE_STATE) self.assertEqual(v_state_after, OperationalResourceState.SUBOPTIMAL) # Test Action - undo action_executor.execute(action_spec, ActionMode.UNDO) processor.process_event(event_queue.get()) host_vertex_after_undo = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions agg_state_after_undo = host_vertex_before.get(VProps.AGGREGATED_STATE) self.assertEqual(agg_state_after_undo, agg_state_before) self.assertTrue( VProps.VITRAGE_STATE not in host_vertex_after_undo.properties)
def test_execute_add_and_remove_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host = host_vertices[0] targets = {TFields.TARGET: host} props = { TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE', TFields.SEVERITY: OperationalAlarmSeverity.CRITICAL, VProps.STATE: AlarmProps.ACTIVE_STATE, VProps.RESOURCE_ID: host[VProps.ID] } action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props) add_vertex_event = TestActionExecutor._get_vitrage_add_vertex_event( host, props[TFields.ALARM_NAME], props[TFields.SEVERITY]) processor.process_event(add_vertex_event) alarm_vertex_attrs = { VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE, VProps.VITRAGE_IS_DELETED: False } before_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) event_queue = queue.Queue() action_executor = ActionExecutor(self.conf, event_queue) # Test Action - undo action_executor.execute(action_spec, ActionMode.UNDO) event = event_queue.get() processor.process_event(event) after_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) # Test Assertions self.assertEqual(len(before_alarms) - 1, len(after_alarms))
def test_execute_add_and_remove_vertex(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host = host_vertices[0] targets = {TFields.TARGET: host.vertex_id} props = { TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE', TFields.SEVERITY: 'CRITICAL', VProps.STATE: AlarmProps.ALARM_ACTIVE_STATE } action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props) add_vertex_event = TestActionExecutor._get_vitrage_add_vertex_event( host.vertex_id, props[TFields.ALARM_NAME], props[TFields.SEVERITY]) processor.process_event(add_vertex_event) alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE, VProps.IS_DELETED: False} before_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) event_queue = queue.Queue() action_executor = ActionExecutor(event_queue) # Test Action - undo action_executor.execute(action_spec, ActionMode.UNDO) event = event_queue.get() processor.process_event(event) after_alarms = processor.entity_graph.get_vertices( vertex_attr_filter=alarm_vertex_attrs) # Test Assertions self.assertEqual(len(before_alarms) - 1, len(after_alarms))
def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._scenario_repo = scenario_repo self._entity_graph = entity_graph self._graph_algs = create_algorithm(entity_graph) self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf)) self.enabled = enabled
def test_execute_set_state(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host_vertex_before = host_vertices[0] targets = {TFields.TARGET: host_vertex_before} props = {TFields.STATE: OperationalResourceState.SUBOPTIMAL} action_spec = ActionSpecs(ActionType.SET_STATE, targets, props) event_queue = queue.Queue() action_executor = ActionExecutor(self.conf, event_queue) # Test Action - do action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) host_vertex_after = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions agg_state_before = \ host_vertex_before.get(VProps.VITRAGE_AGGREGATED_STATE) self.assertNotEqual(agg_state_before, OperationalResourceState.SUBOPTIMAL) self.assertNotIn(VProps.VITRAGE_STATE, host_vertex_before.properties) agg_state_after = \ host_vertex_after.get(VProps.VITRAGE_AGGREGATED_STATE) self.assertEqual(agg_state_after, OperationalResourceState.SUBOPTIMAL) v_state_after = host_vertex_after.get(VProps.VITRAGE_STATE) self.assertEqual(v_state_after, OperationalResourceState.SUBOPTIMAL) # Test Action - undo action_executor.execute(action_spec, ActionMode.UNDO) processor.process_event(event_queue.get()) host_vertex_after_undo = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions agg_state_after_undo = \ host_vertex_before.get(VProps.VITRAGE_AGGREGATED_STATE) self.assertEqual(agg_state_after_undo, agg_state_before) self.assertNotIn(VProps.VITRAGE_STATE, host_vertex_after_undo.properties)
def test_execute_mark_down(self): # Test Setup processor = self._create_processor_with_graph(self.conf) vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE} host_vertices = processor.entity_graph.get_vertices( vertex_attr_filter=vertex_attrs) host_vertex_before = host_vertices[0] targets = {TFields.TARGET: host_vertex_before} props = {} action_spec = ActionSpecs(ActionType.MARK_DOWN, targets, props) event_queue = queue.Queue() action_executor = ActionExecutor(self.conf, event_queue) # Test Action - do action_executor.execute(action_spec, ActionMode.DO) processor.process_event(event_queue.get()) host_vertex_after = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions self.assertTrue(host_vertex_after.get(VProps.IS_MARKED_DOWN)) # Test Action - undo action_executor.execute(action_spec, ActionMode.UNDO) processor.process_event(event_queue.get()) host_vertex_after_undo = processor.entity_graph.get_vertex( host_vertex_before.vertex_id) # Test Assertions self.assertFalse(host_vertex_after_undo.get(VProps.IS_MARKED_DOWN))
class ScenarioEvaluator(EvaluatorBase): def __init__(self, conf, e_graph, scenario_repo, actions_callback, enabled=False): super(ScenarioEvaluator, self).__init__(conf, e_graph) self._db_connection = storage.get_connection_from_config(self._conf) self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(self._conf, actions_callback) self._entity_graph.subscribe(self.process_event) self._active_actions_tracker = ActiveActionsTracker( self._conf, self._db_connection) self.enabled = enabled self.connected_component_cache = defaultdict(dict) @property def scenario_repo(self): return self._scenario_repo @scenario_repo.setter def scenario_repo(self, scenario_repo): self._scenario_repo = scenario_repo def run_evaluator(self): self.enabled = True vertices = self._entity_graph.get_vertices() start_time = time.time() for vertex in vertices: self.process_event(None, vertex, True) LOG.info('Run Evaluator on %s items - took %s', str(len(vertices)), str(time.time() - start_time)) def process_event(self, before, current, is_vertex, *args, **kwargs): """Notification of a change in the entity graph. :param is_vertex: :param before: The graph element (vertex or edge) prior to the change that happened. None if the element was just created. :param current: The graph element (vertex or edge) after the change that happened. Deleted elements should arrive with the vitrage_is_deleted property set to True """ if not self.enabled: LOG.debug("Process event disabled") return LOG.debug('Process event - starting') LOG.debug("Element before event: %s, Current element: %s", str(before), str(current)) before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ self._remove_overlap_scenarios(before_scenarios, current_scenarios) if len(before_scenarios) + len(current_scenarios): LOG.debug("Number of relevant scenarios found: undo = %s, do = %s", str(len(before_scenarios)), str(len(current_scenarios))) actions = self._process_and_get_actions(before, before_scenarios, ActionMode.UNDO) actions.extend(self._process_and_get_actions(current, current_scenarios, ActionMode.DO)) actions_to_preform = [] try: actions_to_preform = self._analyze_and_filter_actions(actions) except Exception as e: LOG.error("Evaluator error, will not execute actions %s", str(actions)) LOG.exception("Caught: %s", e) for action in actions_to_preform: LOG.info('Action: %s', self._action_str(action)) self._action_executor.execute(action.specs, action.mode) LOG.debug('Process event - completed') def _get_element_scenarios(self, element, is_vertex): if not element \ or element.get(VProps.VITRAGE_IS_DELETED) \ or element.get(EProps.VITRAGE_IS_DELETED): return [] elif is_vertex: return self._scenario_repo.get_scenarios_by_vertex(element) else: # is edge edge_desc = self._get_edge_description(element) return self._scenario_repo.get_scenarios_by_edge(edge_desc) def _get_edge_description(self, element): source = self._entity_graph.get_vertex(element.source_id) target = self._entity_graph.get_vertex(element.target_id) edge_desc = EdgeDescription(element, source, target) return edge_desc @staticmethod def _remove_overlap_scenarios(before, current): intersection = list(filter(lambda x: x in before, current)) before = list(filter(lambda x: x not in intersection, before)) current = list(filter(lambda x: x not in intersection, current)) return before, current def _process_and_get_actions(self, element, triggered_scenarios, mode): actions = [] for triggered_scenario in triggered_scenarios: LOG.debug("Processing: %s", str(triggered_scenario)) scenario_element = triggered_scenario[0] scenario = triggered_scenario[1] actions.extend(self._process_scenario(element, scenario, scenario_element, mode)) return actions def _process_scenario(self, element, scenario, scenario_elements, mode): if not isinstance(scenario_elements, list): scenario_elements = [scenario_elements] actions = [] for action in scenario.actions: for scenario_element in scenario_elements: matches = self._evaluate_subgraphs(scenario.subgraphs, element, scenario_element, action.targets[TARGET]) actions.extend(self._get_actions_from_matches(matches, mode, action)) return actions def _evaluate_subgraphs(self, subgraphs, element, scenario_element, action_target): if isinstance(element, Vertex): return self._find_vertex_subgraph_matching(subgraphs, action_target, element, scenario_element) else: return self._find_edge_subgraph_matching(subgraphs, action_target, element, scenario_element) def _get_actions_from_matches(self, combined_matches, mode, action_spec): actions = [] for is_switch_mode, matches in combined_matches: new_mode = mode if is_switch_mode: new_mode = ActionMode.UNDO \ if mode == ActionMode.DO else ActionMode.DO for match in matches: match_action_spec = self._get_action_spec(action_spec, match) items_ids = [match[1].vertex_id for match in match.items()] match_hash = hash(tuple(sorted(items_ids))) actions.append(ActionInfo(match_action_spec, new_mode, match_action_spec.id, match_hash)) return actions @staticmethod def _get_action_spec(action_spec, match): targets = action_spec.targets real_items = { target: match[target_id] for target, target_id in targets.items() } return ActionSpecs(action_spec.id, action_spec.type, real_items, action_spec.properties) @staticmethod def _generate_action_id(action_spec): targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()] return hash( (action_spec.type, tuple(sorted(targets)), tuple(sorted(action_spec.properties.items()))) ) def _analyze_and_filter_actions(self, actions): LOG.debug("Actions before filtering: %s", actions) actions_to_perform = [] for action_info in actions: if action_info.mode == ActionMode.DO: is_highest_score, exists = \ self._active_actions_tracker.calc_do_action(action_info) if is_highest_score and not exists: actions_to_perform.append(action_info) elif action_info.mode == ActionMode.UNDO: is_highest_score, second_highest = \ self._active_actions_tracker.calc_undo_action(action_info) if is_highest_score: # We should 'DO' the Second highest scored action so # to override the existing dominant action. # or, if there is no second highest scored action # So we just 'UNDO' the existing dominant action if second_highest: action_to_perform = self._db_action_to_action_info( second_highest) actions_to_perform.append(action_to_perform) else: actions_to_perform.append(action_info) unique_ordered_actions = OrderedDict() for action in actions_to_perform: id_ = ScenarioEvaluator._generate_action_id(action.specs) unique_ordered_actions[id_] = action return unique_ordered_actions.values() def _find_vertex_subgraph_matching(self, subgraphs, action_target, vertex, scenario_vertex): """calculates subgraph matching for vertex iterates over all the subgraphs, and checks if the triggered vertex is in the same connected component as the action then run subgraph matching on the vertex and return its result, otherwise return an empty list of matches. """ matches = [] for subgraph in subgraphs: connected_component = self.get_connected_component(subgraph, action_target) is_switch_mode = \ connected_component.get_vertex(scenario_vertex.vertex_id) if is_switch_mode: initial_map = Mapping(scenario_vertex, vertex, True) mat = self._entity_graph.algo.sub_graph_matching(subgraph, initial_map) matches.append((False, mat)) else: matches.append((True, [])) return matches def _find_edge_subgraph_matching(self, subgraphs, action_target, edge, scenario_edge): """calculates subgraph matching for edge iterates over all the subgraphs, and checks if the triggered edge is a negative edge then mark it as deleted=false and negative=false so that subgraph matching on that edge will work correctly. after running subgraph matching, we need to remove the negative vertices that were added due to the change above. """ matches = [] for subgraph in subgraphs: subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id, scenario_edge.target.vertex_id, scenario_edge.edge.label) if not subgraph_edge: continue is_switch_mode = subgraph_edge.get(NEG_CONDITION, False) connected_component = self.get_connected_component(subgraph, action_target) # change the vitrage_is_deleted and negative_condition props to # false when is_switch_mode=true so that when we have an event on a # negative_condition=true edge it will find the correct subgraph self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, False) initial_map = Mapping(scenario_edge.edge, edge, False) curr_matches = \ self._entity_graph.algo.sub_graph_matching(subgraph, initial_map) # switch back to the original values self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, True) self._remove_negative_vertices_from_matches(curr_matches, connected_component) matches.append((is_switch_mode, curr_matches)) return matches def get_connected_component(self, subgraph, target): connected_component = self.connected_component_cache.get( id(subgraph), {}).get(id(target)) if not connected_component: connected_component = subgraph.algo.graph_query_vertices( root_id=target, edge_query_dict={'!=': {NEG_CONDITION: True}}) self.connected_component_cache[id(subgraph)][id(target)] = \ connected_component return connected_component def _db_action_to_action_info(self, db_action): target = self._entity_graph.get_vertex(db_action.target_vertex_id) targets = {TARGET: target} if db_action.source_vertex_id: source = self._entity_graph.get_vertex(db_action.source_vertex_id) targets[SOURCE] = source scenario_action = self._scenario_repo.actions.get(db_action.action_id) properties = copy.copy(scenario_action.properties) action_specs = ActionSpecs( id=db_action.action_id, type=db_action.action_type, targets=targets, properties=properties, ) action_info = ActionInfo( specs=action_specs, mode=ActionMode.DO, action_id=db_action.action_id, trigger_id=db_action.trigger, ) return action_info @staticmethod def _switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, status): if is_switch_mode: scenario_edge.edge[NEG_CONDITION] = status scenario_edge.edge[EProps.VITRAGE_IS_DELETED] = status subgraph.update_edge(scenario_edge.edge) @staticmethod def _remove_negative_vertices_from_matches(matches, connected_component): for match in matches: ver_ids = [v.vertex_id for v in connected_component.get_vertices()] ver_to_remove = [id for id in match.keys() if id not in ver_ids] for v_id in ver_to_remove: del match[v_id] @staticmethod def _action_str(action): s = action.specs.targets.get(SOURCE, {}).get(VProps.VITRAGE_ID, '') t = action.specs.targets.get(TARGET, {}).get(VProps.VITRAGE_ID, '') return '%s %s \'%s\' targets (%s,%s)' % (action.mode.upper(), action.specs.type, action.action_id, s, t)
class ScenarioEvaluator(object): def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._entity_graph = entity_graph self._graph_algs = create_algorithm(entity_graph) self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self.enabled = enabled def process_event(self, before, current, is_vertex): """Notification of a change in the entity graph. :param is_vertex: :param before: The graph element (vertex or edge) prior to the change that happened. None if the element was just created. :param current: The graph element (vertex or edge) after the change that happened. Deleted elements should arrive with the is_deleted property set to True """ if not self.enabled: LOG.debug("Process event disabled") return LOG.debug('Process event - starting') LOG.debug("Element before event: %s, Current element: %s", str(before), str(current)) # todo (erosensw): support for NOT conditions - reverse logic before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ self._remove_overlap_scenarios(before_scenarios, current_scenarios) if len(before_scenarios) + len(current_scenarios): LOG.debug("Number of relevant scenarios found: undo = %s, do = %s", str(len(before_scenarios)), str(len(current_scenarios))) actions = self._process_and_get_actions(before, before_scenarios, ActionMode.UNDO) actions.update(self._process_and_get_actions(current, current_scenarios, ActionMode.DO)) if actions: LOG.debug("Actions to perform: %s", actions.values()) for action in actions.values(): action_spec = action[0] action_mode = action[1] self._action_executor.execute(action_spec, action_mode) LOG.debug('Process event - completed') def _get_element_scenarios(self, element, is_vertex): if not element \ or element.get(VProps.IS_DELETED) \ or element.get(EProps.IS_DELETED): return [] elif is_vertex: return self._scenario_repo.get_scenarios_by_vertex(element) else: # is edge edge_desc = self._get_edge_description(element) return self._scenario_repo.get_scenarios_by_edge(edge_desc) def _get_edge_description(self, element): source = self._entity_graph.get_vertex(element.source_id) target = self._entity_graph.get_vertex(element.target_id) edge_desc = EdgeDescription(element, source, target) return edge_desc @staticmethod def _remove_overlap_scenarios(before, current): intersection = list(filter(lambda x: x in before, current)) before = list(filter(lambda x: x not in intersection, before)) current = list(filter(lambda x: x not in intersection, current)) return before, current def _process_and_get_actions(self, element, triggered_scenarios, mode): actions = {} for triggered_scenario in triggered_scenarios: LOG.debug("Processing: %s", str(triggered_scenario)) scenario_element = triggered_scenario[0] scenario = triggered_scenario[1] actions.update(self._process_scenario(element, scenario, scenario_element, mode)) return actions def _process_scenario(self, element, scenario, scenario_elements, mode): if not isinstance(scenario_elements, list): scenario_elements = [scenario_elements] actions = {} for action in scenario.actions: for scenario_element in scenario_elements: matches = self._evaluate_full_condition(scenario.condition, element, scenario_element) if matches: for match in matches: spec, action_id = self._get_action_spec(action, match) actions[action_id] = (spec, mode) return actions @staticmethod def _get_action_spec(action_spec, match): targets = action_spec.targets real_items = { target: match[target_id] for target, target_id in targets.items() } revised_spec = ActionSpecs(action_spec.type, real_items, action_spec.properties) action_id = ScenarioEvaluator._generate_action_id(revised_spec) return revised_spec, action_id @staticmethod def _generate_action_id(action_spec): targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()] return hash( (action_spec.type, tuple(sorted(targets)), tuple(sorted(action_spec.properties.items()))) ) def _evaluate_full_condition(self, condition, element, scenario_element): condition_matches = [] for clause in condition: # OR condition means aggregation of matches, without duplicates and_condition_matches = \ self._evaluate_and_condition(clause, element, scenario_element) condition_matches += and_condition_matches return condition_matches def _evaluate_and_condition(self, condition, element, scenario_element): condition_g = create_graph("scenario condition") for term in condition: if not term.positive: # todo(erosensw): add support for NOT clauses LOG.error('Unsupported template with NOT operator') return [] if term.type == ENTITY: term.variable[VProps.IS_DELETED] = False condition_g.add_vertex(term.variable) else: # type = relationship edge_desc = term.variable self._set_relationship_not_deleted(edge_desc) self._add_relationship(condition_g, edge_desc) if isinstance(element, Vertex): initial_map = Mapping(scenario_element, element, True) else: initial_map = Mapping(scenario_element.edge, element, False) return self._graph_algs.sub_graph_matching(condition_g, [initial_map]) @staticmethod def _set_relationship_not_deleted(edge_description): edge_description.source[VProps.IS_DELETED] = False edge_description.target[VProps.IS_DELETED] = False edge_description.edge[EProps.IS_DELETED] = False @staticmethod def _add_relationship(condition_graph, edge_description): condition_graph.add_vertex(edge_description.source) condition_graph.add_vertex(edge_description.target) condition_graph.add_edge(edge_description.edge)
class ScenarioEvaluator(object): def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._scenario_repo = scenario_repo self._entity_graph = entity_graph self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf)) self.enabled = enabled self.connected_component_cache = defaultdict(dict) @property def scenario_repo(self): return self._scenario_repo @scenario_repo.setter def scenario_repo(self, scenario_repo): self._scenario_repo = scenario_repo def process_event(self, before, current, is_vertex, *args, **kwargs): """Notification of a change in the entity graph. :param is_vertex: :param before: The graph element (vertex or edge) prior to the change that happened. None if the element was just created. :param current: The graph element (vertex or edge) after the change that happened. Deleted elements should arrive with the is_deleted property set to True """ if not self.enabled: LOG.debug("Process event disabled") return LOG.debug('Process event - starting') LOG.debug("Element before event: %s, Current element: %s", str(before), str(current)) before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ self._remove_overlap_scenarios(before_scenarios, current_scenarios) if len(before_scenarios) + len(current_scenarios): LOG.debug("Number of relevant scenarios found: undo = %s, do = %s", str(len(before_scenarios)), str(len(current_scenarios))) actions = self._process_and_get_actions(before, before_scenarios, ActionMode.UNDO) actions.extend(self._process_and_get_actions(current, current_scenarios, ActionMode.DO)) if actions: LOG.debug("Actions to perform: %s", actions) filtered_actions = \ self._analyze_and_filter_actions(actions) LOG.debug("Actions filtered: %s", filtered_actions) for action in filtered_actions: self._action_executor.execute(action.specs, action.mode) LOG.debug('Process event - completed') def _get_element_scenarios(self, element, is_vertex): if not element \ or element.get(VProps.IS_DELETED) \ or element.get(EProps.IS_DELETED): return [] elif is_vertex: return self._scenario_repo.get_scenarios_by_vertex(element) else: # is edge edge_desc = self._get_edge_description(element) return self._scenario_repo.get_scenarios_by_edge(edge_desc) def _get_edge_description(self, element): source = self._entity_graph.get_vertex(element.source_id) target = self._entity_graph.get_vertex(element.target_id) edge_desc = EdgeDescription(element, source, target) return edge_desc @staticmethod def _remove_overlap_scenarios(before, current): intersection = list(filter(lambda x: x in before, current)) before = list(filter(lambda x: x not in intersection, before)) current = list(filter(lambda x: x not in intersection, current)) return before, current def _process_and_get_actions(self, element, triggered_scenarios, mode): actions = [] for triggered_scenario in triggered_scenarios: LOG.debug("Processing: %s", str(triggered_scenario)) scenario_element = triggered_scenario[0] scenario = triggered_scenario[1] actions.extend(self._process_scenario(element, scenario, scenario_element, mode)) return actions def _process_scenario(self, element, scenario, scenario_elements, mode): if not isinstance(scenario_elements, list): scenario_elements = [scenario_elements] actions = [] for action in scenario.actions: for scenario_element in scenario_elements: matches = self._evaluate_subgraphs(scenario.subgraphs, element, scenario_element, action.targets['target']) actions.extend(self._get_actions_from_matches(matches, mode, action, scenario)) return actions def _evaluate_subgraphs(self, subgraphs, element, scenario_element, action_target): if isinstance(element, Vertex): return self._find_vertex_subgraph_matching(subgraphs, action_target, element, scenario_element) else: return self._find_edge_subgraph_matching(subgraphs, action_target, element, scenario_element) def _get_actions_from_matches(self, combined_matches, mode, action_spec, scenario): actions = [] for is_switch_mode, matches in combined_matches: new_mode = mode if is_switch_mode: new_mode = ActionMode.UNDO \ if mode == ActionMode.DO else ActionMode.DO for match in matches: spec = self._get_action_spec(action_spec, match) items_ids = [match[1].vertex_id for match in match.items()] match_hash = hash(tuple(sorted(items_ids))) actions.append(ActionInfo(spec, new_mode, scenario.id, match_hash)) return actions @staticmethod def _get_action_spec(action_spec, match): targets = action_spec.targets real_items = { target: match[target_id] for target, target_id in targets.items() } return ActionSpecs(action_spec.type, real_items, action_spec.properties) @staticmethod def _generate_action_id(action_spec): targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()] return hash( (action_spec.type, tuple(sorted(targets)), tuple(sorted(action_spec.properties.items()))) ) def _analyze_and_filter_actions(self, actions): actions_to_perform = {} for action in actions: key = self._action_tracker.get_key(action.specs) prev_dominant = self._action_tracker.get_dominant_action(key) if action.mode == ActionMode.DO: self._action_tracker.insert_action(key, action) else: self._action_tracker.remove_action(key, action) new_dominant = self._action_tracker.get_dominant_action(key) # todo: (erosensw) improvement - first analyze DOs, then UNDOs if not new_dominant: # removed last entry for key undo_action = ActionInfo(prev_dominant.specs, ActionMode.UNDO, prev_dominant.scenario_id, prev_dominant.trigger_id) actions_to_perform[key] = undo_action elif new_dominant != prev_dominant: actions_to_perform[key] = new_dominant # filter the same action final_actions = {ScenarioEvaluator._generate_action_id(action.specs): action for action in actions_to_perform.values()} return final_actions.values() def _find_vertex_subgraph_matching(self, subgraphs, action_target, vertex, scenario_vertex): """calculates subgraph matching for vertex iterates over all the subgraphs, and checks if the triggered vertex is in the same connected component as the action then run subgraph matching on the vertex and return its result, otherwise return an empty list of matches. """ matches = [] for subgraph in subgraphs: connected_component = self.get_connected_component(subgraph, action_target) is_switch_mode = \ connected_component.get_vertex(scenario_vertex.vertex_id) if is_switch_mode: initial_map = Mapping(scenario_vertex, vertex, True) mat = self._entity_graph.algo.sub_graph_matching(subgraph, initial_map) matches.append((False, mat)) else: matches.append((True, [])) return matches def _find_edge_subgraph_matching(self, subgraphs, action_target, edge, scenario_edge): """calculates subgraph matching for edge iterates over all the subgraphs, and checks if the triggered edge is a negative edge then mark it as deleted=false and negative=false so that subgraph matching on that edge will work correctly. after running subgraph matching, we need to remove the negative vertices that were added due to the change above. """ matches = [] for subgraph in subgraphs: subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id, scenario_edge.target.vertex_id, scenario_edge.edge.label) if not subgraph_edge: continue is_switch_mode = subgraph_edge.get(NEG_CONDITION, False) connected_component = self.get_connected_component(subgraph, action_target) # change the is_deleted and negative_condition props to false when # is_switch_mode=true so that when we have an event on a # negative_condition=true edge it will find the correct subgraph self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, False) initial_map = Mapping(scenario_edge.edge, edge, False) curr_matches = \ self._entity_graph.algo.sub_graph_matching(subgraph, initial_map) # switch back to the original values self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, True) self._remove_negative_vertices_from_matches(curr_matches, connected_component) matches.append((is_switch_mode, curr_matches)) return matches def get_connected_component(self, subgraph, target): connected_component = self.connected_component_cache.get( id(subgraph), {}).get(id(target)) if not connected_component: connected_component = subgraph.algo.graph_query_vertices( root_id=target, edge_query_dict={'!=': {NEG_CONDITION: True}}) self.connected_component_cache[id(subgraph)][id(target)] = \ connected_component return connected_component @staticmethod def _switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, status): if is_switch_mode: scenario_edge.edge[NEG_CONDITION] = status scenario_edge.edge[EProps.IS_DELETED] = status subgraph.update_edge(scenario_edge.edge) @staticmethod def _remove_negative_vertices_from_matches(matches, connected_component): for match in matches: ver_ids = [v.vertex_id for v in connected_component.get_vertices()] ver_to_remove = [id for id in match.keys() if id not in ver_ids] for v_id in ver_to_remove: del match[v_id]
class ScenarioEvaluator(object): def __init__(self, conf, e_graph, scenario_repo, actions_callback, enabled=False): self._conf = conf self._entity_graph = e_graph self._db = storage.get_connection_from_config(self._conf) self._scenario_repo = scenario_repo self._action_executor = ActionExecutor(self._conf, actions_callback) self._entity_graph.subscribe(self.process_event) self.enabled = enabled self.connected_component_cache = defaultdict(dict) @property def scenario_repo(self): return self._scenario_repo @scenario_repo.setter def scenario_repo(self, scenario_repo): self._scenario_repo = scenario_repo def run_evaluator(self, action_mode=ActionMode.DO): self.enabled = True vertices = self._entity_graph.get_vertices() start_time = time.time() for vertex in vertices: if action_mode == ActionMode.DO: self.process_event(None, vertex, True) elif action_mode == ActionMode.UNDO: self.process_event(vertex, None, True) LOG.info('Run %s Evaluator on %s items - took %s', action_mode, len(vertices), (time.time() - start_time)) def process_event(self, before, current, is_vertex, *args, **kwargs): """Notification of a change in the entity graph. :param is_vertex: :param before: The graph element (vertex or edge) prior to the change that happened. None if the element was just created. :param current: The graph element (vertex or edge) after the change that happened. Deleted elements should arrive with the vitrage_is_deleted property set to True """ if not self.enabled: LOG.debug("Process event disabled") return LOG.debug('Process event - starting') LOG.debug("Element before event: %s, Current element: %s", before, current) before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ self._remove_overlap_scenarios(before_scenarios, current_scenarios) if len(before_scenarios) + len(current_scenarios): LOG.debug("Number of relevant scenarios found: undo = %s, do = %s", len(before_scenarios), len(current_scenarios)) actions = self._process_and_get_actions(before, before_scenarios, ActionMode.UNDO) actions.extend( self._process_and_get_actions(current, current_scenarios, ActionMode.DO)) actions_to_preform = [] try: actions_to_preform = self._analyze_and_filter_actions(actions) except Exception: LOG.exception("Evaluator error, will not execute actions %s", actions) self._action_executor.execute(actions_to_preform) LOG.debug('Process event - completed') def _get_element_scenarios(self, element, is_vertex): if not element \ or element.get(VProps.VITRAGE_IS_DELETED) \ or element.get(EProps.VITRAGE_IS_DELETED): return [] elif is_vertex: return self._scenario_repo.get_scenarios_by_vertex(element) else: # is edge edge_desc = self._get_edge_description(element) return self._scenario_repo.get_scenarios_by_edge(edge_desc) def _get_edge_description(self, element): source = self._entity_graph.get_vertex(element.source_id) target = self._entity_graph.get_vertex(element.target_id) edge_desc = EdgeDescription(element, source, target) return edge_desc @staticmethod def _remove_overlap_scenarios(before, current): intersection = list(filter(lambda x: x in before, current)) before = list(filter(lambda x: x not in intersection, before)) current = list(filter(lambda x: x not in intersection, current)) return before, current def _process_and_get_actions(self, element, triggered_scenarios, mode): actions = [] for triggered_scenario in triggered_scenarios: LOG.debug("Processing: %s", triggered_scenario) scenario_element = triggered_scenario[0] scenario = triggered_scenario[1] actions.extend( self._process_scenario(element, scenario, scenario_element, mode)) return actions def _process_scenario(self, element, scenario, scenario_elements, mode): if not isinstance(scenario_elements, list): scenario_elements = [scenario_elements] actions = [] for action in scenario.actions: for scenario_element in scenario_elements: matches = self._evaluate_subgraphs(scenario.subgraphs, element, scenario_element, action.targets[TARGET]) actions.extend( self._get_actions_from_matches(scenario.version, matches, mode, action)) return actions def _evaluate_subgraphs(self, subgraphs, element, scenario_element, action_target): if isinstance(element, Vertex): return self._find_vertex_subgraph_matching(subgraphs, action_target, element, scenario_element) else: return self._find_edge_subgraph_matching(subgraphs, action_target, element, scenario_element) def _get_actions_from_matches(self, scenario_version, combined_matches, mode, action_spec): actions = [] for is_switch_mode, matches in combined_matches: new_mode = mode if is_switch_mode: new_mode = ActionMode.UNDO \ if mode == ActionMode.DO else ActionMode.DO template_schema = \ TemplateSchemaFactory().template_schema(scenario_version) for match in matches: match_action_spec = self._get_action_spec(action_spec, match) items_ids = \ [match_item[1].vertex_id for match_item in match.items()] match_hash = md5(tuple(sorted(items_ids))) self._evaluate_property_functions(template_schema, match, match_action_spec.properties) actions.append( ActionInfo(match_action_spec, new_mode, match_action_spec.id, match_hash)) return actions def _evaluate_property_functions(self, template_schema, match, action_props): """Evaluate the action properties, in case they contain functions In template version 2 we introduced functions, and specifically the get_attr function. This method evaluate its value and updates the action properties, before the action is being executed. Example: - action: action_type: execute_mistral properties: workflow: evacuate_vm input: vm_name: get_attr(instance1,name) force: false In this example, the method will iterate over 'properties', and then recursively over 'input', and for 'vm_name' it will replace the call for get_attr with the actual name of the VM. The input for the Mistral workflow will then be: vm_name: vm_1 force: false """ for key, value in action_props.items(): if isinstance(value, dict): # Recursive call for a dictionary self._evaluate_property_functions(template_schema, match, value) elif value is not None and is_function(value): # The value is a function func_and_args = re.split('[(),]', value) func_name = func_and_args.pop(0) args = [arg.strip() for arg in func_and_args if len(arg) > 0] # Get the function, execute it and update the property value func = template_schema.functions.get(func_name) action_props[key] = func(match, *args) LOG.debug('Changed property %s value from %s to %s', key, value, action_props[key]) @staticmethod def _get_action_spec(action_spec, match): targets = action_spec.targets real_items = { target: match[target_id] for target, target_id in targets.items() } return ActionSpecs(action_spec.id, action_spec.type, real_items, action_spec.properties) @staticmethod def _generate_action_id(action_spec): """Generate a unique action id for the action BEWARE: The value created here should not be stored in database, as in python3, the hash function seed changes after program restart """ targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()] return hash( (action_spec.type, tuple(sorted(targets)), tuple(sorted(recursive_keypairs(action_spec.properties))))) def _analyze_and_filter_actions(self, actions): LOG.debug("Actions before filtering: %s", actions) if not actions: return [] active_actions = ActiveActionsTracker(self._conf, self._db, actions) for action_info in actions: if action_info.mode == ActionMode.DO: active_actions.calc_do_action(action_info) elif action_info.mode == ActionMode.UNDO: active_actions.calc_undo_action(action_info) active_actions.flush_db_updates() unique_ordered_actions = OrderedDict() for action in active_actions.actions_to_perform: if isinstance(action, models.ActiveAction): action = self._db_action_to_action_info(action) id_ = self._generate_action_id(action.specs) unique_ordered_actions[id_] = action return unique_ordered_actions.values() def _find_vertex_subgraph_matching(self, subgraphs, action_target, vertex, scenario_vertex): """calculates subgraph matching for vertex iterates over all the subgraphs, and checks if the triggered vertex is in the same connected component as the action then run subgraph matching on the vertex and return its result, otherwise return an empty list of matches. """ matches = [] for subgraph in subgraphs: connected_component = self.get_connected_component( subgraph, action_target) is_switch_mode = \ connected_component.get_vertex(scenario_vertex.vertex_id) if is_switch_mode: initial_map = Mapping(scenario_vertex, vertex, True) mat = self._entity_graph.algo.sub_graph_matching( subgraph, initial_map) matches.append((False, mat)) else: matches.append((True, [])) return matches def _find_edge_subgraph_matching(self, subgraphs, action_target, edge, scenario_edge): """calculates subgraph matching for edge iterates over all the subgraphs, and checks if the triggered edge is a negative edge then mark it as deleted=false and negative=false so that subgraph matching on that edge will work correctly. after running subgraph matching, we need to remove the negative vertices that were added due to the change above. """ matches = [] for subgraph in subgraphs: subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id, scenario_edge.target.vertex_id, scenario_edge.edge.label) if not subgraph_edge: continue is_switch_mode = subgraph_edge.get(NEG_CONDITION, False) connected_component = self.get_connected_component( subgraph, action_target) # change the vitrage_is_deleted and negative_condition props to # false when is_switch_mode=true so that when we have an event on a # negative_condition=true edge it will find the correct subgraph self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, False) initial_map = Mapping(scenario_edge.edge, edge, False) curr_matches = \ self._entity_graph.algo.sub_graph_matching(subgraph, initial_map) # switch back to the original values self._switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, True) self._remove_negative_vertices_from_matches( curr_matches, connected_component) matches.append((is_switch_mode, curr_matches)) return matches def get_connected_component(self, subgraph, target): connected_component = self.connected_component_cache.get( id(subgraph), {}).get(id(target)) if not connected_component: connected_component = subgraph.algo.graph_query_vertices( root_id=target, edge_query_dict={'!=': { NEG_CONDITION: True }}) self.connected_component_cache[id(subgraph)][id(target)] = \ connected_component return connected_component def _db_action_to_action_info(self, db_action): target = self._entity_graph.get_vertex(db_action.target_vertex_id) targets = {TARGET: target} if db_action.source_vertex_id: source = self._entity_graph.get_vertex(db_action.source_vertex_id) targets[SOURCE] = source scenario_action = self._scenario_repo.actions.get(db_action.action_id) properties = copy.copy(scenario_action.properties) action_specs = ActionSpecs( id=db_action.action_id, type=db_action.action_type, targets=targets, properties=properties, ) action_info = ActionInfo( specs=action_specs, mode=ActionMode.DO, action_id=db_action.action_id, trigger_id=db_action.trigger, ) return action_info @staticmethod def _switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph, status): if is_switch_mode: scenario_edge.edge[NEG_CONDITION] = status scenario_edge.edge[EProps.VITRAGE_IS_DELETED] = status subgraph.update_edge(scenario_edge.edge) @staticmethod def _remove_negative_vertices_from_matches(matches, connected_component): for match in matches: ver_ids = [v.vertex_id for v in connected_component.get_vertices()] ver_to_remove = [id for id in match.keys() if id not in ver_ids] for v_id in ver_to_remove: del match[v_id]
class ScenarioEvaluator(object): def __init__(self, conf, entity_graph, scenario_repo, event_queue, enabled=False): self.conf = conf self._scenario_repo = scenario_repo self._entity_graph = entity_graph self._graph_algs = create_algorithm(entity_graph) self._action_executor = ActionExecutor(event_queue) self._entity_graph.subscribe(self.process_event) self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf)) self.enabled = enabled @property def scenario_repo(self): return self._scenario_repo @scenario_repo.setter def scenario_repo(self, scenario_repo): self._scenario_repo = scenario_repo def process_event(self, before, current, is_vertex, *args, **kwargs): """Notification of a change in the entity graph. :param is_vertex: :param before: The graph element (vertex or edge) prior to the change that happened. None if the element was just created. :param current: The graph element (vertex or edge) after the change that happened. Deleted elements should arrive with the is_deleted property set to True """ if not self.enabled: LOG.debug("Process event disabled") return LOG.debug('Process event - starting') LOG.debug("Element before event: %s, Current element: %s", str(before), str(current)) # todo (erosensw): support for NOT conditions - reverse logic before_scenarios = self._get_element_scenarios(before, is_vertex) current_scenarios = self._get_element_scenarios(current, is_vertex) before_scenarios, current_scenarios = \ self._remove_overlap_scenarios(before_scenarios, current_scenarios) if len(before_scenarios) + len(current_scenarios): LOG.debug("Number of relevant scenarios found: undo = %s, do = %s", str(len(before_scenarios)), str(len(current_scenarios))) actions = self._process_and_get_actions(before, before_scenarios, ActionMode.UNDO) actions.update(self._process_and_get_actions(current, current_scenarios, ActionMode.DO)) if actions: LOG.debug("Actions to perform: %s", actions.values()) filtered_actions = \ self._analyze_and_filter_actions(actions.values()) LOG.debug("Actions filtered: %s", filtered_actions) for action in filtered_actions: self._action_executor.execute(action.specs, action.mode) LOG.debug('Process event - completed') def _get_element_scenarios(self, element, is_vertex): if not element \ or element.get(VProps.IS_DELETED) \ or element.get(EProps.IS_DELETED): return [] elif is_vertex: return self._scenario_repo.get_scenarios_by_vertex(element) else: # is edge edge_desc = self._get_edge_description(element) return self._scenario_repo.get_scenarios_by_edge(edge_desc) def _get_edge_description(self, element): source = self._entity_graph.get_vertex(element.source_id) target = self._entity_graph.get_vertex(element.target_id) edge_desc = EdgeDescription(element, source, target) return edge_desc @staticmethod def _remove_overlap_scenarios(before, current): intersection = list(filter(lambda x: x in before, current)) before = list(filter(lambda x: x not in intersection, before)) current = list(filter(lambda x: x not in intersection, current)) return before, current def _process_and_get_actions(self, element, triggered_scenarios, mode): actions = {} for triggered_scenario in triggered_scenarios: LOG.debug("Processing: %s", str(triggered_scenario)) scenario_element = triggered_scenario[0] scenario = triggered_scenario[1] actions.update(self._process_scenario(element, scenario, scenario_element, mode)) return actions def _process_scenario(self, element, scenario, scenario_elements, mode): if not isinstance(scenario_elements, list): scenario_elements = [scenario_elements] actions = {} for action in scenario.actions: for scenario_element in scenario_elements: matches = self._evaluate_full_condition(scenario.condition, element, scenario_element) if matches: for match in matches: spec, action_id = self._get_action_spec(action, match) match_hash = hash(tuple(sorted(match.items()))) actions[action_id] = \ ActionInfo(spec, mode, scenario.id, match_hash) return actions @staticmethod def _get_action_spec(action_spec, match): targets = action_spec.targets real_items = { target: match[target_id] for target, target_id in targets.items() } revised_spec = ActionSpecs(action_spec.type, real_items, action_spec.properties) # noinspection PyTypeChecker action_id = ScenarioEvaluator._generate_action_id(revised_spec) return revised_spec, action_id @staticmethod def _generate_action_id(action_spec): targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()] return hash( (action_spec.type, tuple(sorted(targets)), tuple(sorted(action_spec.properties.items()))) ) def _evaluate_full_condition(self, condition, element, scenario_element): condition_matches = [] for clause in condition: # OR condition means aggregation of matches, without duplicates and_condition_matches = \ self._evaluate_and_condition(clause, element, scenario_element) condition_matches += and_condition_matches return condition_matches def _evaluate_and_condition(self, condition, element, scenario_element): condition_g = create_graph("scenario condition") for term in condition: if not term.positive: # todo(erosensw): add support for NOT clauses LOG.error('Template with NOT operator current not supported') return [] if term.type == ENTITY: term.variable[VProps.IS_DELETED] = False condition_g.add_vertex(term.variable) else: # type = relationship edge_desc = term.variable self._set_relationship_not_deleted(edge_desc) self._add_relationship(condition_g, edge_desc) if isinstance(element, Vertex): initial_map = Mapping(scenario_element, element, True) else: initial_map = Mapping(scenario_element.edge, element, False) return self._graph_algs.sub_graph_matching(condition_g, [initial_map]) @staticmethod def _set_relationship_not_deleted(edge_description): edge_description.source[VProps.IS_DELETED] = False edge_description.target[VProps.IS_DELETED] = False edge_description.edge[EProps.IS_DELETED] = False @staticmethod def _add_relationship(condition_graph, edge_description): condition_graph.add_vertex(edge_description.source) condition_graph.add_vertex(edge_description.target) condition_graph.add_edge(edge_description.edge) def _analyze_and_filter_actions(self, actions): actions_to_perform = {} for action in actions: key = self._action_tracker.get_key(action.specs) prev_dominant = self._action_tracker.get_dominant_action(key) if action.mode == ActionMode.DO: self._action_tracker.insert_action(key, action) else: self._action_tracker.remove_action(key, action) new_dominant = self._action_tracker.get_dominant_action(key) # todo: (erosensw) improvement - first analyze DOs, then UNDOs if not new_dominant: # removed last entry for key undo_action = ActionInfo(prev_dominant.specs, ActionMode.UNDO, prev_dominant.scenario_id, prev_dominant.trigger_id) actions_to_perform[key] = undo_action elif new_dominant != prev_dominant: actions_to_perform[key] = new_dominant return actions_to_perform.values()