def test_execute_add_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host = host_vertices[0]
targets = {TFields.TARGET: host}
props = {
TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE',
TFields.SEVERITY: 'CRITICAL',
VProps.STATE: AlarmProps.ACTIVE_STATE
}
# Raise alarm action adds new vertex with type vitrage to the graph
action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props)
alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE}
before_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
event_queue = queue.Queue()
action_executor = ActionExecutor(event_queue)
expected_alarm_id = 'ALARM:vitrage:%s:%s' % (props[TFields.ALARM_NAME],
host.vertex_id)
# Test Action
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
after_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
# Assertions
self.assertEqual(len(before_alarms) + 1, len(after_alarms))
self.assert_is_not_empty(after_alarms)
alarms = [
alarm for alarm in after_alarms
if alarm.vertex_id == expected_alarm_id
]
# Expected exactly one alarm with expected id
self.assertEqual(1, len(alarms))
alarm = alarms[0]
self.assertEqual(alarm.properties[VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(alarm.properties[VProps.TYPE], VITRAGE_TYPE)
self.assertEqual(alarm.properties[VProps.SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.OPERATIONAL_SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.STATE],
AlarmProps.ACTIVE_STATE)
def test_execute_add_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host = host_vertices[0]
targets = {TFields.TARGET: host}
props = {
TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE',
TFields.SEVERITY: OperationalAlarmSeverity.CRITICAL,
VProps.STATE: AlarmProps.ACTIVE_STATE,
VProps.RESOURCE_ID: host[VProps.ID],
VProps.VITRAGE_ID: 'DUMMY_ID'
}
# Raise alarm action adds new vertex with type vitrage to the graph
action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props)
alarm_vertex_attrs = {VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE}
before_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
event_queue = queue.Queue()
action_executor = ActionExecutor(self.conf, event_queue)
# Test Action
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
after_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
# Assertions
self.assertEqual(len(before_alarms) + 1, len(after_alarms))
self.assert_is_not_empty(after_alarms)
alarm = after_alarms[0]
self.assertEqual(alarm.properties[VProps.VITRAGE_CATEGORY],
EntityCategory.ALARM)
self.assertEqual(alarm.properties[VProps.VITRAGE_TYPE],
VITRAGE_DATASOURCE)
self.assertEqual(alarm.properties[VProps.SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.VITRAGE_OPERATIONAL_SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.STATE],
AlarmProps.ACTIVE_STATE)
self.assertEqual(
alarm.properties[VProps.VITRAGE_RESOURCE_ID],
action_spec.targets[TTFields.TARGET][VProps.VITRAGE_ID]),
self.assertEqual(alarm.properties[VProps.VITRAGE_RESOURCE_TYPE],
NOVA_HOST_DATASOURCE)
def test_execute_add_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host = host_vertices[0]
targets = {TFields.TARGET: host}
props = {
TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE',
TFields.SEVERITY: 'CRITICAL',
VProps.STATE: AlarmProps.ACTIVE_STATE
}
# Raise alarm action adds new vertex with type vitrage to the graph
action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props)
alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE}
before_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
event_queue = queue.Queue()
action_executor = ActionExecutor(event_queue)
expected_alarm_id = 'ALARM:vitrage:%s:%s' % (props[TFields.ALARM_NAME],
host.vertex_id)
# Test Action
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
after_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
# Assertions
self.assertEqual(len(before_alarms) + 1, len(after_alarms))
self.assert_is_not_empty(after_alarms)
alarms = [alarm for alarm in after_alarms
if alarm.vertex_id == expected_alarm_id]
# Expected exactly one alarm with expected id
self.assertEqual(1, len(alarms))
alarm = alarms[0]
self.assertEqual(alarm.properties[VProps.CATEGORY],
EntityCategory.ALARM)
self.assertEqual(alarm.properties[VProps.TYPE],
VITRAGE_TYPE)
self.assertEqual(alarm.properties[VProps.SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.OPERATIONAL_SEVERITY],
props[TFields.SEVERITY])
self.assertEqual(alarm.properties[VProps.STATE],
AlarmProps.ACTIVE_STATE)
def test_execute_set_state(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host_vertex_before = host_vertices[0]
targets = {TFields.TARGET: host_vertex_before}
props = {TFields.STATE: OperationalResourceState.SUBOPTIMAL}
action_spec = ActionSpecs(ActionType.SET_STATE, targets, props)
event_queue = queue.Queue()
action_executor = ActionExecutor(self.conf, event_queue)
# Test Action - do
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
host_vertex_after = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
agg_state_before = \
host_vertex_before.get(VProps.VITRAGE_AGGREGATED_STATE)
self.assertNotEqual(agg_state_before,
OperationalResourceState.SUBOPTIMAL)
self.assertNotIn(VProps.VITRAGE_STATE, host_vertex_before.properties)
agg_state_after = \
host_vertex_after.get(VProps.VITRAGE_AGGREGATED_STATE)
self.assertEqual(agg_state_after, OperationalResourceState.SUBOPTIMAL)
v_state_after = host_vertex_after.get(VProps.VITRAGE_STATE)
self.assertEqual(v_state_after, OperationalResourceState.SUBOPTIMAL)
# Test Action - undo
action_executor.execute(action_spec, ActionMode.UNDO)
processor.process_event(event_queue.get())
host_vertex_after_undo = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
agg_state_after_undo = \
host_vertex_before.get(VProps.VITRAGE_AGGREGATED_STATE)
self.assertEqual(agg_state_after_undo, agg_state_before)
self.assertNotIn(VProps.VITRAGE_STATE,
host_vertex_after_undo.properties)
def test_execute_add_edge(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host_1 = host_vertices[0]
nagios_event1 = TestActionExecutor._get_nagios_event(
host_1.get(VProps.ID), NOVA_HOST_DATASOURCE)
processor.process_event(nagios_event1)
host_2 = host_vertices[1]
nagios_event2 = TestActionExecutor._get_nagios_event(
host_2.get(VProps.ID), NOVA_HOST_DATASOURCE)
processor.process_event(nagios_event2)
alarms_attrs = {VProps.TYPE: NAGIOS_DATASOURCE}
alarms_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=alarms_attrs)
alarm1 = alarms_vertices[0]
alarm2 = alarms_vertices[1]
targets = {
TFields.TARGET: alarm1.vertex_id,
TFields.SOURCE: alarm2.vertex_id
}
action_spec = ActionSpecs(ActionType.ADD_CAUSAL_RELATIONSHIP,
targets,
{})
event_queue = queue.Queue()
action_executor = ActionExecutor(event_queue)
before_edge = processor.entity_graph.get_edge(alarm2.vertex_id,
alarm1.vertex_id,
EdgeLabels.CAUSES)
# Test Action - do
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
new_edge = processor.entity_graph.get_edge(alarm2.vertex_id,
alarm1.vertex_id,
EdgeLabels.CAUSES)
# Test Assertions
self.assertIsNone(before_edge)
self.assertIsNotNone(new_edge)
def test_execute_update_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host_vertex_before = host_vertices[0]
targets = {TFields.TARGET: host_vertex_before}
props = {TFields.STATE: OperationalResourceState.SUBOPTIMAL}
action_spec = ActionSpecs(ActionType.SET_STATE, targets, props)
event_queue = queue.Queue()
action_executor = ActionExecutor(event_queue)
# Test Action - do
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
host_vertex_after = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
agg_state_before = host_vertex_before.get(VProps.AGGREGATED_STATE)
self.assertTrue(agg_state_before !=
OperationalResourceState.SUBOPTIMAL)
self.assertFalse(VProps.VITRAGE_STATE in host_vertex_before.properties)
agg_state_after = host_vertex_after.get(VProps.AGGREGATED_STATE)
self.assertEqual(agg_state_after, OperationalResourceState.SUBOPTIMAL)
v_state_after = host_vertex_after.get(VProps.VITRAGE_STATE)
self.assertEqual(v_state_after, OperationalResourceState.SUBOPTIMAL)
# Test Action - undo
action_executor.execute(action_spec, ActionMode.UNDO)
processor.process_event(event_queue.get())
host_vertex_after_undo = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
agg_state_after_undo = host_vertex_before.get(VProps.AGGREGATED_STATE)
self.assertEqual(agg_state_after_undo, agg_state_before)
self.assertTrue(
VProps.VITRAGE_STATE not in host_vertex_after_undo.properties)
def test_execute_add_and_remove_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host = host_vertices[0]
targets = {TFields.TARGET: host}
props = {
TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE',
TFields.SEVERITY: OperationalAlarmSeverity.CRITICAL,
VProps.STATE: AlarmProps.ACTIVE_STATE,
VProps.RESOURCE_ID: host[VProps.ID]
}
action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props)
add_vertex_event = TestActionExecutor._get_vitrage_add_vertex_event(
host, props[TFields.ALARM_NAME], props[TFields.SEVERITY])
processor.process_event(add_vertex_event)
alarm_vertex_attrs = {
VProps.VITRAGE_TYPE: VITRAGE_DATASOURCE,
VProps.VITRAGE_IS_DELETED: False
}
before_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
event_queue = queue.Queue()
action_executor = ActionExecutor(self.conf, event_queue)
# Test Action - undo
action_executor.execute(action_spec, ActionMode.UNDO)
event = event_queue.get()
processor.process_event(event)
after_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
# Test Assertions
self.assertEqual(len(before_alarms) - 1, len(after_alarms))
def test_execute_add_and_remove_vertex(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host = host_vertices[0]
targets = {TFields.TARGET: host.vertex_id}
props = {
TFields.ALARM_NAME: 'VM_CPU_SUBOPTIMAL_PERFORMANCE',
TFields.SEVERITY: 'CRITICAL',
VProps.STATE: AlarmProps.ALARM_ACTIVE_STATE
}
action_spec = ActionSpecs(ActionType.RAISE_ALARM, targets, props)
add_vertex_event = TestActionExecutor._get_vitrage_add_vertex_event(
host.vertex_id,
props[TFields.ALARM_NAME],
props[TFields.SEVERITY])
processor.process_event(add_vertex_event)
alarm_vertex_attrs = {VProps.TYPE: VITRAGE_TYPE,
VProps.IS_DELETED: False}
before_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
event_queue = queue.Queue()
action_executor = ActionExecutor(event_queue)
# Test Action - undo
action_executor.execute(action_spec, ActionMode.UNDO)
event = event_queue.get()
processor.process_event(event)
after_alarms = processor.entity_graph.get_vertices(
vertex_attr_filter=alarm_vertex_attrs)
# Test Assertions
self.assertEqual(len(before_alarms) - 1, len(after_alarms))
def test_execute_mark_down(self):
# Test Setup
processor = self._create_processor_with_graph(self.conf)
vertex_attrs = {VProps.VITRAGE_TYPE: NOVA_HOST_DATASOURCE}
host_vertices = processor.entity_graph.get_vertices(
vertex_attr_filter=vertex_attrs)
host_vertex_before = host_vertices[0]
targets = {TFields.TARGET: host_vertex_before}
props = {}
action_spec = ActionSpecs(ActionType.MARK_DOWN, targets, props)
event_queue = queue.Queue()
action_executor = ActionExecutor(self.conf, event_queue)
# Test Action - do
action_executor.execute(action_spec, ActionMode.DO)
processor.process_event(event_queue.get())
host_vertex_after = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
self.assertTrue(host_vertex_after.get(VProps.IS_MARKED_DOWN))
# Test Action - undo
action_executor.execute(action_spec, ActionMode.UNDO)
processor.process_event(event_queue.get())
host_vertex_after_undo = processor.entity_graph.get_vertex(
host_vertex_before.vertex_id)
# Test Assertions
self.assertFalse(host_vertex_after_undo.get(VProps.IS_MARKED_DOWN))
class ScenarioEvaluator(object):
def __init__(self,
conf,
e_graph,
scenario_repo,
actions_callback,
enabled=False):
self._conf = conf
self._entity_graph = e_graph
self._db = storage.get_connection_from_config(self._conf)
self._scenario_repo = scenario_repo
self._action_executor = ActionExecutor(self._conf, actions_callback)
self._entity_graph.subscribe(self.process_event)
self.enabled = enabled
self.connected_component_cache = defaultdict(dict)
@property
def scenario_repo(self):
return self._scenario_repo
@scenario_repo.setter
def scenario_repo(self, scenario_repo):
self._scenario_repo = scenario_repo
def run_evaluator(self, action_mode=ActionMode.DO):
self.enabled = True
vertices = self._entity_graph.get_vertices()
start_time = time.time()
for vertex in vertices:
if action_mode == ActionMode.DO:
self.process_event(None, vertex, True)
elif action_mode == ActionMode.UNDO:
self.process_event(vertex, None, True)
LOG.info('Run %s Evaluator on %s items - took %s', action_mode,
len(vertices), (time.time() - start_time))
def process_event(self, before, current, is_vertex, *args, **kwargs):
"""Notification of a change in the entity graph.
:param is_vertex:
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
vitrage_is_deleted property set to True
"""
if not self.enabled:
LOG.debug("Process event disabled")
return
LOG.debug('Process event - starting')
LOG.debug("Element before event: %s, Current element: %s", before,
current)
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
if len(before_scenarios) + len(current_scenarios):
LOG.debug("Number of relevant scenarios found: undo = %s, do = %s",
len(before_scenarios), len(current_scenarios))
actions = self._process_and_get_actions(before, before_scenarios,
ActionMode.UNDO)
actions.extend(
self._process_and_get_actions(current, current_scenarios,
ActionMode.DO))
actions_to_preform = []
try:
actions_to_preform = self._analyze_and_filter_actions(actions)
except Exception:
LOG.exception("Evaluator error, will not execute actions %s",
actions)
self._action_executor.execute(actions_to_preform)
LOG.debug('Process event - completed')
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.VITRAGE_IS_DELETED) \
or element.get(EProps.VITRAGE_IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
edge_desc = self._get_edge_description(element)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
def _get_edge_description(self, element):
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return edge_desc
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = list(filter(lambda x: x in before, current))
before = list(filter(lambda x: x not in intersection, before))
current = list(filter(lambda x: x not in intersection, current))
return before, current
def _process_and_get_actions(self, element, triggered_scenarios, mode):
actions = []
for triggered_scenario in triggered_scenarios:
LOG.debug("Processing: %s", triggered_scenario)
scenario_element = triggered_scenario[0]
scenario = triggered_scenario[1]
actions.extend(
self._process_scenario(element, scenario, scenario_element,
mode))
return actions
def _process_scenario(self, element, scenario, scenario_elements, mode):
if not isinstance(scenario_elements, list):
scenario_elements = [scenario_elements]
actions = []
for action in scenario.actions:
for scenario_element in scenario_elements:
matches = self._evaluate_subgraphs(scenario.subgraphs, element,
scenario_element,
action.targets[TARGET])
actions.extend(
self._get_actions_from_matches(scenario.version, matches,
mode, action))
return actions
def _evaluate_subgraphs(self, subgraphs, element, scenario_element,
action_target):
if isinstance(element, Vertex):
return self._find_vertex_subgraph_matching(subgraphs,
action_target, element,
scenario_element)
else:
return self._find_edge_subgraph_matching(subgraphs, action_target,
element, scenario_element)
def _get_actions_from_matches(self, scenario_version, combined_matches,
mode, action_spec):
actions = []
for is_switch_mode, matches in combined_matches:
new_mode = mode
if is_switch_mode:
new_mode = ActionMode.UNDO \
if mode == ActionMode.DO else ActionMode.DO
template_schema = \
TemplateSchemaFactory().template_schema(scenario_version)
for match in matches:
match_action_spec = self._get_action_spec(action_spec, match)
items_ids = \
[match_item[1].vertex_id for match_item in match.items()]
match_hash = md5(tuple(sorted(items_ids)))
self._evaluate_property_functions(template_schema, match,
match_action_spec.properties)
actions.append(
ActionInfo(match_action_spec, new_mode,
match_action_spec.id, match_hash))
return actions
def _evaluate_property_functions(self, template_schema, match,
action_props):
"""Evaluate the action properties, in case they contain functions
In template version 2 we introduced functions, and specifically the
get_attr function. This method evaluate its value and updates the
action properties, before the action is being executed.
Example:
- action:
action_type: execute_mistral
properties:
workflow: evacuate_vm
input:
vm_name: get_attr(instance1,name)
force: false
In this example, the method will iterate over 'properties', and then
recursively over 'input', and for 'vm_name' it will replace the
call for get_attr with the actual name of the VM. The input for the
Mistral workflow will then be:
vm_name: vm_1
force: false
"""
for key, value in action_props.items():
if isinstance(value, dict):
# Recursive call for a dictionary
self._evaluate_property_functions(template_schema, match,
value)
elif value is not None and is_function(value):
# The value is a function
func_and_args = re.split('[(),]', value)
func_name = func_and_args.pop(0)
args = [arg.strip() for arg in func_and_args if len(arg) > 0]
# Get the function, execute it and update the property value
func = template_schema.functions.get(func_name)
action_props[key] = func(match, *args)
LOG.debug('Changed property %s value from %s to %s', key,
value, action_props[key])
@staticmethod
def _get_action_spec(action_spec, match):
targets = action_spec.targets
real_items = {
target: match[target_id]
for target, target_id in targets.items()
}
return ActionSpecs(action_spec.id, action_spec.type, real_items,
action_spec.properties)
@staticmethod
def _generate_action_id(action_spec):
"""Generate a unique action id for the action
BEWARE: The value created here should not be stored in database,
as in python3, the hash function seed changes after program restart
"""
targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()]
return hash(
(action_spec.type, tuple(sorted(targets)),
tuple(sorted(recursive_keypairs(action_spec.properties)))))
def _analyze_and_filter_actions(self, actions):
LOG.debug("Actions before filtering: %s", actions)
if not actions:
return []
active_actions = ActiveActionsTracker(self._conf, self._db, actions)
for action_info in actions:
if action_info.mode == ActionMode.DO:
active_actions.calc_do_action(action_info)
elif action_info.mode == ActionMode.UNDO:
active_actions.calc_undo_action(action_info)
active_actions.flush_db_updates()
unique_ordered_actions = OrderedDict()
for action in active_actions.actions_to_perform:
if isinstance(action, models.ActiveAction):
action = self._db_action_to_action_info(action)
id_ = self._generate_action_id(action.specs)
unique_ordered_actions[id_] = action
return unique_ordered_actions.values()
def _find_vertex_subgraph_matching(self, subgraphs, action_target, vertex,
scenario_vertex):
"""calculates subgraph matching for vertex
iterates over all the subgraphs, and checks if the triggered vertex is
in the same connected component as the action then run subgraph
matching on the vertex and return its result, otherwise return an
empty list of matches.
"""
matches = []
for subgraph in subgraphs:
connected_component = self.get_connected_component(
subgraph, action_target)
is_switch_mode = \
connected_component.get_vertex(scenario_vertex.vertex_id)
if is_switch_mode:
initial_map = Mapping(scenario_vertex, vertex, True)
mat = self._entity_graph.algo.sub_graph_matching(
subgraph, initial_map)
matches.append((False, mat))
else:
matches.append((True, []))
return matches
def _find_edge_subgraph_matching(self, subgraphs, action_target, edge,
scenario_edge):
"""calculates subgraph matching for edge
iterates over all the subgraphs, and checks if the triggered edge is a
negative edge then mark it as deleted=false and negative=false so that
subgraph matching on that edge will work correctly. after running
subgraph matching, we need to remove the negative vertices that were
added due to the change above.
"""
matches = []
for subgraph in subgraphs:
subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id,
scenario_edge.target.vertex_id,
scenario_edge.edge.label)
if not subgraph_edge:
continue
is_switch_mode = subgraph_edge.get(NEG_CONDITION, False)
connected_component = self.get_connected_component(
subgraph, action_target)
# change the vitrage_is_deleted and negative_condition props to
# false when is_switch_mode=true so that when we have an event on a
# negative_condition=true edge it will find the correct subgraph
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, False)
initial_map = Mapping(scenario_edge.edge, edge, False)
curr_matches = \
self._entity_graph.algo.sub_graph_matching(subgraph,
initial_map)
# switch back to the original values
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, True)
self._remove_negative_vertices_from_matches(
curr_matches, connected_component)
matches.append((is_switch_mode, curr_matches))
return matches
def get_connected_component(self, subgraph, target):
connected_component = self.connected_component_cache.get(
id(subgraph), {}).get(id(target))
if not connected_component:
connected_component = subgraph.algo.graph_query_vertices(
root_id=target, edge_query_dict={'!=': {
NEG_CONDITION: True
}})
self.connected_component_cache[id(subgraph)][id(target)] = \
connected_component
return connected_component
def _db_action_to_action_info(self, db_action):
target = self._entity_graph.get_vertex(db_action.target_vertex_id)
targets = {TARGET: target}
if db_action.source_vertex_id:
source = self._entity_graph.get_vertex(db_action.source_vertex_id)
targets[SOURCE] = source
scenario_action = self._scenario_repo.actions.get(db_action.action_id)
properties = copy.copy(scenario_action.properties)
action_specs = ActionSpecs(
id=db_action.action_id,
type=db_action.action_type,
targets=targets,
properties=properties,
)
action_info = ActionInfo(
specs=action_specs,
mode=ActionMode.DO,
action_id=db_action.action_id,
trigger_id=db_action.trigger,
)
return action_info
@staticmethod
def _switch_edge_negative_props(is_switch_mode, scenario_edge, subgraph,
status):
if is_switch_mode:
scenario_edge.edge[NEG_CONDITION] = status
scenario_edge.edge[EProps.VITRAGE_IS_DELETED] = status
subgraph.update_edge(scenario_edge.edge)
@staticmethod
def _remove_negative_vertices_from_matches(matches, connected_component):
for match in matches:
ver_ids = [v.vertex_id for v in connected_component.get_vertices()]
ver_to_remove = [id for id in match.keys() if id not in ver_ids]
for v_id in ver_to_remove:
del match[v_id]
class ScenarioEvaluator(object):
def __init__(self,
conf,
entity_graph,
scenario_repo,
event_queue,
enabled=False):
self.conf = conf
self._scenario_repo = scenario_repo
self._entity_graph = entity_graph
self._action_executor = ActionExecutor(event_queue)
self._entity_graph.subscribe(self.process_event)
self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf))
self.enabled = enabled
self.connected_component_cache = defaultdict(dict)
@property
def scenario_repo(self):
return self._scenario_repo
@scenario_repo.setter
def scenario_repo(self, scenario_repo):
self._scenario_repo = scenario_repo
def process_event(self, before, current, is_vertex, *args, **kwargs):
"""Notification of a change in the entity graph.
:param is_vertex:
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
is_deleted property set to True
"""
if not self.enabled:
LOG.debug("Process event disabled")
return
LOG.debug('Process event - starting')
LOG.debug("Element before event: %s, Current element: %s",
str(before),
str(current))
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
if len(before_scenarios) + len(current_scenarios):
LOG.debug("Number of relevant scenarios found: undo = %s, do = %s",
str(len(before_scenarios)),
str(len(current_scenarios)))
actions = self._process_and_get_actions(before,
before_scenarios,
ActionMode.UNDO)
actions.extend(self._process_and_get_actions(current,
current_scenarios,
ActionMode.DO))
if actions:
LOG.debug("Actions to perform: %s", actions)
filtered_actions = \
self._analyze_and_filter_actions(actions)
LOG.debug("Actions filtered: %s", filtered_actions)
for action in filtered_actions:
self._action_executor.execute(action.specs, action.mode)
LOG.debug('Process event - completed')
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.IS_DELETED) \
or element.get(EProps.IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
edge_desc = self._get_edge_description(element)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
def _get_edge_description(self, element):
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return edge_desc
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = list(filter(lambda x: x in before, current))
before = list(filter(lambda x: x not in intersection, before))
current = list(filter(lambda x: x not in intersection, current))
return before, current
def _process_and_get_actions(self, element, triggered_scenarios, mode):
actions = []
for triggered_scenario in triggered_scenarios:
LOG.debug("Processing: %s", str(triggered_scenario))
scenario_element = triggered_scenario[0]
scenario = triggered_scenario[1]
actions.extend(self._process_scenario(element,
scenario,
scenario_element,
mode))
return actions
def _process_scenario(self, element, scenario, scenario_elements, mode):
if not isinstance(scenario_elements, list):
scenario_elements = [scenario_elements]
actions = []
for action in scenario.actions:
for scenario_element in scenario_elements:
matches = self._evaluate_subgraphs(scenario.subgraphs,
element,
scenario_element,
action.targets['target'])
actions.extend(self._get_actions_from_matches(matches,
mode,
action,
scenario))
return actions
def _evaluate_subgraphs(self,
subgraphs,
element,
scenario_element,
action_target):
if isinstance(element, Vertex):
return self._find_vertex_subgraph_matching(subgraphs,
action_target,
element,
scenario_element)
else:
return self._find_edge_subgraph_matching(subgraphs,
action_target,
element,
scenario_element)
def _get_actions_from_matches(self,
combined_matches,
mode,
action_spec,
scenario):
actions = []
for is_switch_mode, matches in combined_matches:
new_mode = mode
if is_switch_mode:
new_mode = ActionMode.UNDO \
if mode == ActionMode.DO else ActionMode.DO
for match in matches:
spec = self._get_action_spec(action_spec, match)
items_ids = [match[1].vertex_id for match in match.items()]
match_hash = hash(tuple(sorted(items_ids)))
actions.append(ActionInfo(spec, new_mode,
scenario.id, match_hash))
return actions
@staticmethod
def _get_action_spec(action_spec, match):
targets = action_spec.targets
real_items = {
target: match[target_id] for target, target_id in targets.items()
}
return ActionSpecs(action_spec.type,
real_items,
action_spec.properties)
@staticmethod
def _generate_action_id(action_spec):
targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()]
return hash(
(action_spec.type,
tuple(sorted(targets)),
tuple(sorted(action_spec.properties.items())))
)
def _analyze_and_filter_actions(self, actions):
actions_to_perform = {}
for action in actions:
key = self._action_tracker.get_key(action.specs)
prev_dominant = self._action_tracker.get_dominant_action(key)
if action.mode == ActionMode.DO:
self._action_tracker.insert_action(key, action)
else:
self._action_tracker.remove_action(key, action)
new_dominant = self._action_tracker.get_dominant_action(key)
# todo: (erosensw) improvement - first analyze DOs, then UNDOs
if not new_dominant: # removed last entry for key
undo_action = ActionInfo(prev_dominant.specs,
ActionMode.UNDO,
prev_dominant.scenario_id,
prev_dominant.trigger_id)
actions_to_perform[key] = undo_action
elif new_dominant != prev_dominant:
actions_to_perform[key] = new_dominant
# filter the same action
final_actions = {ScenarioEvaluator._generate_action_id(action.specs):
action for action in actions_to_perform.values()}
return final_actions.values()
def _find_vertex_subgraph_matching(self,
subgraphs,
action_target,
vertex,
scenario_vertex):
"""calculates subgraph matching for vertex
iterates over all the subgraphs, and checks if the triggered vertex is
in the same connected component as the action then run subgraph
matching on the vertex and return its result, otherwise return an
empty list of matches.
"""
matches = []
for subgraph in subgraphs:
connected_component = self.get_connected_component(subgraph,
action_target)
is_switch_mode = \
connected_component.get_vertex(scenario_vertex.vertex_id)
if is_switch_mode:
initial_map = Mapping(scenario_vertex, vertex, True)
mat = self._entity_graph.algo.sub_graph_matching(subgraph,
initial_map)
matches.append((False, mat))
else:
matches.append((True, []))
return matches
def _find_edge_subgraph_matching(self,
subgraphs,
action_target,
edge,
scenario_edge):
"""calculates subgraph matching for edge
iterates over all the subgraphs, and checks if the triggered edge is a
negative edge then mark it as deleted=false and negative=false so that
subgraph matching on that edge will work correctly. after running
subgraph matching, we need to remove the negative vertices that were
added due to the change above.
"""
matches = []
for subgraph in subgraphs:
subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id,
scenario_edge.target.vertex_id,
scenario_edge.edge.label)
if not subgraph_edge:
continue
is_switch_mode = subgraph_edge.get(NEG_CONDITION, False)
connected_component = self.get_connected_component(subgraph,
action_target)
# change the is_deleted and negative_condition props to false when
# is_switch_mode=true so that when we have an event on a
# negative_condition=true edge it will find the correct subgraph
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, False)
initial_map = Mapping(scenario_edge.edge, edge, False)
curr_matches = \
self._entity_graph.algo.sub_graph_matching(subgraph,
initial_map)
# switch back to the original values
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, True)
self._remove_negative_vertices_from_matches(curr_matches,
connected_component)
matches.append((is_switch_mode, curr_matches))
return matches
def get_connected_component(self, subgraph, target):
connected_component = self.connected_component_cache.get(
id(subgraph), {}).get(id(target))
if not connected_component:
connected_component = subgraph.algo.graph_query_vertices(
root_id=target,
edge_query_dict={'!=': {NEG_CONDITION: True}})
self.connected_component_cache[id(subgraph)][id(target)] = \
connected_component
return connected_component
@staticmethod
def _switch_edge_negative_props(is_switch_mode,
scenario_edge,
subgraph,
status):
if is_switch_mode:
scenario_edge.edge[NEG_CONDITION] = status
scenario_edge.edge[EProps.IS_DELETED] = status
subgraph.update_edge(scenario_edge.edge)
@staticmethod
def _remove_negative_vertices_from_matches(matches, connected_component):
for match in matches:
ver_ids = [v.vertex_id for v in connected_component.get_vertices()]
ver_to_remove = [id for id in match.keys() if id not in ver_ids]
for v_id in ver_to_remove:
del match[v_id]
class ScenarioEvaluator(object):
def __init__(self,
conf,
entity_graph,
scenario_repo,
event_queue,
enabled=False):
self.conf = conf
self._entity_graph = entity_graph
self._graph_algs = create_algorithm(entity_graph)
self._scenario_repo = scenario_repo
self._action_executor = ActionExecutor(event_queue)
self._entity_graph.subscribe(self.process_event)
self.enabled = enabled
def process_event(self, before, current, is_vertex):
"""Notification of a change in the entity graph.
:param is_vertex:
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
is_deleted property set to True
"""
if not self.enabled:
LOG.debug("Process event disabled")
return
LOG.debug('Process event - starting')
LOG.debug("Element before event: %s, Current element: %s",
str(before),
str(current))
# todo (erosensw): support for NOT conditions - reverse logic
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
if len(before_scenarios) + len(current_scenarios):
LOG.debug("Number of relevant scenarios found: undo = %s, do = %s",
str(len(before_scenarios)),
str(len(current_scenarios)))
actions = self._process_and_get_actions(before,
before_scenarios,
ActionMode.UNDO)
actions.update(self._process_and_get_actions(current,
current_scenarios,
ActionMode.DO))
if actions:
LOG.debug("Actions to perform: %s", actions.values())
for action in actions.values():
action_spec = action[0]
action_mode = action[1]
self._action_executor.execute(action_spec, action_mode)
LOG.debug('Process event - completed')
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.IS_DELETED) \
or element.get(EProps.IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
edge_desc = self._get_edge_description(element)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
def _get_edge_description(self, element):
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return edge_desc
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = list(filter(lambda x: x in before, current))
before = list(filter(lambda x: x not in intersection, before))
current = list(filter(lambda x: x not in intersection, current))
return before, current
def _process_and_get_actions(self, element, triggered_scenarios, mode):
actions = {}
for triggered_scenario in triggered_scenarios:
LOG.debug("Processing: %s", str(triggered_scenario))
scenario_element = triggered_scenario[0]
scenario = triggered_scenario[1]
actions.update(self._process_scenario(element,
scenario,
scenario_element,
mode))
return actions
def _process_scenario(self, element, scenario, scenario_elements, mode):
if not isinstance(scenario_elements, list):
scenario_elements = [scenario_elements]
actions = {}
for action in scenario.actions:
for scenario_element in scenario_elements:
matches = self._evaluate_full_condition(scenario.condition,
element,
scenario_element)
if matches:
for match in matches:
spec, action_id = self._get_action_spec(action, match)
actions[action_id] = (spec, mode)
return actions
@staticmethod
def _get_action_spec(action_spec, match):
targets = action_spec.targets
real_items = {
target: match[target_id] for target, target_id in targets.items()
}
revised_spec = ActionSpecs(action_spec.type,
real_items,
action_spec.properties)
action_id = ScenarioEvaluator._generate_action_id(revised_spec)
return revised_spec, action_id
@staticmethod
def _generate_action_id(action_spec):
targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()]
return hash(
(action_spec.type,
tuple(sorted(targets)),
tuple(sorted(action_spec.properties.items())))
)
def _evaluate_full_condition(self, condition, element, scenario_element):
condition_matches = []
for clause in condition:
# OR condition means aggregation of matches, without duplicates
and_condition_matches = \
self._evaluate_and_condition(clause, element, scenario_element)
condition_matches += and_condition_matches
return condition_matches
def _evaluate_and_condition(self, condition, element, scenario_element):
condition_g = create_graph("scenario condition")
for term in condition:
if not term.positive:
# todo(erosensw): add support for NOT clauses
LOG.error('Unsupported template with NOT operator')
return []
if term.type == ENTITY:
term.variable[VProps.IS_DELETED] = False
condition_g.add_vertex(term.variable)
else: # type = relationship
edge_desc = term.variable
self._set_relationship_not_deleted(edge_desc)
self._add_relationship(condition_g, edge_desc)
if isinstance(element, Vertex):
initial_map = Mapping(scenario_element, element, True)
else:
initial_map = Mapping(scenario_element.edge, element, False)
return self._graph_algs.sub_graph_matching(condition_g, [initial_map])
@staticmethod
def _set_relationship_not_deleted(edge_description):
edge_description.source[VProps.IS_DELETED] = False
edge_description.target[VProps.IS_DELETED] = False
edge_description.edge[EProps.IS_DELETED] = False
@staticmethod
def _add_relationship(condition_graph, edge_description):
condition_graph.add_vertex(edge_description.source)
condition_graph.add_vertex(edge_description.target)
condition_graph.add_edge(edge_description.edge)
class ScenarioEvaluator(EvaluatorBase):
def __init__(self,
conf,
e_graph,
scenario_repo,
actions_callback,
enabled=False):
super(ScenarioEvaluator, self).__init__(conf, e_graph)
self._db_connection = storage.get_connection_from_config(self._conf)
self._scenario_repo = scenario_repo
self._action_executor = ActionExecutor(self._conf, actions_callback)
self._entity_graph.subscribe(self.process_event)
self._active_actions_tracker = ActiveActionsTracker(
self._conf, self._db_connection)
self.enabled = enabled
self.connected_component_cache = defaultdict(dict)
@property
def scenario_repo(self):
return self._scenario_repo
@scenario_repo.setter
def scenario_repo(self, scenario_repo):
self._scenario_repo = scenario_repo
def run_evaluator(self):
self.enabled = True
vertices = self._entity_graph.get_vertices()
start_time = time.time()
for vertex in vertices:
self.process_event(None, vertex, True)
LOG.info('Run Evaluator on %s items - took %s', str(len(vertices)),
str(time.time() - start_time))
def process_event(self, before, current, is_vertex, *args, **kwargs):
"""Notification of a change in the entity graph.
:param is_vertex:
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
vitrage_is_deleted property set to True
"""
if not self.enabled:
LOG.debug("Process event disabled")
return
LOG.debug('Process event - starting')
LOG.debug("Element before event: %s, Current element: %s",
str(before),
str(current))
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
if len(before_scenarios) + len(current_scenarios):
LOG.debug("Number of relevant scenarios found: undo = %s, do = %s",
str(len(before_scenarios)),
str(len(current_scenarios)))
actions = self._process_and_get_actions(before,
before_scenarios,
ActionMode.UNDO)
actions.extend(self._process_and_get_actions(current,
current_scenarios,
ActionMode.DO))
actions_to_preform = []
try:
actions_to_preform = self._analyze_and_filter_actions(actions)
except Exception as e:
LOG.error("Evaluator error, will not execute actions %s",
str(actions))
LOG.exception("Caught: %s", e)
for action in actions_to_preform:
LOG.info('Action: %s', self._action_str(action))
self._action_executor.execute(action.specs, action.mode)
LOG.debug('Process event - completed')
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.VITRAGE_IS_DELETED) \
or element.get(EProps.VITRAGE_IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
edge_desc = self._get_edge_description(element)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
def _get_edge_description(self, element):
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return edge_desc
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = list(filter(lambda x: x in before, current))
before = list(filter(lambda x: x not in intersection, before))
current = list(filter(lambda x: x not in intersection, current))
return before, current
def _process_and_get_actions(self, element, triggered_scenarios, mode):
actions = []
for triggered_scenario in triggered_scenarios:
LOG.debug("Processing: %s", str(triggered_scenario))
scenario_element = triggered_scenario[0]
scenario = triggered_scenario[1]
actions.extend(self._process_scenario(element,
scenario,
scenario_element,
mode))
return actions
def _process_scenario(self, element, scenario, scenario_elements, mode):
if not isinstance(scenario_elements, list):
scenario_elements = [scenario_elements]
actions = []
for action in scenario.actions:
for scenario_element in scenario_elements:
matches = self._evaluate_subgraphs(scenario.subgraphs,
element,
scenario_element,
action.targets[TARGET])
actions.extend(self._get_actions_from_matches(matches,
mode,
action))
return actions
def _evaluate_subgraphs(self,
subgraphs,
element,
scenario_element,
action_target):
if isinstance(element, Vertex):
return self._find_vertex_subgraph_matching(subgraphs,
action_target,
element,
scenario_element)
else:
return self._find_edge_subgraph_matching(subgraphs,
action_target,
element,
scenario_element)
def _get_actions_from_matches(self,
combined_matches,
mode,
action_spec):
actions = []
for is_switch_mode, matches in combined_matches:
new_mode = mode
if is_switch_mode:
new_mode = ActionMode.UNDO \
if mode == ActionMode.DO else ActionMode.DO
for match in matches:
match_action_spec = self._get_action_spec(action_spec, match)
items_ids = [match[1].vertex_id for match in match.items()]
match_hash = hash(tuple(sorted(items_ids)))
actions.append(ActionInfo(match_action_spec, new_mode,
match_action_spec.id, match_hash))
return actions
@staticmethod
def _get_action_spec(action_spec, match):
targets = action_spec.targets
real_items = {
target: match[target_id] for target, target_id in targets.items()
}
return ActionSpecs(action_spec.id,
action_spec.type,
real_items,
action_spec.properties)
@staticmethod
def _generate_action_id(action_spec):
targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()]
return hash(
(action_spec.type,
tuple(sorted(targets)),
tuple(sorted(action_spec.properties.items())))
)
def _analyze_and_filter_actions(self, actions):
LOG.debug("Actions before filtering: %s", actions)
actions_to_perform = []
for action_info in actions:
if action_info.mode == ActionMode.DO:
is_highest_score, exists = \
self._active_actions_tracker.calc_do_action(action_info)
if is_highest_score and not exists:
actions_to_perform.append(action_info)
elif action_info.mode == ActionMode.UNDO:
is_highest_score, second_highest = \
self._active_actions_tracker.calc_undo_action(action_info)
if is_highest_score:
# We should 'DO' the Second highest scored action so
# to override the existing dominant action.
# or, if there is no second highest scored action
# So we just 'UNDO' the existing dominant action
if second_highest:
action_to_perform = self._db_action_to_action_info(
second_highest)
actions_to_perform.append(action_to_perform)
else:
actions_to_perform.append(action_info)
unique_ordered_actions = OrderedDict()
for action in actions_to_perform:
id_ = ScenarioEvaluator._generate_action_id(action.specs)
unique_ordered_actions[id_] = action
return unique_ordered_actions.values()
def _find_vertex_subgraph_matching(self,
subgraphs,
action_target,
vertex,
scenario_vertex):
"""calculates subgraph matching for vertex
iterates over all the subgraphs, and checks if the triggered vertex is
in the same connected component as the action then run subgraph
matching on the vertex and return its result, otherwise return an
empty list of matches.
"""
matches = []
for subgraph in subgraphs:
connected_component = self.get_connected_component(subgraph,
action_target)
is_switch_mode = \
connected_component.get_vertex(scenario_vertex.vertex_id)
if is_switch_mode:
initial_map = Mapping(scenario_vertex, vertex, True)
mat = self._entity_graph.algo.sub_graph_matching(subgraph,
initial_map)
matches.append((False, mat))
else:
matches.append((True, []))
return matches
def _find_edge_subgraph_matching(self,
subgraphs,
action_target,
edge,
scenario_edge):
"""calculates subgraph matching for edge
iterates over all the subgraphs, and checks if the triggered edge is a
negative edge then mark it as deleted=false and negative=false so that
subgraph matching on that edge will work correctly. after running
subgraph matching, we need to remove the negative vertices that were
added due to the change above.
"""
matches = []
for subgraph in subgraphs:
subgraph_edge = subgraph.get_edge(scenario_edge.source.vertex_id,
scenario_edge.target.vertex_id,
scenario_edge.edge.label)
if not subgraph_edge:
continue
is_switch_mode = subgraph_edge.get(NEG_CONDITION, False)
connected_component = self.get_connected_component(subgraph,
action_target)
# change the vitrage_is_deleted and negative_condition props to
# false when is_switch_mode=true so that when we have an event on a
# negative_condition=true edge it will find the correct subgraph
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, False)
initial_map = Mapping(scenario_edge.edge, edge, False)
curr_matches = \
self._entity_graph.algo.sub_graph_matching(subgraph,
initial_map)
# switch back to the original values
self._switch_edge_negative_props(is_switch_mode, scenario_edge,
subgraph, True)
self._remove_negative_vertices_from_matches(curr_matches,
connected_component)
matches.append((is_switch_mode, curr_matches))
return matches
def get_connected_component(self, subgraph, target):
connected_component = self.connected_component_cache.get(
id(subgraph), {}).get(id(target))
if not connected_component:
connected_component = subgraph.algo.graph_query_vertices(
root_id=target,
edge_query_dict={'!=': {NEG_CONDITION: True}})
self.connected_component_cache[id(subgraph)][id(target)] = \
connected_component
return connected_component
def _db_action_to_action_info(self, db_action):
target = self._entity_graph.get_vertex(db_action.target_vertex_id)
targets = {TARGET: target}
if db_action.source_vertex_id:
source = self._entity_graph.get_vertex(db_action.source_vertex_id)
targets[SOURCE] = source
scenario_action = self._scenario_repo.actions.get(db_action.action_id)
properties = copy.copy(scenario_action.properties)
action_specs = ActionSpecs(
id=db_action.action_id,
type=db_action.action_type,
targets=targets,
properties=properties,
)
action_info = ActionInfo(
specs=action_specs,
mode=ActionMode.DO,
action_id=db_action.action_id,
trigger_id=db_action.trigger,
)
return action_info
@staticmethod
def _switch_edge_negative_props(is_switch_mode,
scenario_edge,
subgraph,
status):
if is_switch_mode:
scenario_edge.edge[NEG_CONDITION] = status
scenario_edge.edge[EProps.VITRAGE_IS_DELETED] = status
subgraph.update_edge(scenario_edge.edge)
@staticmethod
def _remove_negative_vertices_from_matches(matches, connected_component):
for match in matches:
ver_ids = [v.vertex_id for v in connected_component.get_vertices()]
ver_to_remove = [id for id in match.keys() if id not in ver_ids]
for v_id in ver_to_remove:
del match[v_id]
@staticmethod
def _action_str(action):
s = action.specs.targets.get(SOURCE, {}).get(VProps.VITRAGE_ID, '')
t = action.specs.targets.get(TARGET, {}).get(VProps.VITRAGE_ID, '')
return '%s %s \'%s\' targets (%s,%s)' % (action.mode.upper(),
action.specs.type,
action.action_id, s, t)
class ScenarioEvaluator(object):
def __init__(self,
conf,
entity_graph,
scenario_repo,
event_queue,
enabled=False):
self.conf = conf
self._scenario_repo = scenario_repo
self._entity_graph = entity_graph
self._graph_algs = create_algorithm(entity_graph)
self._action_executor = ActionExecutor(event_queue)
self._entity_graph.subscribe(self.process_event)
self._action_tracker = ActionTracker(DatasourceInfoMapper(self.conf))
self.enabled = enabled
@property
def scenario_repo(self):
return self._scenario_repo
@scenario_repo.setter
def scenario_repo(self, scenario_repo):
self._scenario_repo = scenario_repo
def process_event(self, before, current, is_vertex, *args, **kwargs):
"""Notification of a change in the entity graph.
:param is_vertex:
:param before: The graph element (vertex or edge) prior to the
change that happened. None if the element was just created.
:param current: The graph element (vertex or edge) after the
change that happened. Deleted elements should arrive with the
is_deleted property set to True
"""
if not self.enabled:
LOG.debug("Process event disabled")
return
LOG.debug('Process event - starting')
LOG.debug("Element before event: %s, Current element: %s",
str(before),
str(current))
# todo (erosensw): support for NOT conditions - reverse logic
before_scenarios = self._get_element_scenarios(before, is_vertex)
current_scenarios = self._get_element_scenarios(current, is_vertex)
before_scenarios, current_scenarios = \
self._remove_overlap_scenarios(before_scenarios, current_scenarios)
if len(before_scenarios) + len(current_scenarios):
LOG.debug("Number of relevant scenarios found: undo = %s, do = %s",
str(len(before_scenarios)),
str(len(current_scenarios)))
actions = self._process_and_get_actions(before,
before_scenarios,
ActionMode.UNDO)
actions.update(self._process_and_get_actions(current,
current_scenarios,
ActionMode.DO))
if actions:
LOG.debug("Actions to perform: %s", actions.values())
filtered_actions = \
self._analyze_and_filter_actions(actions.values())
LOG.debug("Actions filtered: %s", filtered_actions)
for action in filtered_actions:
self._action_executor.execute(action.specs, action.mode)
LOG.debug('Process event - completed')
def _get_element_scenarios(self, element, is_vertex):
if not element \
or element.get(VProps.IS_DELETED) \
or element.get(EProps.IS_DELETED):
return []
elif is_vertex:
return self._scenario_repo.get_scenarios_by_vertex(element)
else: # is edge
edge_desc = self._get_edge_description(element)
return self._scenario_repo.get_scenarios_by_edge(edge_desc)
def _get_edge_description(self, element):
source = self._entity_graph.get_vertex(element.source_id)
target = self._entity_graph.get_vertex(element.target_id)
edge_desc = EdgeDescription(element, source, target)
return edge_desc
@staticmethod
def _remove_overlap_scenarios(before, current):
intersection = list(filter(lambda x: x in before, current))
before = list(filter(lambda x: x not in intersection, before))
current = list(filter(lambda x: x not in intersection, current))
return before, current
def _process_and_get_actions(self, element, triggered_scenarios, mode):
actions = {}
for triggered_scenario in triggered_scenarios:
LOG.debug("Processing: %s", str(triggered_scenario))
scenario_element = triggered_scenario[0]
scenario = triggered_scenario[1]
actions.update(self._process_scenario(element,
scenario,
scenario_element,
mode))
return actions
def _process_scenario(self, element, scenario, scenario_elements, mode):
if not isinstance(scenario_elements, list):
scenario_elements = [scenario_elements]
actions = {}
for action in scenario.actions:
for scenario_element in scenario_elements:
matches = self._evaluate_full_condition(scenario.condition,
element,
scenario_element)
if matches:
for match in matches:
spec, action_id = self._get_action_spec(action, match)
match_hash = hash(tuple(sorted(match.items())))
actions[action_id] = \
ActionInfo(spec, mode, scenario.id, match_hash)
return actions
@staticmethod
def _get_action_spec(action_spec, match):
targets = action_spec.targets
real_items = {
target: match[target_id] for target, target_id in targets.items()
}
revised_spec = ActionSpecs(action_spec.type,
real_items,
action_spec.properties)
# noinspection PyTypeChecker
action_id = ScenarioEvaluator._generate_action_id(revised_spec)
return revised_spec, action_id
@staticmethod
def _generate_action_id(action_spec):
targets = [(k, v.vertex_id) for k, v in action_spec.targets.items()]
return hash(
(action_spec.type,
tuple(sorted(targets)),
tuple(sorted(action_spec.properties.items())))
)
def _evaluate_full_condition(self, condition, element, scenario_element):
condition_matches = []
for clause in condition:
# OR condition means aggregation of matches, without duplicates
and_condition_matches = \
self._evaluate_and_condition(clause, element, scenario_element)
condition_matches += and_condition_matches
return condition_matches
def _evaluate_and_condition(self, condition, element, scenario_element):
condition_g = create_graph("scenario condition")
for term in condition:
if not term.positive:
# todo(erosensw): add support for NOT clauses
LOG.error('Template with NOT operator current not supported')
return []
if term.type == ENTITY:
term.variable[VProps.IS_DELETED] = False
condition_g.add_vertex(term.variable)
else: # type = relationship
edge_desc = term.variable
self._set_relationship_not_deleted(edge_desc)
self._add_relationship(condition_g, edge_desc)
if isinstance(element, Vertex):
initial_map = Mapping(scenario_element, element, True)
else:
initial_map = Mapping(scenario_element.edge, element, False)
return self._graph_algs.sub_graph_matching(condition_g, [initial_map])
@staticmethod
def _set_relationship_not_deleted(edge_description):
edge_description.source[VProps.IS_DELETED] = False
edge_description.target[VProps.IS_DELETED] = False
edge_description.edge[EProps.IS_DELETED] = False
@staticmethod
def _add_relationship(condition_graph, edge_description):
condition_graph.add_vertex(edge_description.source)
condition_graph.add_vertex(edge_description.target)
condition_graph.add_edge(edge_description.edge)
def _analyze_and_filter_actions(self, actions):
actions_to_perform = {}
for action in actions:
key = self._action_tracker.get_key(action.specs)
prev_dominant = self._action_tracker.get_dominant_action(key)
if action.mode == ActionMode.DO:
self._action_tracker.insert_action(key, action)
else:
self._action_tracker.remove_action(key, action)
new_dominant = self._action_tracker.get_dominant_action(key)
# todo: (erosensw) improvement - first analyze DOs, then UNDOs
if not new_dominant: # removed last entry for key
undo_action = ActionInfo(prev_dominant.specs,
ActionMode.UNDO,
prev_dominant.scenario_id,
prev_dominant.trigger_id)
actions_to_perform[key] = undo_action
elif new_dominant != prev_dominant:
actions_to_perform[key] = new_dominant
return actions_to_perform.values()