def initializing_process(self):
try:
LOG.info('Consistency Initializing Process - Started')
if not self._wait_for_action(
self.initialization_status.is_received_all_end_messages):
LOG.error('Maximum retries for consistency initializator '
'were done')
LOG.info('All end messages were received')
self.evaluator.enabled = True
timestamp = str(utcnow())
all_vertices = self.graph.get_vertices()
self._run_evaluator(all_vertices)
self._wait_for_processing_evaluator_events()
self._mark_old_deduced_alarms_as_deleted(timestamp)
self.initialization_status.status = \
self.initialization_status.FINISHED
LOG.info('Consistency Initializing Process - Finished')
except Exception as e:
LOG.exception('Error in deleting vertices from entity_graph: %s',
e)
def initializing_process(self):
try:
LOG.info('Consistency Initializing Process - Started')
if not self._wait_for_action(
self.initialization_status.is_received_all_end_messages):
LOG.error('Maximum retries for consistency initializator '
'were done')
LOG.info('All end messages were received')
self.evaluator.enabled = True
timestamp = str(utcnow())
all_vertices = self.graph.get_vertices()
self._run_evaluator(all_vertices)
self._wait_for_processing_evaluator_events()
self._mark_old_deduced_alarms_as_deleted(timestamp)
self.initialization_status.status = \
self.initialization_status.FINISHED
LOG.info('Consistency Initializing Process - Finished')
except Exception as e:
LOG.exception('Error in deleting vertices from entity_graph: %s',
e)
def _periodic_process_setup_stage(self, consistency_interval):
self._create_processor_with_graph(self.conf, processor=self.processor)
current_time = utcnow()
# set all vertices to be have timestamp that consistency won't get
self._update_timestamp(
self.processor.entity_graph.get_vertices(),
current_time + timedelta(seconds=1.5 * consistency_interval))
# check number of instances in graph
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY:
EntityCategory.RESOURCE,
VProps.TYPE:
NOVA_INSTANCE_DATASOURCE
})
self.assertEqual(self.NUM_INSTANCES, len(instance_vertices))
# set current timestamp of part of the instances
self._update_timestamp(instance_vertices[0:3], current_time)
# set part of the instances as deleted
for i in range(3, 6):
instance_vertices[i][VProps.IS_DELETED] = True
self.processor.entity_graph.update_vertex(instance_vertices[i])
# set part of the instances as deleted
for i in range(6, 9):
instance_vertices[i][VProps.IS_DELETED] = True
instance_vertices[i][VProps.SAMPLE_TIMESTAMP] = str(
current_time + timedelta(seconds=2 * consistency_interval + 1))
self.processor.entity_graph.update_vertex(instance_vertices[i])
def _periodic_process_setup_stage(self, consistency_interval):
self._create_processor_with_graph(self.conf, processor=self.processor)
current_time = utcnow()
# set all vertices to be have timestamp that consistency won't get
self._update_timestamp(self.processor.entity_graph.get_vertices(),
current_time +
timedelta(seconds=1.5 * consistency_interval))
# check number of instances in graph
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.CATEGORY: EntityCategory.RESOURCE,
VProps.TYPE: NOVA_INSTANCE_DATASOURCE
})
self.assertEqual(self.NUM_INSTANCES, len(instance_vertices))
# set current timestamp of part of the instances
self._update_timestamp(instance_vertices[0:3], current_time)
# set part of the instances as deleted
for i in range(3, 6):
instance_vertices[i][VProps.IS_DELETED] = True
self.processor.entity_graph.update_vertex(instance_vertices[i])
# set part of the instances as deleted
for i in range(6, 9):
instance_vertices[i][VProps.IS_DELETED] = True
instance_vertices[i][VProps.SAMPLE_TIMESTAMP] = str(
current_time + timedelta(seconds=2 * consistency_interval + 1))
self.processor.entity_graph.update_vertex(instance_vertices[i])
def _push_events_to_queue(self, vertices, action):
for vertex in vertices:
event = {
DSProps.SYNC_TYPE: CONSISTENCY_DATASOURCE,
DSProps.SYNC_MODE: SyncMode.UPDATE,
DSProps.SAMPLE_DATE: str(utcnow()),
DSProps.EVENT_TYPE: action,
VProps.VITRAGE_ID: vertex[VProps.VITRAGE_ID]
}
self.evaluator_queue.put(event)
def _push_events_to_queue(self, vertices, action):
for vertex in vertices:
event = {
DSProps.SYNC_TYPE: CONSISTENCY_DATASOURCE,
DSProps.SYNC_MODE: SyncMode.UPDATE,
DSProps.SAMPLE_DATE: str(utcnow()),
DSProps.EVENT_TYPE: action,
VProps.VITRAGE_ID: vertex[VProps.VITRAGE_ID]
}
self.evaluator_queue.put(event)
def _create_alarm(vitrage_id, alarm_type):
return graph_utils.create_vertex(
vitrage_id,
entity_id=vitrage_id,
entity_category=EntityCategory.ALARM,
entity_type=alarm_type,
entity_state='active',
is_deleted=False,
sample_timestamp=utcnow(),
is_placeholder=False,
)
def _create_alarm(vitrage_id, alarm_type):
return graph_utils.create_vertex(
vitrage_id,
entity_id=vitrage_id,
entity_category=EntityCategory.ALARM,
entity_type=alarm_type,
entity_state='active',
is_deleted=False,
sample_timestamp=utcnow(),
is_placeholder=False,
)
def _find_stale_entities(self):
query = {
'and': [
{'!=': {VProps.TYPE: VITRAGE_TYPE}},
{'<': {VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta(
seconds=2 * self.conf.datasources.snapshots_interval))}}
]
}
vertices = self.graph.get_vertices(query_dict=query)
return set(self._filter_vertices_to_be_deleted(vertices))
def _find_old_deleted_entities(self):
query = {
'and': [
{'==': {VProps.IS_DELETED: True}},
{'<': {VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta(
seconds=self.conf.consistency.min_time_to_delete))}}
]
}
vertices = self.graph.get_vertices(query_dict=query)
return self._filter_vertices_to_be_deleted(vertices)
def _find_old_deleted_entities(self):
query = {
'and': [{
'==': {
VProps.IS_DELETED: True
}
}, {
'<': {
VProps.SAMPLE_TIMESTAMP:
str(utcnow() - timedelta(
seconds=self.conf.consistency.min_time_to_delete))
}
}]
}
vertices = self.graph.get_vertices(query_dict=query)
return self._filter_vertices_to_be_deleted(vertices)
def _find_stale_entities(self):
query = {
'and': [{
'!=': {
VProps.TYPE: VITRAGE_TYPE
}
}, {
'<': {
VProps.SAMPLE_TIMESTAMP:
str(utcnow() - timedelta(
seconds=2 * self.conf.datasources.snapshots_interval))
}
}]
}
vertices = self.graph.get_vertices(query_dict=query)
return set(self._filter_vertices_to_be_deleted(vertices))
def test_update_entity_state(self):
# create instance event with host neighbor and check validity
(vertex, neighbors, processor) =\
self._create_and_check_entity(status='STARTING')
# check added entity
vertex = processor.entity_graph.get_vertex(vertex.vertex_id)
self.assertEqual('STARTING', vertex.properties[VProps.STATE])
# update instance event with state running
vertex.properties[VProps.STATE] = 'RUNNING'
vertex.properties[VProps.SAMPLE_TIMESTAMP] = str(utcnow())
processor.update_entity(vertex, neighbors)
# check state
self._check_graph(processor, self.NUM_VERTICES_AFTER_CREATION,
self.NUM_EDGES_AFTER_CREATION)
vertex = processor.entity_graph.get_vertex(vertex.vertex_id)
self.assertEqual('RUNNING', vertex.properties[VProps.STATE])
def add_template(self, template_def):
current_time = datetime_utils.utcnow()
result = syntax_validation(template_def)
if not result.is_valid:
LOG.info('Unable to load template: %s', result.comment)
else:
result = content_validation(template_def)
if not result.is_valid:
LOG.info('Unable to load template: %s', result.comment)
template_uuid = uuid.uuid4()
self.templates[str(template_uuid)] = Template(template_uuid,
template_def,
current_time, result)
if result.is_valid:
template_data = TemplateData(template_def)
self._add_template_scenarios(template_data)
def _filter_and_cache_alarms(self, alarms, filter_):
alarms_to_update = []
now = datetime_utils.utcnow(False)
for alarm in alarms:
alarm_key = self._alarm_key(alarm)
old_alarm, timestamp = self.cache.get(alarm_key, (None, None))
if filter_(self, alarm, old_alarm):
alarms_to_update.append(alarm)
self.cache[alarm_key] = alarm, now
# add alarms that were deleted
for cached_alarm, timestamp in self.cache.values():
if self._is_erroneous(cached_alarm) and timestamp is not now:
LOG.debug('deleting cached_alarm %s', cached_alarm)
cached_alarm[DSProps.EVENT_TYPE] = EventAction.DELETE_ENTITY
alarms_to_update.append(cached_alarm)
self.cache.pop(self._alarm_key(cached_alarm))
return alarms_to_update
def _add_default_properties(event):
event[DSProps.SYNC_MODE] = SyncMode.UPDATE
event[DSProps.SYNC_TYPE] = VITRAGE_TYPE
event[VProps.UPDATE_TIMESTAMP] = str(datetime_utils.utcnow(False))
event[VProps.SAMPLE_TIMESTAMP] = str(datetime_utils.utcnow())
def _add_default_properties(event):
event[DSProps.SYNC_MODE] = SyncMode.UPDATE
event[DSProps.SYNC_TYPE] = VITRAGE_TYPE
event[VProps.UPDATE_TIMESTAMP] = str(datetime_utils.utcnow(False))
event[VProps.SAMPLE_TIMESTAMP] = str(datetime_utils.utcnow())
def _add_sampling_time(entity):
entity[DSProps.SAMPLE_DATE] = str(datetime_utils.utcnow())
def mark_edge_as_deleted(self, edge):
"""Marks the edge as is deleted, and updates delete timestamp"""
edge[EProps.IS_DELETED] = True
edge[EProps.UPDATE_TIMESTAMP] = str(utcnow())
self.update_edge(edge)
def mark_vertex_as_deleted(self, vertex):
"""Marks the vertex as is deleted, and updates deletion timestamp"""
vertex[VProps.IS_DELETED] = True
vertex[VProps.SAMPLE_TIMESTAMP] = str(utcnow())
self.update_vertex(vertex)
def _add_sampling_time(entity):
entity[DSProps.SAMPLE_DATE] = str(datetime_utils.utcnow())
