def initializing_process(self): try: LOG.info('Consistency Initializing Process - Started') if not self._wait_for_action( self.initialization_status.is_received_all_end_messages): LOG.error('Maximum retries for consistency initializator ' 'were done') LOG.info('All end messages were received') self.evaluator.enabled = True timestamp = str(utcnow()) all_vertices = self.graph.get_vertices() self._run_evaluator(all_vertices) self._wait_for_processing_evaluator_events() self._mark_old_deduced_alarms_as_deleted(timestamp) self.initialization_status.status = \ self.initialization_status.FINISHED LOG.info('Consistency Initializing Process - Finished') except Exception as e: LOG.exception('Error in deleting vertices from entity_graph: %s', e)
def _periodic_process_setup_stage(self, consistency_interval): self._create_processor_with_graph(self.conf, processor=self.processor) current_time = utcnow() # set all vertices to be have timestamp that consistency won't get self._update_timestamp( self.processor.entity_graph.get_vertices(), current_time + timedelta(seconds=1.5 * consistency_interval)) # check number of instances in graph instance_vertices = self.processor.entity_graph.get_vertices({ VProps.CATEGORY: EntityCategory.RESOURCE, VProps.TYPE: NOVA_INSTANCE_DATASOURCE }) self.assertEqual(self.NUM_INSTANCES, len(instance_vertices)) # set current timestamp of part of the instances self._update_timestamp(instance_vertices[0:3], current_time) # set part of the instances as deleted for i in range(3, 6): instance_vertices[i][VProps.IS_DELETED] = True self.processor.entity_graph.update_vertex(instance_vertices[i]) # set part of the instances as deleted for i in range(6, 9): instance_vertices[i][VProps.IS_DELETED] = True instance_vertices[i][VProps.SAMPLE_TIMESTAMP] = str( current_time + timedelta(seconds=2 * consistency_interval + 1)) self.processor.entity_graph.update_vertex(instance_vertices[i])
def _periodic_process_setup_stage(self, consistency_interval): self._create_processor_with_graph(self.conf, processor=self.processor) current_time = utcnow() # set all vertices to be have timestamp that consistency won't get self._update_timestamp(self.processor.entity_graph.get_vertices(), current_time + timedelta(seconds=1.5 * consistency_interval)) # check number of instances in graph instance_vertices = self.processor.entity_graph.get_vertices({ VProps.CATEGORY: EntityCategory.RESOURCE, VProps.TYPE: NOVA_INSTANCE_DATASOURCE }) self.assertEqual(self.NUM_INSTANCES, len(instance_vertices)) # set current timestamp of part of the instances self._update_timestamp(instance_vertices[0:3], current_time) # set part of the instances as deleted for i in range(3, 6): instance_vertices[i][VProps.IS_DELETED] = True self.processor.entity_graph.update_vertex(instance_vertices[i]) # set part of the instances as deleted for i in range(6, 9): instance_vertices[i][VProps.IS_DELETED] = True instance_vertices[i][VProps.SAMPLE_TIMESTAMP] = str( current_time + timedelta(seconds=2 * consistency_interval + 1)) self.processor.entity_graph.update_vertex(instance_vertices[i])
def _push_events_to_queue(self, vertices, action): for vertex in vertices: event = { DSProps.SYNC_TYPE: CONSISTENCY_DATASOURCE, DSProps.SYNC_MODE: SyncMode.UPDATE, DSProps.SAMPLE_DATE: str(utcnow()), DSProps.EVENT_TYPE: action, VProps.VITRAGE_ID: vertex[VProps.VITRAGE_ID] } self.evaluator_queue.put(event)
def _create_alarm(vitrage_id, alarm_type): return graph_utils.create_vertex( vitrage_id, entity_id=vitrage_id, entity_category=EntityCategory.ALARM, entity_type=alarm_type, entity_state='active', is_deleted=False, sample_timestamp=utcnow(), is_placeholder=False, )
def _find_stale_entities(self): query = { 'and': [ {'!=': {VProps.TYPE: VITRAGE_TYPE}}, {'<': {VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta( seconds=2 * self.conf.datasources.snapshots_interval))}} ] } vertices = self.graph.get_vertices(query_dict=query) return set(self._filter_vertices_to_be_deleted(vertices))
def _find_old_deleted_entities(self): query = { 'and': [ {'==': {VProps.IS_DELETED: True}}, {'<': {VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta( seconds=self.conf.consistency.min_time_to_delete))}} ] } vertices = self.graph.get_vertices(query_dict=query) return self._filter_vertices_to_be_deleted(vertices)
def _find_old_deleted_entities(self): query = { 'and': [{ '==': { VProps.IS_DELETED: True } }, { '<': { VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta( seconds=self.conf.consistency.min_time_to_delete)) } }] } vertices = self.graph.get_vertices(query_dict=query) return self._filter_vertices_to_be_deleted(vertices)
def _find_stale_entities(self): query = { 'and': [{ '!=': { VProps.TYPE: VITRAGE_TYPE } }, { '<': { VProps.SAMPLE_TIMESTAMP: str(utcnow() - timedelta( seconds=2 * self.conf.datasources.snapshots_interval)) } }] } vertices = self.graph.get_vertices(query_dict=query) return set(self._filter_vertices_to_be_deleted(vertices))
def test_update_entity_state(self): # create instance event with host neighbor and check validity (vertex, neighbors, processor) =\ self._create_and_check_entity(status='STARTING') # check added entity vertex = processor.entity_graph.get_vertex(vertex.vertex_id) self.assertEqual('STARTING', vertex.properties[VProps.STATE]) # update instance event with state running vertex.properties[VProps.STATE] = 'RUNNING' vertex.properties[VProps.SAMPLE_TIMESTAMP] = str(utcnow()) processor.update_entity(vertex, neighbors) # check state self._check_graph(processor, self.NUM_VERTICES_AFTER_CREATION, self.NUM_EDGES_AFTER_CREATION) vertex = processor.entity_graph.get_vertex(vertex.vertex_id) self.assertEqual('RUNNING', vertex.properties[VProps.STATE])
def add_template(self, template_def): current_time = datetime_utils.utcnow() result = syntax_validation(template_def) if not result.is_valid: LOG.info('Unable to load template: %s', result.comment) else: result = content_validation(template_def) if not result.is_valid: LOG.info('Unable to load template: %s', result.comment) template_uuid = uuid.uuid4() self.templates[str(template_uuid)] = Template(template_uuid, template_def, current_time, result) if result.is_valid: template_data = TemplateData(template_def) self._add_template_scenarios(template_data)
def _filter_and_cache_alarms(self, alarms, filter_): alarms_to_update = [] now = datetime_utils.utcnow(False) for alarm in alarms: alarm_key = self._alarm_key(alarm) old_alarm, timestamp = self.cache.get(alarm_key, (None, None)) if filter_(self, alarm, old_alarm): alarms_to_update.append(alarm) self.cache[alarm_key] = alarm, now # add alarms that were deleted for cached_alarm, timestamp in self.cache.values(): if self._is_erroneous(cached_alarm) and timestamp is not now: LOG.debug('deleting cached_alarm %s', cached_alarm) cached_alarm[DSProps.EVENT_TYPE] = EventAction.DELETE_ENTITY alarms_to_update.append(cached_alarm) self.cache.pop(self._alarm_key(cached_alarm)) return alarms_to_update
def _add_default_properties(event): event[DSProps.SYNC_MODE] = SyncMode.UPDATE event[DSProps.SYNC_TYPE] = VITRAGE_TYPE event[VProps.UPDATE_TIMESTAMP] = str(datetime_utils.utcnow(False)) event[VProps.SAMPLE_TIMESTAMP] = str(datetime_utils.utcnow())
def _add_sampling_time(entity): entity[DSProps.SAMPLE_DATE] = str(datetime_utils.utcnow())
def mark_edge_as_deleted(self, edge): """Marks the edge as is deleted, and updates delete timestamp""" edge[EProps.IS_DELETED] = True edge[EProps.UPDATE_TIMESTAMP] = str(utcnow()) self.update_edge(edge)
def mark_vertex_as_deleted(self, vertex): """Marks the vertex as is deleted, and updates deletion timestamp""" vertex[VProps.IS_DELETED] = True vertex[VProps.SAMPLE_TIMESTAMP] = str(utcnow()) self.update_vertex(vertex)