def setUp(self):
super(TestConsistencyFunctional, self).setUp()
self.conf_reregister_opts(self.CONSISTENCY_OPTS, 'consistency')
self.conf_reregister_opts(self.EVALUATOR_OPTS, 'evaluator')
self.add_db()
self.load_datasources()
self.graph = NXGraph("Entity Graph")
self.processor = Processor(self.graph)
self.event_queue = queue.Queue()
def actions_callback(event_type, data):
"""Mock notify method
Mocks vitrage.messaging.VitrageNotifier.notify(event_type, data)
:param event_type: is currently always the same and is ignored
:param data:
"""
self.event_queue.put(data)
scenario_repo = ScenarioRepository()
self.evaluator = ScenarioEvaluator(self.processor.entity_graph,
scenario_repo, actions_callback)
self.consistency_enforcer = ConsistencyEnforcer(
self.processor.entity_graph, actions_callback)
def __init__(self, graph, driver_exec, persist):
super(Scheduler, self).__init__()
self.graph = graph
self.driver_exec = driver_exec
self.persist = persist
self.consistency = ConsistencyEnforcer(graph)
self.periodic = None
def __init__(self, conf, graph, events_coordination, persist):
super(Scheduler, self).__init__()
self.conf = conf
self.graph = graph
self.events_coordination = events_coordination
self.persist = persist
self.consistency = ConsistencyEnforcer(conf, graph)
self.periodic = None
def __init__(self, conf, graph, events_coordination):
super(Scheduler, self).__init__()
self.conf = conf
self.graph = graph
self.events_coordination = events_coordination
self.graph_persistor = GraphPersistor(conf) if \
self.conf.persistency.enable_persistency else None
self.consistency = ConsistencyEnforcer(conf, graph)
self.periodic = None
class Scheduler(object):
def __init__(self, conf, graph, driver_exec, persist):
super(Scheduler, self).__init__()
self.conf = conf
self.graph = graph
self.driver_exec = driver_exec
self.persist = persist
self.consistency = ConsistencyEnforcer(conf, graph)
self.periodic = None
def start_periodic_tasks(self, immediate_get_all):
thread_num = len(utils.get_pull_drivers_names(self.conf))
thread_num += 2 # for consistency and get_all
self.periodic = periodics.PeriodicWorker.create(
[],
executor_factory=lambda: ThreadPoolExecutor(max_workers=thread_num
))
self._add_consistency_timer()
self._add_datasource_timers(immediate_get_all)
spawn(self.periodic.start)
def _add_consistency_timer(self):
spacing = self.conf.datasources.snapshots_interval
@periodics.periodic(spacing=spacing)
def consistency_periodic():
try:
self.consistency.periodic_process()
except Exception:
LOG.exception('run_consistency failed.')
self.periodic.add(consistency_periodic)
LOG.info("added consistency_periodic (spacing=%s)", spacing)
def _add_datasource_timers(self, run_immediately):
spacing = self.conf.datasources.snapshots_interval
@periodics.periodic(spacing=spacing, run_immediately=run_immediately)
def get_all_periodic():
self.driver_exec.snapshot_get_all(DatasourceAction.SNAPSHOT)
self.periodic.add(get_all_periodic)
LOG.info("added get_all_periodic (spacing=%s)", spacing)
driver_names = utils.get_pull_drivers_names(self.conf)
for d_name in driver_names:
spacing = self.conf[d_name].changes_interval
@periodics.periodic(spacing=spacing)
def get_changes_periodic():
self.driver_exec.get_changes(d_name)
self.periodic.add(get_changes_periodic)
LOG.info("added get_changes_periodic %s (spacing=%s)", d_name,
spacing)
def setUpClass(cls):
super(TestConsistencyFunctional, cls).setUpClass()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.CONSISTENCY_OPTS, group='consistency')
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.EVALUATOR_OPTS, group='evaluator')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.add_db(cls.conf)
cls.load_datasources(cls.conf)
cls.graph = NXGraph("Entity Graph")
cls.processor = Processor(cls.conf, cls.graph)
cls.event_queue = queue.Queue()
def actions_callback(event_type, data):
"""Mock notify method
Mocks vitrage.messaging.VitrageNotifier.notify(event_type, data)
:param event_type: is currently always the same and is ignored
:param data:
"""
cls.event_queue.put(data)
scenario_repo = ScenarioRepository(cls.conf)
cls.evaluator = ScenarioEvaluator(cls.conf,
cls.processor.entity_graph,
scenario_repo,
actions_callback)
cls.consistency_enforcer = ConsistencyEnforcer(
cls.conf,
cls.processor.entity_graph,
actions_callback)
def start(self):
LOG.info("Vitrage Consistency Service - Starting...")
super(VitrageConsistencyService, self).start()
consistency_enf = ConsistencyEnforcer(self.conf,
self.evaluator_queue,
self.entity_graph)
self.tg.add_timer(self.conf.datasources.snapshots_interval,
consistency_enf.periodic_process,
initial_delay=60 +
self.conf.datasources.snapshots_interval)
LOG.info("Vitrage Consistency Service - Started!")
def start(self):
LOG.info("Vitrage Consistency Service - Starting...")
super(VitrageConsistencyService, self).start()
consistency_enf = ConsistencyEnforcer(
conf=self.conf,
actions_callback=self.actions_notifier.notify,
entity_graph=self.entity_graph)
self.tg.add_timer(self.conf.datasources.snapshots_interval,
consistency_enf.periodic_process,
initial_delay=60 +
self.conf.datasources.snapshots_interval)
LOG.info("Vitrage Consistency Service - Started!")
def __init__(self, workers):
self.graph = get_graph_driver()('Entity Graph')
self.db = db_connection = storage.get_connection_from_config()
self.workers = workers
self.events_coordination = EventsCoordination(self.process_event)
self.persist = GraphPersistency(db_connection, self.graph)
self.driver_exec = driver_exec.DriverExec(
self.events_coordination.handle_multiple_low_priority,
self.persist, self.graph)
consistency = ConsistencyEnforcer(
self.graph, self.events_coordination.handle_multiple_high_priority)
self.scheduler = Scheduler(self.graph, self.driver_exec, self.persist,
consistency)
self.processor = Processor(self.graph)
def setUpClass(cls):
super(TestConsistencyFunctional, cls).setUpClass()
cls.initialization_status = InitializationStatus()
cls.conf = cfg.ConfigOpts()
cls.conf.register_opts(cls.CONSISTENCY_OPTS, group='consistency')
cls.conf.register_opts(cls.PROCESSOR_OPTS, group='entity_graph')
cls.conf.register_opts(cls.EVALUATOR_OPTS, group='evaluator')
cls.conf.register_opts(cls.DATASOURCES_OPTS, group='datasources')
cls.load_datasources(cls.conf)
cls.processor = Processor(cls.conf, cls.initialization_status)
cls.event_queue = queue.Queue()
scenario_repo = ScenarioRepository(cls.conf)
cls.evaluator = ScenarioEvaluator(cls.conf, cls.processor.entity_graph,
scenario_repo, cls.event_queue)
cls.consistency_enforcer = ConsistencyEnforcer(
cls.conf, cls.event_queue, cls.evaluator,
cls.processor.entity_graph, cls.initialization_status)
def start(self):
LOG.info("Vitrage Graph Consistency Service - Starting...")
super(VitrageGraphConsistencyService, self).start()
consistency_enf = ConsistencyEnforcer(self.conf, self.event_queue,
self.evaluator,
self.entity_graph,
self.initialization_status)
self.tg.add_timer(self.conf.datasources.snapshots_interval,
consistency_enf.periodic_process,
initial_delay=60 +
self.conf.datasources.snapshots_interval)
initializing_process_thread = \
threading.Thread(target=consistency_enf.initializing_process)
initializing_process_thread.start()
LOG.info("Vitrage Graph Consistency Service - Started!")
class TestConsistencyFunctional(TestFunctionalBase, TestConfiguration):
CONSISTENCY_OPTS = [
cfg.IntOpt('min_time_to_delete', default=1, min=1),
]
EVALUATOR_OPTS = [
cfg.StrOpt(
'templates_dir',
default=utils.get_resources_dir() + '/templates/consistency',
),
cfg.StrOpt(
'equivalences_dir',
default='equivalences',
),
cfg.StrOpt(
'notifier_topic',
default='vitrage.evaluator',
),
]
def setUp(self):
super(TestConsistencyFunctional, self).setUp()
self.conf_reregister_opts(self.CONSISTENCY_OPTS, 'consistency')
self.conf_reregister_opts(self.EVALUATOR_OPTS, 'evaluator')
self.add_db()
self.load_datasources()
self.graph = NXGraph("Entity Graph")
self.processor = Processor(self.graph)
self.event_queue = queue.Queue()
def actions_callback(event_type, data):
"""Mock notify method
Mocks vitrage.messaging.VitrageNotifier.notify(event_type, data)
:param event_type: is currently always the same and is ignored
:param data:
"""
self.event_queue.put(data)
scenario_repo = ScenarioRepository()
self.evaluator = ScenarioEvaluator(self.processor.entity_graph,
scenario_repo, actions_callback)
self.consistency_enforcer = ConsistencyEnforcer(
self.processor.entity_graph, actions_callback)
def test_periodic_process(self):
# Setup
consistency_interval = self.conf.datasources.snapshots_interval
self._periodic_process_setup_stage(consistency_interval)
self._add_alarms_by_type(consistency_interval=consistency_interval,
alarm_type='prometheus')
# Action
time.sleep(2 * consistency_interval + 1)
self.consistency_enforcer.periodic_process()
self._process_events()
# Test Assertions
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY:
EntityCategory.RESOURCE,
VProps.VITRAGE_TYPE:
NOVA_INSTANCE_DATASOURCE
})
deleted_instance_vertices = \
self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY: EntityCategory.RESOURCE,
VProps.VITRAGE_TYPE: NOVA_INSTANCE_DATASOURCE,
VProps.VITRAGE_IS_DELETED: True
})
self.assertThat(instance_vertices,
matchers.HasLength(self.NUM_INSTANCES - 3))
# number of resources:
# number of vertices - 3 (deleted instances)
# number of nics - 1
# number of volumes - 1
# number of prometheus alarms - 1
self.assertThat(
self.processor.entity_graph.get_vertices(),
matchers.HasLength(
# 3 instances deleted
self._num_total_expected_vertices() - 3 + 3 -
1 + # one nic deleted
3 - 1 + # one cinder.volume deleted
3 - 1) # one prometheus deleted
)
self.assertThat(deleted_instance_vertices, matchers.HasLength(3))
# one nic was deleted, one marked as deleted, one untouched
# same for cinder.volume
self._assert_vertices_status(EntityCategory.RESOURCE, 'nic', 2, 1)
self._assert_vertices_status(EntityCategory.RESOURCE, 'cinder.volume',
2, 1)
# one prometheus deleted, other two are untouched
# prometheus vertices should not be marked as deleted, since the
# datasource did not ask to delete outdated vertices.
self._assert_vertices_status(EntityCategory.ALARM, 'prometheus', 2, 0)
def test_should_delete_vertex(self):
# should be deleted because the static datasource asks to delete its
# outdated vertices
static_vertex = {VProps.VITRAGE_DATASOURCE_NAME: 'static'}
self.assertTrue(
self.consistency_enforcer._should_delete_vertex(static_vertex))
# should be deleted because the cinder datasource asks to delete its
# outdated vertices
volume_vertex = {VProps.VITRAGE_DATASOURCE_NAME: 'cinder.volume'}
self.assertTrue(
self.consistency_enforcer._should_delete_vertex(volume_vertex))
# should not be deleted because the prometheus datasource does not ask
# to delete its outdated vertices
prometheus_vertex = {VProps.VITRAGE_DATASOURCE_NAME: 'prometheus'}
self.assertFalse(
self.consistency_enforcer._should_delete_vertex(prometheus_vertex))
# should be deleted because it is a placeholder
placeholder_vertex = {
VProps.VITRAGE_IS_PLACEHOLDER: True,
VProps.VITRAGE_TYPE: 'prometheus'
}
self.assertTrue(
self.consistency_enforcer._should_delete_vertex(
placeholder_vertex))
# should not be deleted because it is an openstack.cluster
cluster_vertex = {
VProps.VITRAGE_IS_PLACEHOLDER: True,
VProps.VITRAGE_TYPE: 'openstack.cluster'
}
self.assertFalse(
self.consistency_enforcer._should_delete_vertex(cluster_vertex))
vertices = [
static_vertex, volume_vertex, prometheus_vertex,
placeholder_vertex, cluster_vertex
]
vertices_to_mark_deleted = self.consistency_enforcer.\
_filter_vertices_to_be_marked_as_deleted(vertices)
self.assertThat(vertices_to_mark_deleted, matchers.HasLength(3))
self.assertTrue(static_vertex in vertices_to_mark_deleted)
self.assertTrue(placeholder_vertex in vertices_to_mark_deleted)
self.assertTrue(volume_vertex in vertices_to_mark_deleted)
self.assertFalse(prometheus_vertex in vertices_to_mark_deleted)
self.assertFalse(cluster_vertex in vertices_to_mark_deleted)
def _assert_vertices_status(self, category, vitrage_type, num_vertices,
num_marked_deleted):
vertices = \
self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY: category,
VProps.VITRAGE_TYPE: vitrage_type,
})
self.assertThat(vertices, matchers.HasLength(num_vertices))
marked_deleted_vertices = \
self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY: category,
VProps.VITRAGE_TYPE: vitrage_type,
VProps.VITRAGE_IS_DELETED: True
})
self.assertThat(marked_deleted_vertices,
matchers.HasLength(num_marked_deleted))
def _periodic_process_setup_stage(self, consistency_interval):
self._create_processor_with_graph(processor=self.processor)
current_time = utcnow()
# set all vertices to be have timestamp that consistency won't get
self._update_timestamp(
self.processor.entity_graph.get_vertices(),
current_time + timedelta(seconds=1.5 * consistency_interval))
# check number of instances in graph
instance_vertices = self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY:
EntityCategory.RESOURCE,
VProps.VITRAGE_TYPE:
NOVA_INSTANCE_DATASOURCE
})
self.assertThat(instance_vertices,
matchers.HasLength(self.NUM_INSTANCES))
# set current timestamp of part of the instances
self._update_timestamp(instance_vertices[0:3], current_time)
# set part of the instances as deleted
for i in range(3, 6):
instance_vertices[i][VProps.VITRAGE_IS_DELETED] = True
self.processor.entity_graph.update_vertex(instance_vertices[i])
# set part of the instances as deleted
for i in range(6, 9):
instance_vertices[i][VProps.VITRAGE_IS_DELETED] = True
instance_vertices[i][VProps.VITRAGE_SAMPLE_TIMESTAMP] = str(
current_time + timedelta(seconds=2 * consistency_interval + 1))
self.processor.entity_graph.update_vertex(instance_vertices[i])
self._add_resources_by_type(consistency_interval=consistency_interval,
datasource_name='static',
resource_type='nic')
self._add_resources_by_type(consistency_interval=consistency_interval,
datasource_name='cinder.volume',
resource_type='cinder.volume')
def _update_timestamp(self, lst, timestamp):
for vertex in lst:
vertex[VProps.VITRAGE_SAMPLE_TIMESTAMP] = str(timestamp)
self.processor.entity_graph.update_vertex(vertex)
def _process_events(self):
num_retries = 0
while True:
if self.event_queue.empty():
time.sleep(0.3)
if not self.event_queue.empty():
time.sleep(1)
count = 0
while not self.event_queue.empty():
count += 1
data = self.event_queue.get()
if isinstance(data, list):
for event in data:
self.processor.process_event(event)
else:
self.processor.process_event(data)
return
num_retries += 1
if num_retries == 30:
return
def _add_resources_by_type(self, consistency_interval, resource_type,
datasource_name):
def _create_resource_by_type(v_id,
v_type,
ds_name,
timestamp,
is_deleted=False):
return self._create_resource(vitrage_id=v_id,
resource_type=v_type,
datasource_name=ds_name,
sample_timestamp=timestamp,
is_deleted=is_deleted)
self._add_entities_with_different_timestamps(
consistency_interval=consistency_interval,
create_func=_create_resource_by_type,
category=EntityCategory.RESOURCE,
datasource_name=datasource_name,
resource_type=resource_type)
def _add_alarms_by_type(self, consistency_interval, alarm_type):
def _create_alarm_by_type(v_id,
v_type,
ds_name,
timestamp,
is_deleted=False):
return self._create_alarm(vitrage_id=v_id,
alarm_type=v_type,
datasource_name=ds_name,
project_id=None,
vitrage_resource_project_id=None,
metadata=None,
vitrage_sample_timestamp=timestamp,
is_deleted=is_deleted)
self._add_entities_with_different_timestamps(
consistency_interval=consistency_interval,
create_func=_create_alarm_by_type,
category=EntityCategory.ALARM,
datasource_name=alarm_type,
resource_type=alarm_type)
def _add_entities_with_different_timestamps(self, consistency_interval,
create_func, category,
datasource_name,
resource_type):
# add resources to the graph:
# - updated_resource
# - outdated_resource with an old timestamp
# - deleted_resource with an old timestamp and is_deleted==true
future_timestamp = \
str(utcnow() + timedelta(seconds=2 * consistency_interval))
past_timestamp = \
str(utcnow() - timedelta(seconds=2 * consistency_interval - 1))
updated_resource = create_func(v_id=resource_type + '1234',
v_type=resource_type,
ds_name=datasource_name,
timestamp=future_timestamp)
outdated_resource = create_func(v_id=resource_type + '5678',
v_type=resource_type,
ds_name=datasource_name,
timestamp=past_timestamp)
deleted_resource = create_func(v_id=resource_type + '9999',
v_type=resource_type,
ds_name=datasource_name,
timestamp=past_timestamp,
is_deleted=True)
self.graph.add_vertex(updated_resource)
self.graph.add_vertex(outdated_resource)
self.graph.add_vertex(deleted_resource)
# get the list of vertices
resource_vertices = self.processor.entity_graph.get_vertices({
VProps.VITRAGE_CATEGORY:
category,
VProps.VITRAGE_TYPE:
resource_type
})
self.assertThat(resource_vertices, matchers.HasLength(3),
'Wrong number of vertices of type %s', resource_type)
class Scheduler(object):
def __init__(self, conf, graph, events_coordination):
super(Scheduler, self).__init__()
self.conf = conf
self.graph = graph
self.events_coordination = events_coordination
self.graph_persistor = GraphPersistor(conf) if \
self.conf.persistency.enable_persistency else None
self.consistency = ConsistencyEnforcer(conf, graph)
self.periodic = None
def start_periodic_tasks(self):
self.periodic = periodics.PeriodicWorker.create(
[], executor_factory=lambda: ThreadPoolExecutor(max_workers=10))
self.add_persist_timer()
self.add_consistency_timer()
self.add_rpc_datasources_timers()
spawn(self.periodic.start)
def add_persist_timer(self):
if not self.graph_persistor:
return
spacing = self.conf.persistency.graph_persistency_interval
@periodics.periodic(spacing=spacing)
def persist_periodic():
if self.graph_persistor:
try:
self.graph_persistor.store_graph(graph=self.graph)
except Exception as e:
LOG.exception('persist failed %s', e)
self.periodic.add(persist_periodic)
LOG.info("added persist_periodic (spacing=%s)", spacing)
def add_consistency_timer(self):
spacing = self.conf.datasources.snapshots_interval
@periodics.periodic(spacing=spacing)
def consistency_periodic():
try:
self.consistency.periodic_process()
except Exception as e:
LOG.exception('run_consistency failed %s', e)
self.periodic.add(consistency_periodic)
LOG.info("added consistency_periodic (spacing=%s)", spacing)
def add_rpc_datasources_timers(self):
spacing = self.conf.datasources.snapshots_interval
rpc_client = ds_rpc.create_rpc_client_instance(self.conf)
@periodics.periodic(spacing=spacing)
def get_all_periodic():
try:
ds_rpc.get_all(rpc_client,
self.events_coordination,
self.conf.datasources.types,
DatasourceAction.SNAPSHOT)
except Exception as e:
LOG.exception('get_all_periodic failed %s', e)
self.periodic.add(get_all_periodic)
LOG.info("added get_all_periodic (spacing=%s)", spacing)
driver_names = utils.get_pull_drivers_names(self.conf)
for d_name in driver_names:
spacing = self.conf[d_name].changes_interval
rpc_client = ds_rpc.create_rpc_client_instance(self.conf)
@periodics.periodic(spacing=spacing)
def get_changes_periodic(driver_name=d_name):
try:
ds_rpc.get_changes(rpc_client,
self.events_coordination,
driver_name)
except Exception as e:
LOG.exception('get_changes_periodic %s failed %s',
driver_name, e)
self.periodic.add(get_changes_periodic)
LOG.info("added get_changes_periodic %s (spacing=%s)",
d_name, spacing)