def schedule_boot_jobs(self, executor):
"""Schedule periodic collection if any task is currently assigned to
this executor """
try:
filters = {'executor': executor, 'deleted': False}
context = ctxt.get_admin_context()
tasks = db.task_get_all(context, filters=filters)
failed_tasks = db.failed_task_get_all(context, filters=filters)
LOG.info("Scheduling boot time jobs for this executor: total "
"jobs to be handled :%s" % len(tasks))
for task in tasks:
self.assign_job(context, task['id'], executor)
LOG.debug('Periodic collection job assigned for id: '
'%s ' % task['id'])
for failed_task in failed_tasks:
self.assign_failed_job(context, failed_task['id'], executor)
LOG.debug('Failed job assigned for id: '
'%s ' % failed_task['id'])
except Exception as e:
LOG.error(
"Failed to schedule boot jobs for this executor "
"reason: %s.", six.text_type(e))
else:
LOG.debug("Boot job scheduling completed.")
def process_alert_info(self, alert):
"""Fills alert model using driver manager interface."""
ctxt = context.get_admin_context()
storage = db.storage_get(ctxt, alert['storage_id'])
alert_model = {}
try:
alert_model = self.driver_manager.parse_alert(
ctxt, alert['storage_id'], alert)
# Fill storage specific info
if alert_model:
storage = self.get_storage_from_parsed_alert(
ctxt, storage, alert_model)
alert_util.fill_storage_attributes(alert_model, storage)
except exception.IncompleteTrapInformation as e:
LOG.warning(e)
threading.Thread(target=self.sync_storage_alert,
args=(ctxt, alert['storage_id'])).start()
except exception.AlertSourceNotFound:
LOG.info("Could not identify alert source from parsed alert. "
"Skipping the dispatch of alert")
return
except Exception as e:
LOG.error(e)
raise exception.InvalidResults(
"Failed to fill the alert model from driver.")
# Export to base exporter which handles dispatch for all exporters
if alert_model:
LOG.info("Dispatching one SNMP Trap to {} with sn {}".format(
alert_model['storage_id'], alert_model['serial_number']))
self.exporter_manager.dispatch(ctxt, [alert_model])
def on_process_leave(self, event):
LOG.info('Member %s left the group %s' %
(event.member_id, event.group_id))
executor_topic = event.member_id.decode('utf-8')
name = executor_topic.split(':')[1]
if name in self.executor_map.keys():
host = event.group_id.decode('utf-8')
LOG.info(
"Re-create process {0} in {1} that is handling tasks".format(
executor_topic, host))
launcher = self.create_process(executor_topic, host)
self.executor_map[name]["launcher"] = launcher
context = ctxt.get_admin_context()
for storage_id in self.executor_map[name]["storages"]:
tasks, failed_tasks = self.get_all_tasks(storage_id)
for task in tasks:
LOG.info("Re-scheduling task {0} of storage {1}".format(
task['id'], storage_id))
self.rpcapi.assign_job_local(context, task['id'],
executor_topic)
for f_task in failed_tasks:
LOG.info("Re-scheduling failed failed task {0},"
" of storage {1}".format(f_task['id'],
storage_id))
self.rpcapi.assign_failed_job_local(
context, f_task['id'], executor_topic)
def test_failed_job_fail_max_times(self, mock_collect_telemetry,
mock_failed_task_update,
mock_remove_job, mock_failed_task_get):
mock_collect_telemetry.return_value = TelemetryTaskStatus. \
TASK_EXEC_STATUS_FAILURE
failed_job = fake_failed_job.copy()
failed_job[
FailedTask.retry_count.name] = \
TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT - 1
# return with maximum retry count
mock_failed_task_get.return_value = failed_job
ctx = context.get_admin_context()
failed_job_handler = FailedPerformanceCollectionHandler.get_instance(
ctx, fake_failed_job_id)
# call failed job
failed_job_handler()
self.assertEqual(mock_remove_job.call_count, 1)
mock_failed_task_update.assert_called_once_with(
ctx, fake_failed_job_id, {
FailedTask.retry_count.name:
TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT,
FailedTask.result.name:
TelemetryJobStatus.FAILED_JOB_STATUS_INIT
})
def test_deleted_storage_exception(self, mock_collect_telemetry):
ctx = context.get_admin_context()
perf_collection_handler = PerformanceCollectionHandler(
ctx, fake_task_id, fake_storage_id, "", 100, fake_executor)
perf_collection_handler()
# Verify that collect telemetry for deleted storage
self.assertEqual(mock_collect_telemetry.call_count, 0)
def test_telemetry_job_scheduling(self, mock_add_job):
ctx = context.get_admin_context()
telemetry_job = JobHandler(ctx, fake_telemetry_job['id'],
fake_telemetry_job['storage_id'],
fake_telemetry_job['args'],
fake_telemetry_job['interval'])
# call telemetry job scheduling
telemetry_job.schedule_job(fake_telemetry_job['id'])
self.assertEqual(mock_add_job.call_count, 1)
def test_telemetry_removal_success(self, mock_log_error):
ctx = context.get_admin_context()
telemetry_job = JobHandler(ctx, fake_telemetry_job['id'],
fake_telemetry_job['storage_id'],
fake_telemetry_job['args'],
fake_telemetry_job['interval'])
# call telemetry job scheduling
telemetry_job.remove_job(fake_telemetry_job['id'])
self.assertEqual(mock_log_error.call_count, 0)
def test_distribute_new_job(self, mock_task_update, mock_assign_job,
mock_partitioner_start,
mock_get_task_executor):
ctx = context.get_admin_context()
task_distributor = TaskDistributor(ctx)
task_distributor.distribute_new_job('fake_task_id')
self.assertEqual(mock_assign_job.call_count, 1)
self.assertEqual(mock_task_update.call_count, 1)
self.assertEqual(mock_partitioner_start.call_count, 1)
self.assertEqual(mock_get_task_executor.call_count, 1)
def test_deleted_storage_exception(self, mock_collect_telemetry,
mock_failed_task_update):
ctx = context.get_admin_context()
failed_job_handler = FailedPerformanceCollectionHandler(
ctx, 1122, '12c2d52f-01bc-41f5-b73f-7abf6f38a2a6', '', 1234, 2,
1122334400, 1122334800, 'node1')
failed_job_handler()
# Verify that no action performed for deleted storage failed tasks
self.assertEqual(mock_collect_telemetry.call_count, 0)
self.assertEqual(mock_failed_task_update.call_count, 0)
def test_failed_job_deleted_storage(self, mock_collect_telemetry,
mock_failed_task_update,
mock_pause_job):
ctx = context.get_admin_context()
failed_job_handler = FailedPerformanceCollectionHandler.get_instance(
ctx, fake_failed_job_id)
failed_job_handler()
# Verify that no action performed for deleted storage failed tasks
self.assertEqual(mock_collect_telemetry.call_count, 0)
self.assertEqual(mock_failed_task_update.call_count, 0)
def test_performance_collection_failure(self, mock_collect_telemetry,
mock_failed_task_create):
mock_collect_telemetry.return_value = TelemetryTaskStatus. \
TASK_EXEC_STATUS_FAILURE
ctx = context.get_admin_context()
perf_collection_handler = PerformanceCollectionHandler.get_instance(
ctx, fake_task_id)
# call performance collection handler
perf_collection_handler()
# Verify that failed task create is called if collect telemetry fails
self.assertEqual(mock_failed_task_create.call_count, 1)
def test_performance_collection_success(self, mock_collect_telemetry,
mock_task_update):
mock_collect_telemetry.return_value = TelemetryTaskStatus. \
TASK_EXEC_STATUS_SUCCESS
ctx = context.get_admin_context()
perf_collection_handler = PerformanceCollectionHandler.get_instance(
ctx, fake_task_id)
# call performance collection handler
perf_collection_handler()
self.assertEqual(mock_collect_telemetry.call_count, 1)
self.assertEqual(mock_task_update.call_count, 1)
def test_performance_collection_deleted_storage(self,
mock_collect_telemetry,
mock_task_update):
mock_collect_telemetry.return_value = TelemetryTaskStatus. \
TASK_EXEC_STATUS_SUCCESS
ctx = context.get_admin_context()
perf_collection_handler = PerformanceCollectionHandler.get_instance(
ctx, fake_task_id)
perf_collection_handler()
# Verify that collect telemetry and db updated is not called
# for deleted storage
self.assertEqual(mock_collect_telemetry.call_count, 0)
self.assertEqual(mock_task_update.call_count, 0)
def test_failed_job_with_job_already_scheduled(self, mock_failed_get_all,
mock_add_job, mock_get_job):
# configure to return entry with job id
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid()
mock_failed_get_all.return_value = failed_jobs
# configure to have job in scheduler
mock_get_job.return_value = failed_jobs
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.remove_failed_job(fake_failed_job['id'])
# the job will not be scheduled
self.assertEqual(mock_add_job.call_count, 0)
def test_failed_job_scheduling_with_no_task(self, mock_failed_get_all,
mock_failed_task_delete,
mock_remove_job):
# configure to return entry with max retry count
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid()
mock_failed_get_all.return_value = failed_jobs
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.remove_failed_job(fake_failed_job)
# entry get deleted and job get removed
self.assertEqual(mock_failed_task_delete.call_count, 1)
self.assertEqual(mock_remove_job.call_count, 0)
def test_failed_job_with_max_retry(self, mock_failed_get_all,
mock_failed_task_delete,
mock_remove_job):
# configure to return entry with max retry count
failed_jobs = fake_failed_jobs.copy()
failed_jobs[0][FailedTask.retry_count.name] = \
TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT
mock_failed_get_all.return_value = failed_jobs
failed_job = FailedTelemetryJob(context.get_admin_context())
# call failed job scheduling
failed_job()
# entry get deleted and job get removed
self.assertEqual(mock_failed_task_delete.call_count, 2)
self.assertEqual(mock_remove_job.call_count, 1)
def _load_snmp_config(self):
"""Load snmp config from database when service start."""
ctxt = context.get_admin_context()
marker = None
finished = False
limit = constants.DEFAULT_LIMIT
while not finished:
alert_sources = db_api.alert_source_get_all(ctxt, marker=marker,
limit=limit)
for alert_source in alert_sources:
snmp_config = dict()
snmp_config.update(alert_source)
self._add_snmp_config(ctxt, snmp_config)
marker = alert_source['storage_id']
if len(alert_sources) < limit:
finished = True
def process_alert_info(self, alert):
"""Fills alert model using driver manager interface."""
ctxt = context.get_admin_context()
storage = db.storage_get(ctxt, alert['storage_id'])
try:
alert_model = self.driver_manager.parse_alert(
ctxt, alert['storage_id'], alert)
# Fill storage specific info
alert_util.fill_storage_attributes(alert_model, storage)
except Exception as e:
LOG.error(e)
raise exception.InvalidResults(
"Failed to fill the alert model from driver.")
# Export to base exporter which handles dispatch for all exporters
self.exporter_manager.dispatch(ctxt, alert_model)
def test_failed_job_success(self, mock_collect_telemetry,
mock_failed_task_update, mock_failed_job):
mock_collect_telemetry.return_value = TelemetryTaskStatus. \
TASK_EXEC_STATUS_SUCCESS
ctx = context.get_admin_context()
failed_job_handler = FailedPerformanceCollectionHandler.get_instance(
ctx, fake_failed_job_id)
# call failed job
failed_job_handler()
self.assertEqual(mock_failed_job.call_count, 1)
mock_failed_task_update.assert_called_once_with(
ctx, fake_failed_job_id, {
FailedTask.retry_count.name:
1,
FailedTask.result.name:
TelemetryJobStatus.FAILED_JOB_STATUS_SUCCESS
})
def test_process_alert_info_incompletetrap_exception(
self, mock_sync_alert, mock_parse_alert, mock_storage, mock_ctxt):
""" Mock parse alert for raising exception"""
alert = {
'storage_id': 'abcd-1234-56789',
'storage_name': 'storage1',
'vendor': 'fake vendor',
'model': 'fake mode',
'serial_number': 'serial-1234'
}
mock_ctxt.return_value = context.get_admin_context()
mock_storage.return_value = fakes.fake_storage_info()
mock_parse_alert.side_effect = exception.IncompleteTrapInformation(
'abcd-1234-56789')
alert_processor_inst = self._get_alert_processor()
alert_processor_inst.process_alert_info(alert)
self.assertTrue(mock_sync_alert.called)
def start(self):
""" Initialise the schedulers for periodic job creation
"""
ctxt = context.get_admin_context()
try:
# Create a jobs for periodic scheduling
periodic_scheduler_job_id = uuidutils.generate_uuid()
self.schedule_instance.add_job(
telemetry_job.TelemetryJob(ctxt), 'interval', args=[ctxt],
seconds=TelemetryCollection.PERIODIC_JOB_INTERVAL,
next_run_time=datetime.now(),
id=periodic_scheduler_job_id)
except Exception as e:
# TODO: Currently failure of scheduler is failing task manager
# start flow, it is logged and ignored.
LOG.error("Failed to initialize periodic tasks, reason: %s.",
six.text_type(e))
else:
self.schedule_instance.start()
def test_sync_successful(self, mock_masking_view_create,
mock_masking_view_update, mock_masking_view_del,
mock_masking_views_get_all,
mock_list_masking_views, get_lock):
cntxt = context.get_admin_context()
masking_view_obj = resources.MaskingViewTask(
cntxt, 'c5c91c98-91aa-40e6-85ac-37a1d3b32bda')
masking_view_obj.sync()
self.assertTrue(mock_list_masking_views.called)
self.assertTrue(mock_masking_views_get_all.called)
self.assertTrue(get_lock.called)
# Collect the volume groups from fake_storage
fake_storage_obj = fake_storage.FakeStorageDriver()
# Add the volume groups to DB
mock_list_masking_views.return_value \
= fake_storage_obj.list_masking_views(context)
mock_masking_views_get_all.return_value = list()
masking_view_obj.sync()
self.assertTrue(mock_masking_view_create.called)
# Update the volume groups to DB
mock_list_masking_views.return_value \
= masking_views_list
mock_masking_views_get_all.return_value \
= masking_views_list
masking_view_obj.sync()
self.assertTrue(mock_masking_view_update.called)
# Delete the volume groups to DB
mock_list_masking_views.return_value = list()
mock_masking_views_get_all.return_value \
= masking_views_list
masking_view_obj.sync()
self.assertTrue(mock_masking_view_del.called)
def test_sync_successful(self, mock_port_group_create,
mock_port_group_update, mock_port_group_del,
mock_port_groups_get_all, mock_list_port_groups,
get_lock):
ctxt = context.get_admin_context()
port_group_obj = resources.PortGroupTask(
ctxt, 'c5c91c98-91aa-40e6-85ac-37a1d3b32bda')
port_group_obj.sync()
self.assertTrue(mock_list_port_groups.called)
self.assertTrue(mock_port_groups_get_all.called)
self.assertTrue(get_lock.called)
# Collect the storage host groups from fake_storage
fake_storage_obj = fake_storage.FakeStorageDriver()
# Add the storage host groups to DB
mock_list_port_groups.return_value \
= fake_storage_obj.list_port_groups(context)
mock_port_groups_get_all.return_value = list()
port_group_obj.sync()
self.assertTrue(mock_port_group_create.called)
# Update the storage host groups to DB
mock_list_port_groups.return_value \
= port_groups_list
mock_port_groups_get_all.return_value \
= pg_list
port_group_obj.sync()
self.assertTrue(mock_port_group_update.called)
# Delete the storage host groups to DB
mock_list_port_groups.return_value = empty_port_groups_list
mock_port_groups_get_all.return_value \
= pg_list
port_group_obj.sync()
self.assertTrue(mock_port_group_del.called)
def process_alert_info(self, alert):
"""Fills alert model using driver manager interface."""
ctxt = context.get_admin_context()
storage = db.storage_get(ctxt, alert['storage_id'])
alert_model = {}
try:
alert_model = self.driver_manager.parse_alert(
ctxt, alert['storage_id'], alert)
# Fill storage specific info
if alert_model:
alert_util.fill_storage_attributes(alert_model, storage)
except exception.IncompleteTrapInformation as e:
LOG.warn(e)
threading.Thread(target=self.sync_storage_alert,
args=(ctxt, alert['storage_id'])).start()
except Exception as e:
LOG.error(e)
raise exception.InvalidResults(
"Failed to fill the alert model from driver.")
# Export to base exporter which handles dispatch for all exporters
if alert_model:
self.exporter_manager.dispatch(ctxt, alert_model)
def test_process_alert_info_success(self, mock_ctxt, mock_export_model,
mock_parse_alert, mock_storage):
fake_storage_info = fakes.fake_storage_info()
input_alert = {
'storage_id': 'abcd-1234-56789',
'connUnitEventId': 79,
'connUnitName': '000192601409',
'connUnitEventType': constants.EventType.EQUIPMENT_ALARM,
'connUnitEventDescr': 'Diagnostic '
'event trace triggered.',
'connUnitEventSeverity': 'warning',
'connUnitType': 'storage-subsystem',
'asyncEventSource': 'eventsource1',
'asyncEventCode': '1050',
'asyncEventComponentType': '1051',
'asyncEventComponentName': 'comp1'
}
expected_alert_model = {
'storage_id':
fake_storage_info['id'],
'storage_name':
fake_storage_info['name'],
'vendor':
fake_storage_info['vendor'],
'model':
fake_storage_info['model'],
'serial_number':
fake_storage_info['serial_number'],
'location':
'Array id=000192601409,Component '
'type=location1 '
'Group,Component name=comp1,Event '
'source=symmetrix',
'type':
input_alert['connUnitEventType'],
'severity':
constants.Severity.WARNING,
'category':
constants.Category.NOT_SPECIFIED,
'description':
input_alert['connUnitEventDescr'],
'resource_type':
constants.DEFAULT_RESOURCE_TYPE,
'alert_id':
input_alert['asyncEventCode'],
'alert_name':
'SAMPLE_ALERT_NAME',
'sequence_number':
79,
'recovery_advice':
'NA'
}
mock_storage.return_value = fake_storage_info
expected_ctxt = context.get_admin_context()
mock_ctxt.return_value = expected_ctxt
mock_parse_alert.return_value = fakes.fake_alert_model()
alert_processor_inst = self._get_alert_processor()
alert_processor_inst.process_alert_info(input_alert)
# Verify that model returned by driver is exported
mock_export_model.assert_called_once_with(expected_ctxt,
[expected_alert_model])
def periodic_tasks(self, raise_on_error=False):
"""Tasks to be run at a periodic interval."""
ctxt = context.get_admin_context()
self.manager.periodic_tasks(ctxt, raise_on_error=raise_on_error)
def test_telemetry_job_scheduling_exception(self, mock_log_error):
ctx = context.get_admin_context()
telemetry_job = TelemetryJob(ctx)
# call telemetry job scheduling
telemetry_job(ctx)
self.assertEqual(mock_log_error.call_count, 2)
def test_failed_job_scheduling(self, mock_add_job):
failed_job = FailedJobHandler(context.get_admin_context())
# call failed job scheduling
failed_job.schedule_failed_job(fake_failed_job['id'])
self.assertEqual(mock_add_job.call_count, 1)
def get_all_tasks(self, storage_id):
filters = {'storage_id': storage_id, 'deleted': False}
context = ctxt.get_admin_context()
tasks = db.task_get_all(context, filters=filters)
failed_tasks = db.failed_task_get_all(context, filters=filters)
return tasks, failed_tasks
from unittest import mock
from delfin import context, exception
from delfin import test
from delfin.db import api as db_api
from delfin.db.sqlalchemy import api, models
ctxt = context.get_admin_context()
class TestSIMDBAPI(test.TestCase):
@mock.patch('sqlalchemy.create_engine', mock.Mock())
def test_register_db(self):
db_api.register_db()
def test_get_session(self):
api.get_session()
def test_get_engine(self):
api.get_engine()
@mock.patch('delfin.db.sqlalchemy.api.get_session')
def test_storage_get(self, mock_session):
fake_storage = {}
mock_session.return_value.__enter__.return_value.query.return_value \
= fake_storage
result = db_api.storage_get(ctxt,
'c5c91c98-91aa-40e6-85ac-37a1d3b32bda')
assert len(result) == 0
@mock.patch('delfin.db.sqlalchemy.api.get_session')
