def schedule_boot_jobs(self, executor): """Schedule periodic collection if any task is currently assigned to this executor """ try: filters = {'executor': executor, 'deleted': False} context = ctxt.get_admin_context() tasks = db.task_get_all(context, filters=filters) failed_tasks = db.failed_task_get_all(context, filters=filters) LOG.info("Scheduling boot time jobs for this executor: total " "jobs to be handled :%s" % len(tasks)) for task in tasks: self.assign_job(context, task['id'], executor) LOG.debug('Periodic collection job assigned for id: ' '%s ' % task['id']) for failed_task in failed_tasks: self.assign_failed_job(context, failed_task['id'], executor) LOG.debug('Failed job assigned for id: ' '%s ' % failed_task['id']) except Exception as e: LOG.error( "Failed to schedule boot jobs for this executor " "reason: %s.", six.text_type(e)) else: LOG.debug("Boot job scheduling completed.")
def process_alert_info(self, alert): """Fills alert model using driver manager interface.""" ctxt = context.get_admin_context() storage = db.storage_get(ctxt, alert['storage_id']) alert_model = {} try: alert_model = self.driver_manager.parse_alert( ctxt, alert['storage_id'], alert) # Fill storage specific info if alert_model: storage = self.get_storage_from_parsed_alert( ctxt, storage, alert_model) alert_util.fill_storage_attributes(alert_model, storage) except exception.IncompleteTrapInformation as e: LOG.warning(e) threading.Thread(target=self.sync_storage_alert, args=(ctxt, alert['storage_id'])).start() except exception.AlertSourceNotFound: LOG.info("Could not identify alert source from parsed alert. " "Skipping the dispatch of alert") return except Exception as e: LOG.error(e) raise exception.InvalidResults( "Failed to fill the alert model from driver.") # Export to base exporter which handles dispatch for all exporters if alert_model: LOG.info("Dispatching one SNMP Trap to {} with sn {}".format( alert_model['storage_id'], alert_model['serial_number'])) self.exporter_manager.dispatch(ctxt, [alert_model])
def on_process_leave(self, event): LOG.info('Member %s left the group %s' % (event.member_id, event.group_id)) executor_topic = event.member_id.decode('utf-8') name = executor_topic.split(':')[1] if name in self.executor_map.keys(): host = event.group_id.decode('utf-8') LOG.info( "Re-create process {0} in {1} that is handling tasks".format( executor_topic, host)) launcher = self.create_process(executor_topic, host) self.executor_map[name]["launcher"] = launcher context = ctxt.get_admin_context() for storage_id in self.executor_map[name]["storages"]: tasks, failed_tasks = self.get_all_tasks(storage_id) for task in tasks: LOG.info("Re-scheduling task {0} of storage {1}".format( task['id'], storage_id)) self.rpcapi.assign_job_local(context, task['id'], executor_topic) for f_task in failed_tasks: LOG.info("Re-scheduling failed failed task {0}," " of storage {1}".format(f_task['id'], storage_id)) self.rpcapi.assign_failed_job_local( context, f_task['id'], executor_topic)
def test_failed_job_fail_max_times(self, mock_collect_telemetry, mock_failed_task_update, mock_remove_job, mock_failed_task_get): mock_collect_telemetry.return_value = TelemetryTaskStatus. \ TASK_EXEC_STATUS_FAILURE failed_job = fake_failed_job.copy() failed_job[ FailedTask.retry_count.name] = \ TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT - 1 # return with maximum retry count mock_failed_task_get.return_value = failed_job ctx = context.get_admin_context() failed_job_handler = FailedPerformanceCollectionHandler.get_instance( ctx, fake_failed_job_id) # call failed job failed_job_handler() self.assertEqual(mock_remove_job.call_count, 1) mock_failed_task_update.assert_called_once_with( ctx, fake_failed_job_id, { FailedTask.retry_count.name: TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT, FailedTask.result.name: TelemetryJobStatus.FAILED_JOB_STATUS_INIT })
def test_deleted_storage_exception(self, mock_collect_telemetry): ctx = context.get_admin_context() perf_collection_handler = PerformanceCollectionHandler( ctx, fake_task_id, fake_storage_id, "", 100, fake_executor) perf_collection_handler() # Verify that collect telemetry for deleted storage self.assertEqual(mock_collect_telemetry.call_count, 0)
def test_telemetry_job_scheduling(self, mock_add_job): ctx = context.get_admin_context() telemetry_job = JobHandler(ctx, fake_telemetry_job['id'], fake_telemetry_job['storage_id'], fake_telemetry_job['args'], fake_telemetry_job['interval']) # call telemetry job scheduling telemetry_job.schedule_job(fake_telemetry_job['id']) self.assertEqual(mock_add_job.call_count, 1)
def test_telemetry_removal_success(self, mock_log_error): ctx = context.get_admin_context() telemetry_job = JobHandler(ctx, fake_telemetry_job['id'], fake_telemetry_job['storage_id'], fake_telemetry_job['args'], fake_telemetry_job['interval']) # call telemetry job scheduling telemetry_job.remove_job(fake_telemetry_job['id']) self.assertEqual(mock_log_error.call_count, 0)
def test_distribute_new_job(self, mock_task_update, mock_assign_job, mock_partitioner_start, mock_get_task_executor): ctx = context.get_admin_context() task_distributor = TaskDistributor(ctx) task_distributor.distribute_new_job('fake_task_id') self.assertEqual(mock_assign_job.call_count, 1) self.assertEqual(mock_task_update.call_count, 1) self.assertEqual(mock_partitioner_start.call_count, 1) self.assertEqual(mock_get_task_executor.call_count, 1)
def test_deleted_storage_exception(self, mock_collect_telemetry, mock_failed_task_update): ctx = context.get_admin_context() failed_job_handler = FailedPerformanceCollectionHandler( ctx, 1122, '12c2d52f-01bc-41f5-b73f-7abf6f38a2a6', '', 1234, 2, 1122334400, 1122334800, 'node1') failed_job_handler() # Verify that no action performed for deleted storage failed tasks self.assertEqual(mock_collect_telemetry.call_count, 0) self.assertEqual(mock_failed_task_update.call_count, 0)
def test_failed_job_deleted_storage(self, mock_collect_telemetry, mock_failed_task_update, mock_pause_job): ctx = context.get_admin_context() failed_job_handler = FailedPerformanceCollectionHandler.get_instance( ctx, fake_failed_job_id) failed_job_handler() # Verify that no action performed for deleted storage failed tasks self.assertEqual(mock_collect_telemetry.call_count, 0) self.assertEqual(mock_failed_task_update.call_count, 0)
def test_performance_collection_failure(self, mock_collect_telemetry, mock_failed_task_create): mock_collect_telemetry.return_value = TelemetryTaskStatus. \ TASK_EXEC_STATUS_FAILURE ctx = context.get_admin_context() perf_collection_handler = PerformanceCollectionHandler.get_instance( ctx, fake_task_id) # call performance collection handler perf_collection_handler() # Verify that failed task create is called if collect telemetry fails self.assertEqual(mock_failed_task_create.call_count, 1)
def test_performance_collection_success(self, mock_collect_telemetry, mock_task_update): mock_collect_telemetry.return_value = TelemetryTaskStatus. \ TASK_EXEC_STATUS_SUCCESS ctx = context.get_admin_context() perf_collection_handler = PerformanceCollectionHandler.get_instance( ctx, fake_task_id) # call performance collection handler perf_collection_handler() self.assertEqual(mock_collect_telemetry.call_count, 1) self.assertEqual(mock_task_update.call_count, 1)
def test_performance_collection_deleted_storage(self, mock_collect_telemetry, mock_task_update): mock_collect_telemetry.return_value = TelemetryTaskStatus. \ TASK_EXEC_STATUS_SUCCESS ctx = context.get_admin_context() perf_collection_handler = PerformanceCollectionHandler.get_instance( ctx, fake_task_id) perf_collection_handler() # Verify that collect telemetry and db updated is not called # for deleted storage self.assertEqual(mock_collect_telemetry.call_count, 0) self.assertEqual(mock_task_update.call_count, 0)
def test_failed_job_with_job_already_scheduled(self, mock_failed_get_all, mock_add_job, mock_get_job): # configure to return entry with job id failed_jobs = fake_failed_jobs.copy() failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid() mock_failed_get_all.return_value = failed_jobs # configure to have job in scheduler mock_get_job.return_value = failed_jobs failed_job = FailedJobHandler(context.get_admin_context()) # call failed job scheduling failed_job.remove_failed_job(fake_failed_job['id']) # the job will not be scheduled self.assertEqual(mock_add_job.call_count, 0)
def test_failed_job_scheduling_with_no_task(self, mock_failed_get_all, mock_failed_task_delete, mock_remove_job): # configure to return entry with max retry count failed_jobs = fake_failed_jobs.copy() failed_jobs[0][FailedTask.job_id.name] = uuidutils.generate_uuid() mock_failed_get_all.return_value = failed_jobs failed_job = FailedJobHandler(context.get_admin_context()) # call failed job scheduling failed_job.remove_failed_job(fake_failed_job) # entry get deleted and job get removed self.assertEqual(mock_failed_task_delete.call_count, 1) self.assertEqual(mock_remove_job.call_count, 0)
def test_failed_job_with_max_retry(self, mock_failed_get_all, mock_failed_task_delete, mock_remove_job): # configure to return entry with max retry count failed_jobs = fake_failed_jobs.copy() failed_jobs[0][FailedTask.retry_count.name] = \ TelemetryCollection.MAX_FAILED_JOB_RETRY_COUNT mock_failed_get_all.return_value = failed_jobs failed_job = FailedTelemetryJob(context.get_admin_context()) # call failed job scheduling failed_job() # entry get deleted and job get removed self.assertEqual(mock_failed_task_delete.call_count, 2) self.assertEqual(mock_remove_job.call_count, 1)
def _load_snmp_config(self): """Load snmp config from database when service start.""" ctxt = context.get_admin_context() marker = None finished = False limit = constants.DEFAULT_LIMIT while not finished: alert_sources = db_api.alert_source_get_all(ctxt, marker=marker, limit=limit) for alert_source in alert_sources: snmp_config = dict() snmp_config.update(alert_source) self._add_snmp_config(ctxt, snmp_config) marker = alert_source['storage_id'] if len(alert_sources) < limit: finished = True
def process_alert_info(self, alert): """Fills alert model using driver manager interface.""" ctxt = context.get_admin_context() storage = db.storage_get(ctxt, alert['storage_id']) try: alert_model = self.driver_manager.parse_alert( ctxt, alert['storage_id'], alert) # Fill storage specific info alert_util.fill_storage_attributes(alert_model, storage) except Exception as e: LOG.error(e) raise exception.InvalidResults( "Failed to fill the alert model from driver.") # Export to base exporter which handles dispatch for all exporters self.exporter_manager.dispatch(ctxt, alert_model)
def test_failed_job_success(self, mock_collect_telemetry, mock_failed_task_update, mock_failed_job): mock_collect_telemetry.return_value = TelemetryTaskStatus. \ TASK_EXEC_STATUS_SUCCESS ctx = context.get_admin_context() failed_job_handler = FailedPerformanceCollectionHandler.get_instance( ctx, fake_failed_job_id) # call failed job failed_job_handler() self.assertEqual(mock_failed_job.call_count, 1) mock_failed_task_update.assert_called_once_with( ctx, fake_failed_job_id, { FailedTask.retry_count.name: 1, FailedTask.result.name: TelemetryJobStatus.FAILED_JOB_STATUS_SUCCESS })
def test_process_alert_info_incompletetrap_exception( self, mock_sync_alert, mock_parse_alert, mock_storage, mock_ctxt): """ Mock parse alert for raising exception""" alert = { 'storage_id': 'abcd-1234-56789', 'storage_name': 'storage1', 'vendor': 'fake vendor', 'model': 'fake mode', 'serial_number': 'serial-1234' } mock_ctxt.return_value = context.get_admin_context() mock_storage.return_value = fakes.fake_storage_info() mock_parse_alert.side_effect = exception.IncompleteTrapInformation( 'abcd-1234-56789') alert_processor_inst = self._get_alert_processor() alert_processor_inst.process_alert_info(alert) self.assertTrue(mock_sync_alert.called)
def start(self): """ Initialise the schedulers for periodic job creation """ ctxt = context.get_admin_context() try: # Create a jobs for periodic scheduling periodic_scheduler_job_id = uuidutils.generate_uuid() self.schedule_instance.add_job( telemetry_job.TelemetryJob(ctxt), 'interval', args=[ctxt], seconds=TelemetryCollection.PERIODIC_JOB_INTERVAL, next_run_time=datetime.now(), id=periodic_scheduler_job_id) except Exception as e: # TODO: Currently failure of scheduler is failing task manager # start flow, it is logged and ignored. LOG.error("Failed to initialize periodic tasks, reason: %s.", six.text_type(e)) else: self.schedule_instance.start()
def test_sync_successful(self, mock_masking_view_create, mock_masking_view_update, mock_masking_view_del, mock_masking_views_get_all, mock_list_masking_views, get_lock): cntxt = context.get_admin_context() masking_view_obj = resources.MaskingViewTask( cntxt, 'c5c91c98-91aa-40e6-85ac-37a1d3b32bda') masking_view_obj.sync() self.assertTrue(mock_list_masking_views.called) self.assertTrue(mock_masking_views_get_all.called) self.assertTrue(get_lock.called) # Collect the volume groups from fake_storage fake_storage_obj = fake_storage.FakeStorageDriver() # Add the volume groups to DB mock_list_masking_views.return_value \ = fake_storage_obj.list_masking_views(context) mock_masking_views_get_all.return_value = list() masking_view_obj.sync() self.assertTrue(mock_masking_view_create.called) # Update the volume groups to DB mock_list_masking_views.return_value \ = masking_views_list mock_masking_views_get_all.return_value \ = masking_views_list masking_view_obj.sync() self.assertTrue(mock_masking_view_update.called) # Delete the volume groups to DB mock_list_masking_views.return_value = list() mock_masking_views_get_all.return_value \ = masking_views_list masking_view_obj.sync() self.assertTrue(mock_masking_view_del.called)
def test_sync_successful(self, mock_port_group_create, mock_port_group_update, mock_port_group_del, mock_port_groups_get_all, mock_list_port_groups, get_lock): ctxt = context.get_admin_context() port_group_obj = resources.PortGroupTask( ctxt, 'c5c91c98-91aa-40e6-85ac-37a1d3b32bda') port_group_obj.sync() self.assertTrue(mock_list_port_groups.called) self.assertTrue(mock_port_groups_get_all.called) self.assertTrue(get_lock.called) # Collect the storage host groups from fake_storage fake_storage_obj = fake_storage.FakeStorageDriver() # Add the storage host groups to DB mock_list_port_groups.return_value \ = fake_storage_obj.list_port_groups(context) mock_port_groups_get_all.return_value = list() port_group_obj.sync() self.assertTrue(mock_port_group_create.called) # Update the storage host groups to DB mock_list_port_groups.return_value \ = port_groups_list mock_port_groups_get_all.return_value \ = pg_list port_group_obj.sync() self.assertTrue(mock_port_group_update.called) # Delete the storage host groups to DB mock_list_port_groups.return_value = empty_port_groups_list mock_port_groups_get_all.return_value \ = pg_list port_group_obj.sync() self.assertTrue(mock_port_group_del.called)
def process_alert_info(self, alert): """Fills alert model using driver manager interface.""" ctxt = context.get_admin_context() storage = db.storage_get(ctxt, alert['storage_id']) alert_model = {} try: alert_model = self.driver_manager.parse_alert( ctxt, alert['storage_id'], alert) # Fill storage specific info if alert_model: alert_util.fill_storage_attributes(alert_model, storage) except exception.IncompleteTrapInformation as e: LOG.warn(e) threading.Thread(target=self.sync_storage_alert, args=(ctxt, alert['storage_id'])).start() except Exception as e: LOG.error(e) raise exception.InvalidResults( "Failed to fill the alert model from driver.") # Export to base exporter which handles dispatch for all exporters if alert_model: self.exporter_manager.dispatch(ctxt, alert_model)
def test_process_alert_info_success(self, mock_ctxt, mock_export_model, mock_parse_alert, mock_storage): fake_storage_info = fakes.fake_storage_info() input_alert = { 'storage_id': 'abcd-1234-56789', 'connUnitEventId': 79, 'connUnitName': '000192601409', 'connUnitEventType': constants.EventType.EQUIPMENT_ALARM, 'connUnitEventDescr': 'Diagnostic ' 'event trace triggered.', 'connUnitEventSeverity': 'warning', 'connUnitType': 'storage-subsystem', 'asyncEventSource': 'eventsource1', 'asyncEventCode': '1050', 'asyncEventComponentType': '1051', 'asyncEventComponentName': 'comp1' } expected_alert_model = { 'storage_id': fake_storage_info['id'], 'storage_name': fake_storage_info['name'], 'vendor': fake_storage_info['vendor'], 'model': fake_storage_info['model'], 'serial_number': fake_storage_info['serial_number'], 'location': 'Array id=000192601409,Component ' 'type=location1 ' 'Group,Component name=comp1,Event ' 'source=symmetrix', 'type': input_alert['connUnitEventType'], 'severity': constants.Severity.WARNING, 'category': constants.Category.NOT_SPECIFIED, 'description': input_alert['connUnitEventDescr'], 'resource_type': constants.DEFAULT_RESOURCE_TYPE, 'alert_id': input_alert['asyncEventCode'], 'alert_name': 'SAMPLE_ALERT_NAME', 'sequence_number': 79, 'recovery_advice': 'NA' } mock_storage.return_value = fake_storage_info expected_ctxt = context.get_admin_context() mock_ctxt.return_value = expected_ctxt mock_parse_alert.return_value = fakes.fake_alert_model() alert_processor_inst = self._get_alert_processor() alert_processor_inst.process_alert_info(input_alert) # Verify that model returned by driver is exported mock_export_model.assert_called_once_with(expected_ctxt, [expected_alert_model])
def periodic_tasks(self, raise_on_error=False): """Tasks to be run at a periodic interval.""" ctxt = context.get_admin_context() self.manager.periodic_tasks(ctxt, raise_on_error=raise_on_error)
def test_telemetry_job_scheduling_exception(self, mock_log_error): ctx = context.get_admin_context() telemetry_job = TelemetryJob(ctx) # call telemetry job scheduling telemetry_job(ctx) self.assertEqual(mock_log_error.call_count, 2)
def test_failed_job_scheduling(self, mock_add_job): failed_job = FailedJobHandler(context.get_admin_context()) # call failed job scheduling failed_job.schedule_failed_job(fake_failed_job['id']) self.assertEqual(mock_add_job.call_count, 1)
def get_all_tasks(self, storage_id): filters = {'storage_id': storage_id, 'deleted': False} context = ctxt.get_admin_context() tasks = db.task_get_all(context, filters=filters) failed_tasks = db.failed_task_get_all(context, filters=filters) return tasks, failed_tasks
from unittest import mock from delfin import context, exception from delfin import test from delfin.db import api as db_api from delfin.db.sqlalchemy import api, models ctxt = context.get_admin_context() class TestSIMDBAPI(test.TestCase): @mock.patch('sqlalchemy.create_engine', mock.Mock()) def test_register_db(self): db_api.register_db() def test_get_session(self): api.get_session() def test_get_engine(self): api.get_engine() @mock.patch('delfin.db.sqlalchemy.api.get_session') def test_storage_get(self, mock_session): fake_storage = {} mock_session.return_value.__enter__.return_value.query.return_value \ = fake_storage result = db_api.storage_get(ctxt, 'c5c91c98-91aa-40e6-85ac-37a1d3b32bda') assert len(result) == 0 @mock.patch('delfin.db.sqlalchemy.api.get_session')