예제 #1
0
 def test_checkpoint_fail001_bad_restart_mode(self):
     ''' Fail 001: Check bad restart mode caught when overriding on get start event '''
     teal = Teal('data/checkpoint_test/noop_monitor.conf','stderr',msgLevel=self.msglevel, commit_checkpoints=False)
     checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
     self.assertRaisesTealError(ConfigurationError, 'Unrecognized restart mode specified: bad', checkpoint_mgr.get_starting_event_rec_id, override_restart_mode='bad')
     teal.shutdown()
     return
예제 #2
0
 def _add_journal(self, json_file):
     ''' Load the alert DB from the journal JSON file '''
     self.teal = Teal('data/tllsalert_test/test.conf','stderr',msgLevel=self.msglevel)
     # Alerts
     ja = Journal('temp_journal', file=json_file)
     ja.insert_in_db(truncate=False, no_delay=True)
     self.teal.shutdown()
예제 #3
0
def get_table_date_time_pattern():
    time_pattern = os.environ.get(TEAL_TEST_DB_TIME_PATTERN, None)

    if time_pattern is None:
        myteal = Teal('data/common/dbinterfaceonly.conf', 'stderr', msgLevel='warning')
        dbi = get_service(SERVICE_DB_INTERFACE)
        if isinstance(dbi, db_interface_pyodbc.DBInterfacePyODBC):
            if isinstance(dbi.sql_generator, db_interface_pyodbc.SQLGeneratorDB2):
                time_pattern = '%Y-%m-%d %H:%M:%S.%f'
            elif isinstance(dbi.sql_generator, db_interface_pyodbc.SQLGeneratorMySQL):
                time_pattern = '%Y-%m-%d %H:%M:%S'
            else:
                print 'Unknown generator type ... unable to determine pattern'
        else:
            print 'Unknown DB Interface type ... unable to determine pattern'
        myteal.shutdown()
    return time_pattern
예제 #4
0
    def setUp(self):
        t = Teal('data/tllsalert_test/test.conf',data_only=True)
        
        teal_path = registry.get_service(registry.TEAL_ROOT_DIR)
        
        global LSALERT
        LSALERT = os.path.join(teal_path,'bin/tllsalert')
        
        t.shutdown()
        
        self.prepare_db()
        self._add_journal('data/tllsalert_test/events_001.json')
        self._add_journal('data/tllsalert_test/alerts_001.json')

        tmp_data_dir = os.path.join(os.environ.get('TEAL_ROOT_DIR','/opt/teal'),'data')
        self.teal_data_dir = self.force_env('TEAL_DATA_DIR', tmp_data_dir)
        
        self.time_pattern = get_table_date_time_pattern()
예제 #5
0
 def test_checkpoint_fail003_deleted_checkpoint(self):
     ''' Fail 003: Check bad restart mode caught when overriding on get start event '''
     teal = Teal('data/checkpoint_test/noop_monitor.conf','stderr',msgLevel=self.msglevel, commit_checkpoints=False)
     #checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
     k1_ckpt = EventCheckpoint('deltest1')
     k1_ckpt.set_checkpoint(123, 'data')
     k1_ckpt.delete()
     # Make sure can't get at stuff
     self.assertEqual(k1_ckpt.get_status(), CHECKPOINT_STATUS_DELETED)
     self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
     k1_ckpt.set_status(CHECKPOINT_STATUS_RUNNING)
     self.assertEqual(k1_ckpt.get_status(), CHECKPOINT_STATUS_DELETED)
     self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
     k1_ckpt.set_checkpoint(555, 'moredata')
     self.assertEqual(k1_ckpt.get_status(), CHECKPOINT_STATUS_DELETED)
     self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
     teal.shutdown()
     return
예제 #6
0
def truncate_all_teal_tables():  
    ''' Truncate all of the teal tables ''' 
    myteal = Teal('data/common/dbinterfaceonly.conf', 'stderr', msgLevel='warning')
    dbi = get_service(SERVICE_DB_INTERFACE)
    cnxn = dbi.get_connection()
    cursor = cnxn.cursor()
    # ORDER IS IMPORTANT!
    dbi.truncate(cursor, db_interface.TABLE_CHECKPOINT)
    cnxn.commit()
    dbi.truncate(cursor, db_interface.TABLE_ALERT2ALERT)
    cnxn.commit()
    dbi.truncate(cursor, db_interface.TABLE_ALERT2EVENT)
    cnxn.commit()
    dbi.delete(cursor, db_interface.TABLE_EVENT_LOG)
    cnxn.commit()
    # Must close all alerts before they can be removed
    #   don't have to worry about relationships because they are all gone
    dbi.update(cursor, ['state'], db_interface.TABLE_ALERT_LOG, parms=(ALERT_STATE_CLOSED,))
    cnxn.commit()
    dbi.delete(cursor, db_interface.TABLE_ALERT_LOG)
    cnxn.commit()
    cnxn.close()
    myteal.shutdown()
    return
예제 #7
0
class TestTllsalert(TealTestCase):

    def setUp(self):
        t = Teal('data/tllsalert_test/test.conf',data_only=True)
        
        teal_path = registry.get_service(registry.TEAL_ROOT_DIR)
        
        global LSALERT
        LSALERT = os.path.join(teal_path,'bin/tllsalert')
        
        t.shutdown()
        
        self.prepare_db()
        self._add_journal('data/tllsalert_test/events_001.json')
        self._add_journal('data/tllsalert_test/alerts_001.json')

        tmp_data_dir = os.path.join(os.environ.get('TEAL_ROOT_DIR','/opt/teal'),'data')
        self.teal_data_dir = self.force_env('TEAL_DATA_DIR', tmp_data_dir)
        
        self.time_pattern = get_table_date_time_pattern()

    def tearDown(self):
        self.restore_env('TEAL_DATA_DIR', self.teal_data_dir)
    
    def _add_journal(self, json_file):
        ''' Load the alert DB from the journal JSON file '''
        self.teal = Teal('data/tllsalert_test/test.conf','stderr',msgLevel=self.msglevel)
        # Alerts
        ja = Journal('temp_journal', file=json_file)
        ja.insert_in_db(truncate=False, no_delay=True)
        self.teal.shutdown()
   
    def test_tllsalert(self):
        ''' Test associations returned '''
        # NOTE:
        # CMVC does not allow checking in of very long lines
        # use + to separate long strings
        tmp_good_msg = '1: Alert US {0} C:MB-SL1-ET1-PT2    3: Alert US {1} C:MB-SL1-ET1-PT2    4: Alert US {2} C:MB-SL1-ET1-PT2    5: Alert 03 {3} C:MB-SL1-ET1-PT2    6: Alert US {4} C:MB-SL1-ET1-PT1'
        tmp_dts = apply_time_pattern(['2010-07-22 16:14:13.437000', '2010-07-22 16:14:21.437000', '2010-07-22 16:14:25.437000', '2010-07-22 16:14:26.453000', '2010-07-22 16:14:33.468000'], self.time_pattern)
        tmp_good_msg = tmp_good_msg.format(*tmp_dts)
        self.assertCmdWorks([LSALERT], exp_good_msg=tmp_good_msg)
        tmp_good_msg = \
          '{{"associations": {{"D:A": "[2]", "C:E": "[]", "C:A": "[]", "S:A": "[]", "S:E": "[1,2]"}}, "severity": "W", "state": 1, "rec_id": 1, "creation_time": "{0}", "reason": "no value", "alert_id": "Alert US", "event_loc": "MB-SL1-ET1-PT2", "recommendation": "Recommend doing something", "src_name": "AnalyzerTest057a", "raw_data": null, "event_loc_type": "C", "urgency": "N", "fru_loc": null}}'  \
        + '{{"associations": {{"D:A": "[]", "C:E": "[]", "C:A": "[]", "S:A": "[4]", "S:E": "[3]"}}, "severity": "W", "state": 1, "rec_id": 3, "creation_time": "{1}", "reason": "no value", "alert_id": "Alert US", "event_loc": "MB-SL1-ET1-PT2", "recommendation": "Recommend doing something", "src_name": "AnalyzerTest057a", "raw_data": null, "event_loc_type": "C", "urgency": "N", "fru_loc": null}}' \
        + '{{"associations": {{"D:A": "[]", "C:E": "[4]", "C:A": "[]", "S:A": "[]", "S:E": "[]"}}, "severity": "W", "state": 1, "rec_id": 4, "creation_time": "{2}", "reason": "no value", "alert_id": "Alert US", "event_loc": "MB-SL1-ET1-PT2", "recommendation": "Recommend doing something", "src_name": "AnalyzerTest057a", "raw_data": null, "event_loc_type": "C", "urgency": "N", "fru_loc": null}}'  \
        + '{{"associations": {{"D:A": "[]", "C:E": "[5]", "C:A": "[]", "S:A": "[]", "S:E": "[]"}}, "severity": "E", "state": 1, "rec_id": 5, "creation_time": "{3}", "reason": "reason from config1", "alert_id": "Alert 03", "event_loc": "MB-SL1-ET1-PT2", "recommendation": "try again", "src_name": "AnalyzerTest057a", "raw_data": "This is another test", "event_loc_type": "C", "urgency": "I", "fru_loc": "fru_loc2"}}'  \
        + '{{"associations": {{"D:A": "[]", "C:E": "[]", "C:A": "[]", "S:A": "[]", "S:E": "[6]"}}, "severity": "W", "state": 1, "rec_id": 6, "creation_time": "{4}", "reason": "no value", "alert_id": "Alert US", "event_loc": "MB-SL1-ET1-PT1", "recommendation": "Recommend doing something", "src_name": "AnalyzerTest057a", "raw_data": null, "event_loc_type": "C", "urgency": "N", "fru_loc": null}}'
        tmp_dts = apply_time_pattern(['2010-07-22 16:14:13.437000', '2010-07-22 16:14:21.437000', '2010-07-22 16:14:25.437000', '2010-07-22 16:14:26.453000', '2010-07-22 16:14:33.468000'], self.time_pattern)
        tmp_good_msg = tmp_good_msg.format(*tmp_dts)
        self.assertCmdWorks([LSALERT,'--with-assoc','--format=json'], exp_good_msg= tmp_good_msg)
        tmp_good_msg = 'rec_id,alert_id,creation_time,severity,urgency,event_loc,event_loc_type,fru_loc,recommendation,reason,src_name,state,raw_data\r\n1,Alert US,{0},W,N,MB-SL1-ET1-PT2,C,,Recommend doing something,no value,AnalyzerTest057a,1,\r\n3,Alert US,{1},W,N,MB-SL1-ET1-PT2,C,,Recommend doing something,no value,AnalyzerTest057a,1,\r\n4,Alert US,{2},W,N,MB-SL1-ET1-PT2,C,,Recommend doing something,no value,AnalyzerTest057a,1,\r\n5,Alert 03,{3},E,I,MB-SL1-ET1-PT2,C,fru_loc2,try again,reason from config1,AnalyzerTest057a,1,This is another test\r\n6,Alert US,{4},W,N,MB-SL1-ET1-PT1,C,,Recommend doing something,no value,AnalyzerTest057a,1,'
        tmp_dts = apply_time_pattern(['2010-07-22 16:14:13.437000', '2010-07-22 16:14:21.437000', '2010-07-22 16:14:25.437000', '2010-07-22 16:14:26.453000', '2010-07-22 16:14:33.468000'], self.time_pattern)
        tmp_good_msg = tmp_good_msg.format(*tmp_dts)
        self.assertCmdWorks([LSALERT,'-f','csv'], exp_good_msg=tmp_good_msg)
        
        tmp_good_msg = '===================================================\r\nrec_id : 1\r\nalert_id : Alert US\r\ncreation_time : {0}\r\nseverity : W\r\nurgency : N\r\nevent_loc : MB-SL1-ET1-PT2\r\nevent_loc_type : C\r\nfru_loc : None\r\nrecommendation : Recommend doing something\r\nreason : no value\r\nsrc_name : AnalyzerTest057a\r\nstate : 1\r\nraw_data : None\r\n===================================================\r\n' \
        + 'rec_id : 3\r\nalert_id : Alert US\r\ncreation_time : {1}\r\nseverity : W\r\nurgency : N\r\nevent_loc : MB-SL1-ET1-PT2\r\nevent_loc_type : C\r\nfru_loc : None\r\nrecommendation : Recommend doing something\r\nreason : no value\r\nsrc_name : AnalyzerTest057a\r\nstate : 1\r\nraw_data : None\r\n===================================================\r\n' \
        + 'rec_id : 4\r\nalert_id : Alert US\r\ncreation_time : {2}\r\nseverity : W\r\nurgency : N\r\nevent_loc : MB-SL1-ET1-PT2\r\nevent_loc_type : C\r\nfru_loc : None\r\nrecommendation : Recommend doing something\r\nreason : no value\r\nsrc_name : AnalyzerTest057a\r\nstate : 1\r\nraw_data : None\r\n===================================================\r\n'  \
        + 'rec_id : 5\r\nalert_id : Alert 03\r\ncreation_time : {3}\r\nseverity : E\r\nurgency : I\r\nevent_loc : MB-SL1-ET1-PT2\r\nevent_loc_type : C\r\nfru_loc : fru_loc2\r\nrecommendation : try again\r\nreason : reason from config1\r\nsrc_name : AnalyzerTest057a\r\nstate : 1\r\nraw_data : This is another test\r\n===================================================\r\nrec_id : 6\r\nalert_id : Alert US\r\ncreation_time : {4}\r\nseverity : W\r\nurgency : N\r\nevent_loc : MB-SL1-ET1-PT1\r\nevent_loc_type : C\r\nfru_loc : None\r\nrecommendation : Recommend doing something\r\nreason : no value\r\nsrc_name : AnalyzerTest057a\r\nstate : 1\r\nraw_data : None'
        tmp_dts = apply_time_pattern(['2010-07-22 16:14:13.437000', '2010-07-22 16:14:21.437000', '2010-07-22 16:14:25.437000', '2010-07-22 16:14:26.453000', '2010-07-22 16:14:33.468000'], self.time_pattern)
        tmp_good_msg = tmp_good_msg.format(*tmp_dts)
        self.assertCmdWorks([LSALERT,'-f','text'], exp_good_msg=tmp_good_msg)
        
        tmp_good_msg = '1: Alert US {0} C:MB-SL1-ET1-PT2    3: Alert US {1} C:MB-SL1-ET1-PT2    6: Alert US {2} C:MB-SL1-ET1-PT1'
        tmp_dts = apply_time_pattern(['2010-07-22 16:14:13.437000', '2010-07-22 16:14:21.437000', '2010-07-22 16:14:33.468000'], self.time_pattern)
        tmp_good_msg = tmp_good_msg.format(*tmp_dts)
        self.assertCmdWorks([LSALERT,'-q','rec_id=1,3,6'], exp_good_msg=tmp_good_msg)
예제 #8
0
 def setUp(self):
     self.teal = Teal('data/common/configurationtest.conf', 'stderr', msgLevel=self.msglevel, commit_alerts=False, commit_checkpoints=False)
예제 #9
0
class ExtensionDataTest(TealTestCase):

    def setUp(self):
        self.teal = Teal('data/common/configurationtest.conf', 'stderr', msgLevel=self.msglevel, commit_alerts=False, commit_checkpoints=False)

    def tearDown(self):
        self.teal.shutdown()

    def testInvalidFormat(self):
        ''' Test invalid format handled '''
        raw_data = 'This is raw data'
        fmt = 0x4142434400000000
        self.assertRaises(IOError, ExtensionData, fmt, None, raw_data)
         
    def testInvalidTable(self):
        ''' Test invalid table handled '''
        raw_data = 'This is raw data'
        fmt = 0x5445535400001234
        self.assertRaises(ValueError, ExtensionData, fmt, None, raw_data)
    
    def testTableName(self):
        ''' Test table name generation '''
        # This is to circumvent the need to start up TEAL to set the variable
        db_interface.TABLE_TEMPLATE = 'x_{0}'
        # Tests
        fmt_raw_data = 0x5445535400001234
        fmt_db_data1 = 0x5445535480001234
        fmt_db_data2 = 0x5445535480F812AB
        self.assertFalse(extdata_fmt2table_name(fmt_raw_data))
        self.assertEqual(extdata_fmt2table_name(fmt_db_data1),'x_TEST_0_1234')
        self.assertEqual(extdata_fmt2table_name(fmt_db_data2),'x_TEST_F8_12AB')
        
    def testRawDataOnly(self):
        ''' Test raw data only '''
        raw_data = 'This is raw data'
        ed = ExtensionData(0, None, raw_data)
        
        self.assertEqual(raw_data, ed['raw_data'])
        self.assertEqual(len(ed.keys()), 1)
        self.assertEqual(0, ed.get_format())
    
    def testString(self):
        ''' Test string data ''' 
        raw_data = "0123456789ABCDEFGH"
        fmt = 0x5445535400000001
        ed = ExtensionData(fmt,None, raw_data)
        
        self.assertEqual(raw_data, ed['raw_data'])
        self.assertEqual(raw_data, ed['info'])
        self.assertEqual(len(ed.keys()), 2)
        self.assertEqual(fmt, ed.get_format())

    def testNumbers(self):   
        ''' Test numbers are handled correctly '''     
        s = struct.Struct("i")
        fmt = 0x5445535400000004
        raw_data = s.pack(0x12345678)        
        ed = ExtensionData(fmt, None, raw_data)
        #print hex(ed['number'])
        self.assertEqual(0x12345678,ed['number'])
                
        s = struct.Struct("q")
        fmt = 0x5445535400000005
        raw_data = s.pack(0x123456789ABCDEF0)        
        ed = ExtensionData(fmt,None, raw_data)
        #print hex(ed['number'])
        self.assertEqual(0x123456789ABCDEF0,ed['number'])
                
    def testLocation(self):
        ''' Test location is handled correctly '''
        s = struct.Struct("3p256p")
        fmt = 0x5445535400000003
        loc = Location('C','MB-SL3-ET1-PT2')
        
        raw_data = s.pack(loc.get_id(),loc.get_location())        
        ed = ExtensionData(fmt,None, raw_data)
        
        self.assertEqual(raw_data,ed['raw_data'])
        self.assertEqual(loc,ed['neighbor'])
        self.assertEqual(len(ed.keys()),2)
        self.assertEqual(fmt,ed.get_format())
        
    def testToFromDict(self):
        ''' Test to and from dict is handled correctly '''
        s = struct.Struct("3p256p")
        fmt = 0x5445535400000003
        loc = Location('C','MB-SL3-ET1-PT2')
        
        raw_data = s.pack(loc.get_id(),loc.get_location())        
        ed = ExtensionData(fmt,None, raw_data)
        
        self.assertEqual(raw_data,ed['raw_data'])
        self.assertEqual(loc,ed['neighbor'])
        self.assertEqual(len(ed.keys()),2)
        self.assertEqual(fmt,ed.get_format())
        tmp_dict = ed.write_to_dictionary()
        new_ed = ExtensionData(fmt, None, None, tmp_dict)
        self.assertEqual(raw_data, new_ed['raw_data'])
        self.assertEqual(loc, new_ed['neighbor'])
        self.assertEqual(len(new_ed.keys()), 2)
        self.assertEqual(fmt, new_ed.get_format())
        return
예제 #10
0
 def setUp(self):        
     self.teal = Teal('data/checkpoint_test/noop_monitor.conf','stderr',msgLevel=self.msglevel, commit_checkpoints=False)
     return 
예제 #11
0
class TealCheckpointMgrTestExecution(TealTestCase):
    
    def setUp(self):        
        self.teal = Teal('data/checkpoint_test/noop_monitor.conf','stderr',msgLevel=self.msglevel, commit_checkpoints=False)
        return 

    def tearDown(self):
        self.teal.shutdown()
        return 
        
    def test_checkpoint001_no_db_init(self):
        ''' 001: Check that the checkpoint service is correctly setup and available '''
        checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
        self.assertNotEqual(checkpoint_mgr, None)
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 0)
        #print str(checkpoint_mgr)
        return 
        
    def test_checkpoint002_no_db_add_one(self):
        ''' 002: Check that a single event checkpoint works '''
        checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
        self.assertNotEqual(checkpoint_mgr, None)
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 0)
        # Create a new checkpoint 
        k_ckpt = EventCheckpoint('test')
        # Validate it got created correctly 
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 1)
        self.assertEqual(k_ckpt.name, 'test')
        self.assertEqual(k_ckpt.get_status(), CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k_ckpt.status, CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k_ckpt.start_rec_id, None)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None,'recovery'))
        # Set the status and check it got set
        k_ckpt.set_status(CHECKPOINT_STATUS_FAILED)
        self.assertEqual(k_ckpt.get_status(), CHECKPOINT_STATUS_FAILED)
        # Set the checkpoint and check it got set
        k_ckpt.set_checkpoint(345)
        self.assertEqual(k_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k_ckpt.start_rec_id, 345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (345, 'recovery'))
        k_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(k_ckpt.get_checkpoint(), (123, 'data'))
        self.assertEqual(k_ckpt.start_rec_id, 123)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (123, 'recovery'))
        # Delete the checkpoint and check 
        k_ckpt.delete()
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 0)
        self.assertEqual(k_ckpt.status, CHECKPOINT_STATUS_DELETED)
        self.assertEqual(k_ckpt.start_rec_id, None)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None, 'recovery'))
        return 
        
    def test_checkpoint003_no_db_add_two(self):
        ''' 003: Check that two event checkpoints work '''
        checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
        k1_ckpt = EventCheckpoint('test1')
        k2_ckpt = EventCheckpoint('test2')
        # Validate it got created correctly 
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 2)
        self.assertEqual(k1_ckpt.name, 'test1')
        self.assertEqual(k1_ckpt.get_status(), CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k1_ckpt.status, CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k1_ckpt.start_rec_id, None)
        self.assertEqual(k2_ckpt.name, 'test2')
        self.assertEqual(k2_ckpt.get_status(), CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k2_ckpt.status, CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.start_rec_id, None)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (None, 'lastproc'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (None, 'recovery'))
        # Set the checkpoint and check it got set
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k1_ckpt.start_rec_id, 345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(k2_ckpt.get_checkpoint(), (123, 'data'))
        self.assertEqual(k2_ckpt.start_rec_id, 123)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (123, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (123, 'data'))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (123, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (123, 'data'))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(123, 'data')
        # Delete the checkpoint and check 
        k2_ckpt.delete()
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 1)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k2_ckpt.status, CHECKPOINT_STATUS_DELETED)
        self.assertEqual(k2_ckpt.start_rec_id, None)
        k1_ckpt.delete()
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 0)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (None, 'lastproc'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (None, 'recovery'))
        return 
    
    def test_checkpoint004_no_db_cb_test(self):
        ''' 004: Check that the event checkpoint callback gets called '''
        checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
        k1_ckpt = EventCheckpoint('test1')
        k2_ckpt = EventCheckpoint('test2')
        # Validate it got created correctly 
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 2)
        self.assertEqual(k1_ckpt.name, 'test1')
        self.assertEqual(k1_ckpt.get_status(), CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k1_ckpt.status, CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k1_ckpt.start_rec_id, None)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (None, 'lastproc'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (None, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id('recovery'), (None, 'recovery'))
        self.assertEqual(k2_ckpt.name, 'test2')
        self.assertEqual(k2_ckpt.get_status(), CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k2_ckpt.status, CHECKPOINT_STATUS_RUNNING)
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.start_rec_id, None)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (None, 'lastproc'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (None, 'recovery'))
        # Set the checkpoint and check it got set
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k1_ckpt.start_rec_id, 345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(1, 'data')
        self.assertEqual(k2_ckpt.get_checkpoint(), (1, 'data'))
        self.assertEqual(k2_ckpt.start_rec_id, 1)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (1, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(1, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(1, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(1, 'data')
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (1, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (1, 'data'))
        k1_ckpt.set_checkpoint(345)
        k2_ckpt.set_checkpoint(1, 'data')
        # Delete the checkpoint and check 
        k2_ckpt.delete()
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 1)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        self.assertEqual(k2_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (None, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (345, 'recovery'))
        self.assertEqual(k1_ckpt.get_checkpoint(), (345, None))
        k1_ckpt.set_checkpoint(345)
        self.assertEqual(k2_ckpt.status, CHECKPOINT_STATUS_DELETED)
        self.assertEqual(k2_ckpt.start_rec_id, None)
        k1_ckpt.delete()
        self.assertEqual(len(checkpoint_mgr.event_checkpoints), 0)
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(), (None, 'recovery'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (None, 'lastproc'))
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (None, 'recovery'))
        return 
    
    
    def test_checkpoint005_no_db_cb_test(self):
        ''' 005: Check that the checkpoint state doesn't matter for recovery '''
        checkpoint_mgr = get_service(SERVICE_CHECKPOINT_MGR)
        k_list = []
        for i in xrange(0,5):
            k_list.append(EventCheckpoint('test' + str(i)))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        k_force = [ (CHECKPOINT_STATUS_SHUTDOWN, 300, None), 
                    (CHECKPOINT_STATUS_FAILED, 200, None),
                    (CHECKPOINT_STATUS_SHUTDOWN, 3, None),
                    (CHECKPOINT_STATUS_RUNNING, 10, None),
                    (CHECKPOINT_STATUS_SHUTDOWN, 40, None)]
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        for k in k_list:
            print str(k)
        # Check each override option 
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (300, 'lastproc'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (3, 'recovery'))
        for k in k_list:
            if k.name == 'test1':
                self.assertEqual(k.get_status(), CHECKPOINT_STATUS_FAILED)  # Failed aren't touched when recovery mode
            else:
                self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        # Different status            
        k_force = [ (CHECKPOINT_STATUS_SHUTDOWN, 300, None), 
                    (CHECKPOINT_STATUS_SHUTDOWN, 200, None),
                    (CHECKPOINT_STATUS_SHUTDOWN, 345, None),
                    (CHECKPOINT_STATUS_SHUTDOWN, 10, None),
                    (CHECKPOINT_STATUS_SHUTDOWN, 40, None)]
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        for k in k_list:
            print str(k)
        # Check each override option 
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='now'), (None, 'now'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='begin'), (0, 'begin'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='lastproc'), (345, 'lastproc'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        self.assertEqual(checkpoint_mgr.get_starting_event_rec_id(override_restart_mode='recovery'), (10, 'recovery'))
        for k in k_list: 
            self.assertEqual(k.get_status(), CHECKPOINT_STATUS_RUNNING)
        for i in xrange(0,5):
            k_list[i].set_status(k_force[i][0])
            k_list[i].set_checkpoint(k_force[i][1], k_force[i][2])
        return