def test_09_verify_object_table_query(self):
     '''
     This test verifies the Object Table query.
     '''
     logging.info('%%% test_09_verify_object_table_query %%%')
     vizd_obj = self.useFixture(
         AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
     assert vizd_obj.verify_on_setup()
     collectors = [vizd_obj.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture('contrail-control',
                          collectors,
                          logging,
                          None,
                          node_type='Control'))
     assert generator_obj.verify_on_setup()
     msg_types = generator_obj.send_all_sandesh_types_object_logs(
         socket.gethostname())
     assert vizd_obj.verify_object_table_sandesh_types(
         'ObjectBgpRouter', socket.gethostname(), msg_types)
     # Check if ObjectId's can be fetched properly for ObjectTable queries
     vm_generator_obj = self.useFixture(
         GeneratorFixture("contrail-vrouter-agent", collectors, logging,
                          vizd_obj.get_opserver_port()))
     assert vm_generator_obj.verify_on_setup()
     assert vizd_obj.verify_object_table_objectid_values(
         'ObjectBgpRouter', [socket.gethostname()])
     vm1_name = 'vm1'
     vm2_name = 'vm2'
     vm_generator_obj.send_vm_uve(vm_id=vm1_name, num_vm_ifs=2, msg_count=2)
     vm_generator_obj.send_vm_uve(vm_id=vm2_name, num_vm_ifs=2, msg_count=2)
     assert vizd_obj.verify_object_table_objectid_values(
         'ObjectVMTable', [vm1_name, vm2_name])
    def test_02_vm_uve_with_password(self):
        '''
        This test starts redis, vizd, opserver, qed, and a python generator
        that simulates vrouter and sends UveVirtualMachineAgentTrace messages.
        Then it checks that the VM UVE (via redis) can be accessed from
        opserver.
        '''
        logging.info("*** test_02_vm_uve_with_password ***")
        if AnalyticsUveTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(
            AnalyticsFixture(logging,
                             builddir,
                             -1,
                             0,
                             redis_password='******'))
        assert vizd_obj.verify_on_setup()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("contrail-vrouter-agent", collectors, logging,
                             vizd_obj.get_opserver_port()))
        assert generator_obj.verify_on_setup()
        generator_obj.send_vm_uve(vm_id='abcd', num_vm_ifs=5, msg_count=5)
        assert generator_obj.verify_vm_uve(vm_id='abcd',
                                           num_vm_ifs=5,
                                           msg_count=5)
        return True
示例#3
0
    def test_15_verify_introspect_ssl(self):
        '''
        This test enables introspect ssl and starts all the analytics
        generators in the AnalyticsFixture and verifies that the introspect
        port is accessible with https.
        '''
        logging.info('%%% test_15_verify_introspect_ssl %%%')
        sandesh_cfg = {
            'sandesh_keyfile': builddir+'/opserver/test/data/ssl/server-privkey.pem',
            'sandesh_certfile': builddir+'/opserver/test/data/ssl/server.pem',
            'sandesh_ca_cert': builddir+'/opserver/test/data/ssl/ca-cert.pem',
            'introspect_ssl_enable': 'True'
        }
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir,
                             self.__class__.cassandra_port,
                             sandesh_config=sandesh_cfg))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()

        # remove the config from vizd so that it tries to access introspect
        # with http, it should fail
        vizd_obj.set_sandesh_config(None)
        assert not vizd_obj.verify_collector_gen(vizd_obj.collectors[0])

        # start a python generator with introspect_ssl_enable = True
        # and verify that its introspect page is accessible.
        vizd_obj.set_sandesh_config(sandesh_cfg)
        test_gen = self.useFixture(
            GeneratorFixture("contrail-test-generator",
                vizd_obj.get_collectors(), logging,
                vizd_obj.get_opserver_port(), sandesh_config=sandesh_cfg))
        assert test_gen.verify_on_setup()
示例#4
0
    def test_03_vm_uve(self):
        '''
        This test starts redis, vizd, opserver, qed, and a python generator
        that simulates vrouter and sends UveVirtualMachineAgentTrace messages.
        It uses the test class' cassandra instance. Then it checks that the
        VM UVE (via redis) can be accessed from opserver.
        '''
        logging.info("*** test_03_vm_uve ***")
        if AnalyticsTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(AnalyticsFixture(logging, builddir, 0))
        assert vizd_obj.verify_on_setup()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("VRouterAgent", collectors, logging,
                             vizd_obj.get_opserver_port()))
        assert generator_obj.verify_on_setup()
        generator_obj.send_vm_uve(vm_id='abcd', num_vm_ifs=5, msg_count=5)
        assert generator_obj.verify_vm_uve(vm_id='abcd',
                                           num_vm_ifs=5,
                                           msg_count=5)
        # Delete the VM UVE and verify that the deleted flag is set
        # in the UVE cache
        generator_obj.delete_vm_uve('abcd')
        assert generator_obj.verify_vm_uve_cache(vm_id='abcd', delete=True)
        # Add the VM UVE with the same vm_id and verify that the deleted flag
        # is cleared in the UVE cache
        generator_obj.send_vm_uve(vm_id='abcd', num_vm_ifs=5, msg_count=5)
        assert generator_obj.verify_vm_uve_cache(vm_id='abcd')
        assert generator_obj.verify_vm_uve(vm_id='abcd',
                                           num_vm_ifs=5,
                                           msg_count=5)
        return True
示例#5
0
 def test_05_fieldname_query(self):
     '''
     This test starts redis,vizd,opserver and qed
     It uses the test class' cassandra instance
     It then queries the stats table for messagetypes
     and objecttypes
     '''
     logging.info("%%% test_05_fieldname_query %%%")
     vizd_obj = self.useFixture(
         AnalyticsFixture(logging, builddir,
                          self.__class__.cassandra_port))
     assert vizd_obj.verify_on_setup()
     assert vizd_obj.verify_collector_obj_count()
     collectors = [vizd_obj.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture("VRouterAgent", collectors,
                          logging, vizd_obj.get_opserver_port(),
                          sandesh_config = {'system_logs_rate_limit' : 10}))
     assert generator_obj.verify_on_setup()
     # Messagetype is stored in the FieldNames.fields table only
     # for objectlog and systemlog. For systemlog it is stored as
     # MessageTable:Messagetype and for objectlog it is stored as
     # <ObjectTableName>:Messagetype. Send only systemlog
     logging.info("Starting sandesh types gen " + str(UTCTimestampUsec()))
     generator_obj.send_sandesh_types_object_logs(socket.gethostname(),
         types=[SandeshType.SYSTEM])
     logging.info("Ending sandesh types gen " + str(UTCTimestampUsec()))
     # Sends 2 different vn uves in 1 sec spacing
     logging.info("Starting intervn gen " + str(UTCTimestampUsec()))
     generator_obj.generate_intervn()
     logging.info("Ending intervn gen " + str(UTCTimestampUsec()))
     assert vizd_obj.verify_fieldname_messagetype()
     assert vizd_obj.verify_fieldname_table()
     return True
    def test_16_rbac(self, mock_is_role_cloud_admin,
                     mock_get_resource_list_from_uve_type):
        logging.info("%%% test_16_rbac %%%")
        if AnalyticsTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        collectors = [vizd_obj.get_collector()]
        start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
        generator_obj = self.useFixture(
            GeneratorFixture("contrail-vrouter-agent", collectors, logging,
                             vizd_obj.get_opserver_port(), start_time))
        assert generator_obj.verify_on_setup()
        logging.info("Starting intervn gen " + str(UTCTimestampUsec()))
        # send 2 vn UVEs
        generator_obj.generate_intervn()
        token = 'user:admin'
        mock_is_role_cloud_admin.return_value = True
        mock_get_resource_list_from_uve_type.return_value = None
        # for admin role, there should be 2 vn uves
        uves = vizd_obj.get_opserver_vns(token)
        assert (len(uves) == 2)
        mock_is_role_cloud_admin.return_value = False
        mock_get_resource_list_from_uve_type.return_value = set(
            ["default-domain:vn0"])
        token = 'user:default-domain:vn0'
        # for non-admin role, there should be 1 vn uve
        uves = vizd_obj.get_opserver_vns(token)
        assert (len(uves) == 1)
        return True
 def test_05_fieldname_query(self):
     '''
     This test starts redis,vizd,opserver and qed
     It uses the test class' cassandra instance
     It then queries the stats table for messagetypes
     and objecttypes
     '''
     logging.info("%%% test_05_fieldname_query %%%")
     start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
     self._update_analytics_start_time(start_time)
     vizd_obj = self.useFixture(
         AnalyticsFixture(logging, builddir,
                          self.__class__.redis_port,
                          self.__class__.cassandra_port))
     assert vizd_obj.verify_on_setup()
     assert vizd_obj.verify_collector_obj_count()
     collectors = [vizd_obj.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture("VRouterAgent", collectors,
                          logging, vizd_obj.get_opserver_port()))
     assert generator_obj.verify_on_setup()
     # Sends 2 different vn uves in 1 sec spacing
     generator_obj.generate_intervn()
     assert vizd_obj.verify_fieldname_messagetype()
     assert vizd_obj.verify_fieldname_table()
     return True
 def test_04_session_query(self):
     '''
     This test starts redis, vizd, opserver and qed
     It uses the test class' cassandra instance
     Then it sends session stats to the collector
     and checks if session stats can be accessed from QE
     '''
     logging.info("%%% test_04_session_query  %%%")
     if AnalyticsTest._check_skip_test() is True:
         return True
     vizd_obj = self.useFixture(
         AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
     assert vizd_obj.verify_on_setup()
     assert vizd_obj.verify_collector_obj_count()
     # set the start time in analytics db 1 hour earlier than
     # the current time. For flow series test, we need to create
     # flow samples older than the current time.
     start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
     self._update_analytics_start_time(start_time)
     collectors = [vizd_obj.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture("contrail-vrouter-agent", collectors, logging,
                          vizd_obj.get_opserver_port(), start_time))
     assert generator_obj.verify_on_setup()
     generator_obj.generate_session_samples()
     assert vizd_obj.verify_session_samples(generator_obj)
     assert vizd_obj.verify_session_table(generator_obj)
     assert vizd_obj.verify_session_series_aggregation_binning(
         generator_obj)
     return True
示例#9
0
    def test_06_intervn_query(self):
        '''
        This test starts redis,vizd,opserver and qed
        It uses the test class' cassandra instance
        Then it sends intervn stats to the collector
        and checks if intervn stats can be accessed from
        QE.
        '''
        logging.info("*** test_06_intervn_query ***")
        if AnalyticsTest._check_skip_test() is True:
            return True

        # set the start time in analytics db 1 hour earlier than
        # the current time. For flow series test, we need to create
        # flow samples older than the current time.
        start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
        self._update_analytics_start_time(start_time)
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("VRouterAgent", collectors, logging,
                             vizd_obj.get_opserver_port(), start_time))
        assert generator_obj.verify_on_setup()
        logging.info("Starting intervn gen " + str(UTCTimestampUsec()))
        generator_obj.generate_intervn()
        logging.info("Ending intervn gen " + str(UTCTimestampUsec()))
        assert vizd_obj.verify_intervn_all(generator_obj)
        assert vizd_obj.verify_intervn_sum(generator_obj)
        return True
    def test_10_verify_object_value_table_query(self):
        '''
        This test verifies the ObjectValueTable query.
        '''
        logging.info('%%% test_10_verify_object_value_table_query %%%')

        if AnalyticsTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()
        assert vizd_obj.verify_object_value_table_query(
            table='ObjectCollectorInfo',
            exp_object_values=[vizd_obj.collectors[0].hostname])
        # verify that the object table query works for object id containing
        # XML control characters.
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture('contrail-vrouter-agent', collectors, logging,
                             vizd_obj.get_opserver_port()))
        assert generator_obj.verify_on_setup()
        generator_obj.send_vm_uve(vm_id='vm11&>', num_vm_ifs=2, msg_count=1)
        assert vizd_obj.verify_object_value_table_query(
            table='ObjectVMTable', exp_object_values=['vm11&>'])
示例#11
0
    def test_11_where_clause_query(self):
        '''
        This test is used to check the working of integer 
        fields in the where query 
        '''
        logging.info("*** test_11_where_clause_query ***")

        if AnalyticsTest._check_skip_test() is True:
            return True

        start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
        self._update_analytics_start_time(start_time)
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_where_query()
        #Query the flowseriestable with different where options
        assert vizd_obj.verify_collector_obj_count()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("VRouterAgent", collectors, logging,
                             vizd_obj.get_opserver_port(), start_time))
        assert generator_obj.verify_on_setup()
        generator_obj.generate_flow_samples()
        assert vizd_obj.verify_where_query_prefix(generator_obj)
        return True
    def test_03_flow_query(self):
        '''
        This test starts redis,vizd,opserver and qed
        It uses the test class' cassandra instance
        Then it sends flow stats to the collector
        and checks if flow stats can be accessed from
        QE.
        '''
        logging.info("*** test_03_flow_query ***")
        if AnalyticsTest._check_skip_test() is True:
            return True

        # set the start time in analytics db 1 hour earlier than
        # the current time. For flow series test, we need to create
        # flow samples older than the current time.
        start_time = UTCTimestampUsec() - 3600 * 1000 * 1000
        self._update_analytics_start_time(start_time)
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.redis_port,
                             self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("contrail-vrouter-agent", collectors, logging,
                             vizd_obj.get_opserver_port(), start_time))
        assert generator_obj.verify_on_setup()
        generator_obj.generate_flow_samples()
        generator_obj1 = self.useFixture(
            GeneratorFixture("contrail-vrouter-agent",
                             collectors,
                             logging,
                             vizd_obj.get_opserver_port(),
                             start_time,
                             hostname=socket.gethostname() + "dup"))
        assert generator_obj1.verify_on_setup()
        generator_obj1.generate_flow_samples()
        generator_object = [generator_obj, generator_obj1]
        for obj in generator_object:
            assert vizd_obj.verify_flow_samples(obj)
        assert vizd_obj.verify_flow_table(generator_obj)
        assert vizd_obj.verify_flow_series_aggregation_binning(
            generator_object)
        return True
示例#13
0
 def test_12_verify_message_non_ascii(self):
     '''
     This test verifies message sent with non ascii character does not
     crash vizd.
     '''
     logging.info('%%% test_12_verify_message_non_ascii %%%')
     analytics = self.useFixture(
         AnalyticsFixture(logging, builddir,
                          self.__class__.cassandra_port))
     assert analytics.verify_on_setup()
     collectors = [analytics.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture('contrail-vrouter-agent-12', collectors,
                          logging, None, node_type='Compute'))
     assert generator_obj.verify_on_setup()
     generator_obj.send_vrouterinfo(socket.gethostname(),
         b_info = True, deleted = False, non_ascii=True)
     # Verify vizd is still running
     assert analytics.verify_collector_gen(analytics.collectors[0])
示例#14
0
 def test_09_verify_object_table_query(self):
     '''
     This test verifies the Object Table query.
     '''
     logging.info('%%% test_09_verify_object_table_query %%%')
     vizd_obj = self.useFixture(
         AnalyticsFixture(logging, builddir,
                          self.__class__.redis_port,
                          self.__class__.cassandra_port))
     assert vizd_obj.verify_on_setup()
     collectors = [vizd_obj.get_collector()]
     generator_obj = self.useFixture(
         GeneratorFixture('contrail-control', collectors,
                          logging, None, node_type='Control'))
     assert generator_obj.verify_on_setup()
     msg_types = generator_obj.send_all_sandesh_types_object_logs(
                     socket.gethostname())
     assert vizd_obj.verify_object_table_sandesh_types('ObjectBgpRouter',
             socket.gethostname(), msg_types)
示例#15
0
    def test_14_fieldname_table(self):
        '''
        This test starts vizd and a python generators that simulates
        vrouter and sends messages. It uses the test class' cassandra
        instance.Then it checks that the Field names table got the
        values.
        '''
        logging.info("*** test_14_fieldname_table ***")
        if AnalyticsTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, self.__class__.cassandra_port))
        assert vizd_obj.verify_on_setup()
        collectors = [vizd_obj.get_collector()]
        generator_obj = self.useFixture(
            GeneratorFixture("VRouterAgent", collectors, logging,
                             vizd_obj.get_opserver_port()))
        assert generator_obj.verify_on_setup()
        # Sends 2 different vn uves in 1 sec spacing
        generator_obj.generate_intervn()
        assert vizd_obj.verify_fieldname_table()
        return True
    def test_07_alarm(self):
        '''
        This test starts redis, collectors, analytics-api and
        python generators that simulates alarm generator. This
        test sends alarms from alarm generators and verifies the
        retrieval of alarms from analytics-api.
        '''
        logging.info('*** test_07_alarm ***')
        # collector_ha_test flag is set to True, because we wanna test
        # retrieval of alarms across multiple redis servers.
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging, builddir, -1, 0, collector_ha_test=True))
        assert vizd_obj.verify_on_setup()

        # create alarm-generator and attach it to the first collector.
        collectors = [
            vizd_obj.collectors[0].get_addr(),
            vizd_obj.collectors[1].get_addr()
        ]
        alarm_gen1 = self.useFixture(
            GeneratorFixture('contrail-alarm-gen', [collectors[0]],
                             logging,
                             None,
                             hostname=socket.gethostname() + '_1'))
        alarm_gen1.verify_on_setup()

        # send process state alarm for analytics-node
        alarms = alarm_gen1.create_process_state_alarm('contrail-query-engine')
        alarms += alarm_gen1.create_process_state_alarm(
            'contrail-snmp-collector')
        alarm_gen1.send_alarm(socket.gethostname() + '_1', alarms,
                              COLLECTOR_INFO_TABLE)
        analytics_tbl = _OBJECT_TABLES[COLLECTOR_INFO_TABLE].log_query_name

        # send proces state alarm for control-node
        alarms = alarm_gen1.create_process_state_alarm('contrail-dns')
        alarm_gen1.send_alarm('<&' + socket.gethostname() + '_1>', alarms,
                              BGP_ROUTER_TABLE)
        control_tbl = _OBJECT_TABLES[BGP_ROUTER_TABLE].log_query_name

        # create another alarm-generator and attach it to the second collector.
        alarm_gen2 = self.useFixture(
            GeneratorFixture('contrail-alarm-gen', [collectors[1]],
                             logging,
                             None,
                             hostname=socket.gethostname() + '_2'))
        alarm_gen2.verify_on_setup()

        # send process state alarm for analytics-node
        alarms = alarm_gen2.create_process_state_alarm('contrail-topology')
        alarm_gen2.send_alarm(socket.gethostname() + '_2', alarms,
                              COLLECTOR_INFO_TABLE)

        keys = [socket.gethostname() + '_1', socket.gethostname() + '_2']
        assert (vizd_obj.verify_alarms_table(analytics_tbl, keys))
        assert (vizd_obj.verify_alarm(
            analytics_tbl, keys[0],
            obj_to_dict(
                alarm_gen1.alarms[COLLECTOR_INFO_TABLE][keys[0]].data)))
        assert (vizd_obj.verify_alarm(
            analytics_tbl, keys[1],
            obj_to_dict(
                alarm_gen2.alarms[COLLECTOR_INFO_TABLE][keys[1]].data)))

        keys = ['<&' + socket.gethostname() + '_1>']
        assert (vizd_obj.verify_alarms_table(control_tbl, keys))
        assert (vizd_obj.verify_alarm(
            control_tbl, keys[0],
            obj_to_dict(alarm_gen1.alarms[BGP_ROUTER_TABLE][keys[0]].data)))

        # delete analytics-node alarm generated by alarm_gen2
        alarm_gen2.delete_alarm(socket.gethostname() + '_2',
                                COLLECTOR_INFO_TABLE)

        # verify analytics-node alarms
        keys = [socket.gethostname() + '_1']
        assert (vizd_obj.verify_alarms_table(analytics_tbl, keys))
        assert (vizd_obj.verify_alarm(
            analytics_tbl, keys[0],
            obj_to_dict(
                alarm_gen1.alarms[COLLECTOR_INFO_TABLE][keys[0]].data)))
        assert (vizd_obj.verify_alarm(analytics_tbl,
                                      socket.gethostname() + '_2', {}))

        # Disconnect alarm_gen1 from Collector and verify that all
        # alarms generated by alarm_gen1 is removed by the Collector.
        alarm_gen1.disconnect_from_collector()
        assert (vizd_obj.verify_alarms_table(analytics_tbl, []))
        assert (vizd_obj.verify_alarm(analytics_tbl,
                                      socket.gethostname() + '_1', {}))
        assert (vizd_obj.verify_alarms_table(control_tbl, []))
        assert (vizd_obj.verify_alarm(control_tbl,
                                      '<&' + socket.gethostname() + '_1', {}))

        # update analytics-node alarm in disconnect state
        alarms = alarm_gen1.create_process_state_alarm(
            'contrail-snmp-collector')
        alarm_gen1.send_alarm(socket.gethostname() + '_1', alarms,
                              COLLECTOR_INFO_TABLE)

        # Connect alarm_gen1 to Collector and verify that all
        # alarms generated by alarm_gen1 is synced with Collector.
        alarm_gen1.connect_to_collector()
        keys = [socket.gethostname() + '_1']
        assert (vizd_obj.verify_alarms_table(analytics_tbl, keys))
        assert (vizd_obj.verify_alarm(
            analytics_tbl, keys[0],
            obj_to_dict(
                alarm_gen1.alarms[COLLECTOR_INFO_TABLE][keys[0]].data)))

        keys = ['<&' + socket.gethostname() + '_1>']
        assert (vizd_obj.verify_alarms_table(control_tbl, keys))
        assert (vizd_obj.verify_alarm(
            control_tbl, keys[0],
            obj_to_dict(alarm_gen1.alarms[BGP_ROUTER_TABLE][keys[0]].data)))
示例#17
0
    def test_05_collector_ha(self):
        logging.info('*** test_05_collector_ha ***')
        if AnalyticsTest._check_skip_test() is True:
            return True

        vizd_obj = self.useFixture(
            AnalyticsFixture(logging,
                             builddir,
                             self.__class__.cassandra_port,
                             collector_ha_test=True))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()
        # OpServer and QueryEngine are started with collectors[0] as
        # primary and collectors[1] as secondary
        exp_genlist = ['Collector', 'OpServer', 'QueryEngine']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[0],
                                              exp_genlist)
        # start the VRouterAgent with collectors[1] as primary and
        # collectors[0] as secondary
        collectors = [
            vizd_obj.collectors[1].get_addr(),
            vizd_obj.collectors[0].get_addr()
        ]
        vr_agent = self.useFixture(
            GeneratorFixture("VRouterAgent", collectors, logging,
                             vizd_obj.get_opserver_port()))
        assert vr_agent.verify_on_setup()
        exp_genlist = ['Collector', 'VRouterAgent']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[1],
                                              exp_genlist)
        # stop collectors[0] and verify that OpServer and QE switch
        # from primary to secondary collector
        vizd_obj.collectors[0].stop()
        exp_genlist = ['Collector', 'VRouterAgent', 'OpServer', 'QueryEngine']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[1],
                                              exp_genlist)
        # start collectors[0]
        vizd_obj.collectors[0].start()
        exp_genlist = ['Collector']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[0],
                                              exp_genlist)
        # verify that the old UVEs are flushed from redis when collector restarts
        exp_genlist = [vizd_obj.collectors[0].get_generator_id()]
        assert vizd_obj.verify_generator_list_in_redis(\
                                vizd_obj.collectors[0].get_redis_uve(),
                                exp_genlist)

        # stop collectors[1] and verify that OpServer and QE switch
        # from secondary to primary and VRouterAgent from primary to
        # secondary
        vizd_obj.collectors[1].stop()
        exp_genlist = ['Collector', 'VRouterAgent', 'OpServer', 'QueryEngine']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[0],
                                              exp_genlist)
        # verify the generator list in redis
        exp_genlist = [
            vizd_obj.collectors[0].get_generator_id(),
            vr_agent.get_generator_id(),
            vizd_obj.opserver.get_generator_id(),
            vizd_obj.query_engine.get_generator_id()
        ]
        assert vizd_obj.verify_generator_list_in_redis(\
                                vizd_obj.collectors[0].get_redis_uve(),
                                exp_genlist)

        # stop Opserver and QE
        vizd_obj.opserver.stop()
        vizd_obj.query_engine.stop()
        exp_genlist = ['Collector', 'VRouterAgent']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[0],
                                              exp_genlist)

        # verify the generator list in redis
        exp_genlist = [
            vizd_obj.collectors[0].get_generator_id(),
            vr_agent.get_generator_id()
        ]
        assert vizd_obj.verify_generator_list_in_redis(\
                                vizd_obj.collectors[0].get_redis_uve(),
                                exp_genlist)

        # start Opserver and QE with collectors[1] as the primary and
        # collectors[0] as the secondary. On generator startup, verify
        # that it connects to the secondary collector, if the
        # connection to the primary fails
        vizd_obj.opserver.set_primary_collector(
            vizd_obj.collectors[1].get_addr())
        vizd_obj.opserver.set_secondary_collector(
            vizd_obj.collectors[0].get_addr())
        vizd_obj.opserver.start()
        vizd_obj.query_engine.set_primary_collector(
            vizd_obj.collectors[1].get_addr())
        vizd_obj.query_engine.set_secondary_collector(
            vizd_obj.collectors[0].get_addr())
        vizd_obj.query_engine.start()
        exp_genlist = ['Collector', 'VRouterAgent', 'OpServer', 'QueryEngine']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[0],
                                              exp_genlist)
        # stop the collectors[0] - both collectors[0] and collectors[1] are down
        # send the VM UVE and verify that the VM UVE is synced after connection
        # to the collector
        vizd_obj.collectors[0].stop()
        # Make sure the connection to the collector is teared down before
        # sending the VM UVE
        while True:
            if vr_agent.verify_on_setup() is False:
                break
        vr_agent.send_vm_uve(vm_id='abcd-1234-efgh-5678',
                             num_vm_ifs=5,
                             msg_count=5)
        vizd_obj.collectors[1].start()
        exp_genlist = ['Collector', 'VRouterAgent', 'OpServer', 'QueryEngine']
        assert vizd_obj.verify_generator_list(vizd_obj.collectors[1],
                                              exp_genlist)
        assert vr_agent.verify_vm_uve(vm_id='abcd-1234-efgh-5678',
                                      num_vm_ifs=5,
                                      msg_count=5)
    def test_13_verify_sandesh_ssl(self):
        '''
        This test enables sandesh ssl on contrail-collector and all the
        analytics generators in the AnalyticsFixture and verifies that the
        secure sandesh connection is established between the Collector and all
        the Generators.
        '''
        logging.info('%%% test_13_verify_sandesh_ssl %%%')
        sandesh_cfg = {
            'sandesh_keyfile':
            builddir + '/opserver/test/data/ssl/server-privkey.pem',
            'sandesh_certfile':
            builddir + '/opserver/test/data/ssl/server.pem',
            'sandesh_ca_cert':
            builddir + '/opserver/test/data/ssl/ca-cert.pem',
            'sandesh_ssl_enable': 'True'
        }
        vizd_obj = self.useFixture(
            AnalyticsFixture(logging,
                             builddir,
                             self.__class__.cassandra_port,
                             sandesh_config=sandesh_cfg))
        assert vizd_obj.verify_on_setup()
        assert vizd_obj.verify_collector_obj_count()

        source = socket.gethostname()
        exp_genlist = [
            source + ':Analytics:contrail-collector:0',
            source + ':Analytics:contrail-analytics-api:0',
            source + ':Analytics:contrail-query-engine:0'
        ]
        assert vizd_obj.verify_generator_list(vizd_obj.collectors, exp_genlist)

        # start a python generator without enabling sandesh ssl
        # and verify that it is not connected to the Collector.
        test_gen1 = self.useFixture(
            GeneratorFixture("contrail-test-generator1",
                             vizd_obj.get_collectors(), logging,
                             vizd_obj.get_opserver_port()))
        assert not test_gen1.verify_on_setup()

        # start a python generator with sandesh_ssl_enable = True
        # and verify that it is connected to the Collector.
        test_gen2 = self.useFixture(
            GeneratorFixture("contrail-test-generator2",
                             vizd_obj.get_collectors(),
                             logging,
                             vizd_obj.get_opserver_port(),
                             sandesh_config=sandesh_cfg))
        assert test_gen2.verify_on_setup()

        # stop QE and verify the generator list
        vizd_obj.query_engine.stop()
        exp_genlist = [
            source + ':Analytics:contrail-collector:0',
            source + ':Analytics:contrail-analytics-api:0',
            source + ':Test:contrail-test-generator2:0'
        ]
        assert vizd_obj.verify_generator_list(vizd_obj.collectors, exp_genlist)

        # Start QE with sandesh_ssl_enable = False and verify that the
        # QE is not connected to the Collector
        vizd_obj.query_engine.set_sandesh_config(None)
        vizd_obj.query_engine.start()
        assert not vizd_obj.verify_generator_collector_connection(
            vizd_obj.query_engine.http_port)
        assert vizd_obj.verify_generator_list(vizd_obj.collectors, exp_genlist)

        # Restart Collector with sandesh_ssl_enable = False and verify the
        # generator list in the Collector.
        vizd_obj.collectors[0].stop()
        vizd_obj.collectors[0].set_sandesh_config(None)
        vizd_obj.collectors[0].start()
        assert not vizd_obj.verify_generator_collector_connection(
            test_gen2._http_port)
        assert not vizd_obj.verify_generator_collector_connection(
            vizd_obj.opserver.http_port)
        exp_genlist = [
            source + ':Analytics:contrail-collector:0',
            source + ':Analytics:contrail-query-engine:0',
            source + ':Test:contrail-test-generator1:0'
        ]
        assert vizd_obj.verify_generator_list(vizd_obj.collectors, exp_genlist)