Пример #1
0
    def __init__(self, config):
        '''
            Base runner Class Initiator Sets Data files from where Data need
            to be retrieved and to be used for Building Tests.
        '''
        logger.info("Initializing TestRunner")
        # The keys are test link ids and the values are import information.
        self.__test_map = {}
        if not os.path.isdir(fwork.RESULTS_DIR):
            os.makedirs(fwork.RESULTS_DIR)
        self.reporter = TestReporter(config)
        self.config = config
        if not 'device' in os.environ:
            if not self.config.options.switch:
                os.environ['device'] = "IAP_1"
            else:
                os.environ['device'] = "Switch_1"
        self.config.global_vars = ConfigReader("global_vars")
        self.config.config_vars = ConfigReader("config_vars")
        self._test_discover_count = 0
        self.filter_input_files = False
        self.filter_testlink_ids = False
        self.filter_test_classes = False
        self.filter_test_methods = False
        self.filtered_test_id_list = []
        if self.config.options.input_files != "ALL":
            self.filter_input_files = True
            self.config.input_files = [i.strip().upper() for i in self.config.options.input_files.split(",")]
        if self.config.options.testlink_ids != "ALL":
            self.filter_testlink_ids = True
            self.config.testlink_ids = [i.strip().upper() for i in self.config.options.testlink_ids.split(",")]
        if self.config.options.test_classes != "ALL":
            self.filter_test_classes = True
            self.config.test_classes = [i.strip().upper() for i in self.config.options.test_classes.split(",")]
        if self.config.options.test_methods != "ALL":
            self.filter_test_methods = True
            self.config.test_methods = [i.strip().upper() for i in self.config.options.test_methods.split(",")]

        import re
        self.aClassRef = {}
        self.aObjectRef = {}
        if not self.config.options.ignore_device:
            from inspect import isclass
            from Device_Module import IAPDevice
            # from Device_Module import ClientDevice
            from Device_Module import SwitchDevice
            classes = [x for x in dir(devices) if isclass(getattr(devices, x))]
            print classes
            for clas in classes:
                if getattr(devices, clas).type == "IAP":
                    self.aClassRef[clas] = type(clas, (IAPDevice.IAPDevice,), {})
                # elif getattr(devices, clas).type == "CLIENT":
                    # self.aClassRef[clas] = type(clas, (ClientDevice.ClientDevice,), {})
                elif getattr(devices, clas).type == "SWITCH":
                   self.aClassRef[clas] = type(clas, (SwitchDevice.SwitchDevice,), {})
                self.aObjectRef[clas] = self.aClassRef[clas]()
                print self.aObjectRef.values()
                val = self.aClassRef[clas]
                print clas
                print val
                for att in dir(getattr(devices, clas)):
                    if att != "__doc__" and att != "__module__":
                        setattr(val, att, getattr(getattr(devices, clas), att))
        self.error_type = None
Пример #2
0
    def __init__(self, config):
        '''
            Base runner Class Initiator Sets Data files from where Data need
            to be retrieved and to be used for Building Tests.
        '''
        logger.info("Initializing TestRunner")
        # The keys are test link ids and the values are import information.
        self.__test_map = {}
        if not os.path.isdir(fwork.RESULTS_DIR):
            os.makedirs(fwork.RESULTS_DIR)
        self.reporter = TestReporter(config)
        self.config = config
        if not 'device' in os.environ:
            if not self.config.options.switch:
                os.environ['device'] = "IAP_1"
            else:
                os.environ['device'] = "Switch_1"
        self.config.global_vars = ConfigReader("global_vars")
        self.config.config_vars = ConfigReader("config_vars")
        self._test_discover_count = 0
        self.filter_input_files = False
        self.filter_testlink_ids = False
        self.filter_test_classes = False
        self.filter_test_methods = False
        self.filtered_test_id_list = []
        if self.config.options.input_files != "ALL":
            self.filter_input_files = True
            self.config.input_files = [
                i.strip().upper()
                for i in self.config.options.input_files.split(",")
            ]
        if self.config.options.testlink_ids != "ALL":
            self.filter_testlink_ids = True
            self.config.testlink_ids = [
                i.strip().upper()
                for i in self.config.options.testlink_ids.split(",")
            ]
        if self.config.options.test_classes != "ALL":
            self.filter_test_classes = True
            self.config.test_classes = [
                i.strip().upper()
                for i in self.config.options.test_classes.split(",")
            ]
        if self.config.options.test_methods != "ALL":
            self.filter_test_methods = True
            self.config.test_methods = [
                i.strip().upper()
                for i in self.config.options.test_methods.split(",")
            ]

        import re
        self.aClassRef = {}
        self.aObjectRef = {}
        if not self.config.options.ignore_device:
            from inspect import isclass
            from Device_Module import IAPDevice
            # from Device_Module import ClientDevice
            from Device_Module import SwitchDevice
            classes = [x for x in dir(devices) if isclass(getattr(devices, x))]
            print classes
            for clas in classes:
                if getattr(devices, clas).type == "IAP":
                    self.aClassRef[clas] = type(clas, (IAPDevice.IAPDevice, ),
                                                {})
                # elif getattr(devices, clas).type == "CLIENT":
                # self.aClassRef[clas] = type(clas, (ClientDevice.ClientDevice,), {})
                elif getattr(devices, clas).type == "SWITCH":
                    self.aClassRef[clas] = type(clas,
                                                (SwitchDevice.SwitchDevice, ),
                                                {})
                self.aObjectRef[clas] = self.aClassRef[clas]()
                print self.aObjectRef.values()
                val = self.aClassRef[clas]
                print clas
                print val
                for att in dir(getattr(devices, clas)):
                    if att != "__doc__" and att != "__module__":
                        setattr(val, att, getattr(getattr(devices, clas), att))
        self.error_type = None
Пример #3
0
class TestRunner:
    '''
        Base runner Class for all Test Cases
        Add tests to run from Excel data file thus 
        providing Data Driven Support to the frame work
    '''
    def __init__(self, config):
        '''
            Base runner Class Initiator Sets Data files from where Data need
            to be retrieved and to be used for Building Tests.
        '''
        logger.info("Initializing TestRunner")
        # The keys are test link ids and the values are import information.
        self.__test_map = {}
        if not os.path.isdir(fwork.RESULTS_DIR):
            os.makedirs(fwork.RESULTS_DIR)
        self.reporter = TestReporter(config)
        self.config = config
        if not 'device' in os.environ:
            if not self.config.options.switch:
                os.environ['device'] = "IAP_1"
            else:
                os.environ['device'] = "Switch_1"
        self.config.global_vars = ConfigReader("global_vars")
        self.config.config_vars = ConfigReader("config_vars")
        self._test_discover_count = 0
        self.filter_input_files = False
        self.filter_testlink_ids = False
        self.filter_test_classes = False
        self.filter_test_methods = False
        self.filtered_test_id_list = []
        if self.config.options.input_files != "ALL":
            self.filter_input_files = True
            self.config.input_files = [i.strip().upper() for i in self.config.options.input_files.split(",")]
        if self.config.options.testlink_ids != "ALL":
            self.filter_testlink_ids = True
            self.config.testlink_ids = [i.strip().upper() for i in self.config.options.testlink_ids.split(",")]
        if self.config.options.test_classes != "ALL":
            self.filter_test_classes = True
            self.config.test_classes = [i.strip().upper() for i in self.config.options.test_classes.split(",")]
        if self.config.options.test_methods != "ALL":
            self.filter_test_methods = True
            self.config.test_methods = [i.strip().upper() for i in self.config.options.test_methods.split(",")]

        import re
        self.aClassRef = {}
        self.aObjectRef = {}
        if not self.config.options.ignore_device:
            from inspect import isclass
            from Device_Module import IAPDevice
            # from Device_Module import ClientDevice
            from Device_Module import SwitchDevice
            classes = [x for x in dir(devices) if isclass(getattr(devices, x))]
            print classes
            for clas in classes:
                if getattr(devices, clas).type == "IAP":
                    self.aClassRef[clas] = type(clas, (IAPDevice.IAPDevice,), {})
                # elif getattr(devices, clas).type == "CLIENT":
                    # self.aClassRef[clas] = type(clas, (ClientDevice.ClientDevice,), {})
                elif getattr(devices, clas).type == "SWITCH":
                   self.aClassRef[clas] = type(clas, (SwitchDevice.SwitchDevice,), {})
                self.aObjectRef[clas] = self.aClassRef[clas]()
                print self.aObjectRef.values()
                val = self.aClassRef[clas]
                print clas
                print val
                for att in dir(getattr(devices, clas)):
                    if att != "__doc__" and att != "__module__":
                        setattr(val, att, getattr(getattr(devices, clas), att))
        self.error_type = None

    def _include_input_file(self, dd_file):
        logger.debug("Executing Input File Filter.")    
        if not dd_file.endswith(".xls"):    
            return False
        if self.filter_input_files:
            logger.debug("Check %s in %s" % (dd_file, self.config.input_files))    
            file_name = dd_file.split(".")[0]
            if file_name.upper() in self.config.input_files:
                return True
            else:
                return False
        else:
            return True

    def _include_test_id_based_on_column_filter(self, test_info):
        filter = test_info.record.get("FILTER?", None)
        if (filter is None) or (filter == u''):
            return True
        filter = filter.upper().strip()
        if self.config.options.exclude_filter:
            if filter == "Y":
                return False
            else:
                return True
        else:
            if filter == "Y":
                return True
            else:
                return False            

                            
    def _include_test_id(self, test_info):
        include = False
        if self.filter_testlink_ids:
            logger.debug("Check filter: %s in %s" % (test_info.id.upper(), self.config.testlink_ids))
            if test_info.id.upper() in self.config.testlink_ids:
                include = self._include_test_id_based_on_column_filter(test_info)
            else:
                include = False
        else:
            include = self._include_test_id_based_on_column_filter(test_info)

        
        return include

        
    def _include_test_class(self, class_name):
        if self.filter_test_classes:
            if class_name.upper() in self.config.test_classes:
                return True
            else:
                return False
        else:
            return True    

    def _include_test_method(self, method_name):
        if self.filter_test_methods:
            if method_name.upper() in self.config.test_methods:
                return True
            else:
                return False
        else:
            return True            

    def _add_test(self, test_module_meta):
        '''
            Adds Test meta data and forms Test_Queue from data files
        '''
        Test = None
        exec ("from %s import %s as Test" % (test_module_meta[0], test_module_meta[1]))
        test = Test(self.config)
        method_tuples = []
        logger.debug("Disovering Test Methods for Test Class: %s" % test.__class__.__name__)
        for m in test.__class__.__dict__.keys():
            if m.startswith("test"):
                test_id = "-".join([i.upper() for i in m.split("_")[1:3]])            
                if not self._include_test_method(m):
                    self.filtered_test_id_list.append(test_id)
                    continue
                self._test_discover_count += 1
                logger.debug(m)
                self.__test_map[test_id] = {
                                                "PARENT" : test,
                                                "PARENT_IMPORT_PATH" : test_module_meta[0],
                                                "METHOD_NAME" : m,
                                            }
        
    def discover(self, tests_filter=None):
        test_queue = []
        counter = 1
        for root, dirs, files in os.walk(self.config.options.tests_dir):
            index = root.find(self.config.options.tests_dir)
            import_path_prefix = ".".join(["athenataf", "tests", root[index + len(self.config.options.tests_dir) + 1:].replace("\\",".").replace("/", "."), ""])
            for test_module in files:
                if test_module.endswith(".py") and not test_module.startswith("_"):
                    test_module_name = test_module.split(".")[0]
                    if not self._include_test_class(test_module_name):
                        continue
                    test_module_import = import_path_prefix + test_module_name
                    include = True
                    if tests_filter is not None:
                        if test_module_name not in tests_filter:
                            include = False
                    if include:
                        test_queue.append([test_module_import, test_module_name])

        for test_module_meta in test_queue:
            self._add_test(test_module_meta)

            
        logger.info("TOTAL TESTS DISCOVERED IN CODE: %d" % self._test_discover_count)
        logger.debug("Test ID Keys:")    
        keys = self.__test_map.keys()
        keys.sort()
        for key in keys:
            logger.debug(key)

            
    def prepare(self):
        pass

        
    def setUp(self):
        self.reporter.setUp()
        if not self.config.options.ignore_device:
            retries = 0
            print(self.aObjectRef.values())
            for dev in self.aObjectRef.values():
                dev.connect()
            # while retries < 5:
                # dev = self.aObjectRef[os.environ['device']]
                # if dev.get_device_status():
                    # break
                # retries = retries + 1
            # if retries == 5:
                # logger.info("*** DEVICE IS NOT UP. ABORTING TEST EXECUTION. ***")
                # for dev in self.aObjectRef.values():
                    # dev.disconnect()
                # import sys
                # sys.exit(1)

        
    def _get_test_result_map(self, test_info):
        test_idea = test_info.record.get("TEST_IDEA", None)
        if (test_idea == u"") or (test_idea is None):
            test_idea = "Not Provided"
        test_result = {
                    "TESTLINK_ID" : test_info.record["TESTLINK_ID"],
                    "TEST_IDEA" : test_idea,                                    
                    "IMPORT_PATH" : test_info.ipath,
                    "TEST_CLASS" : test_info.klass,
                    "TEST_METHOD"    : test_info.method,
                    "TEST_DATA" : str(test_info.record),
                    "RESULT" : "NOT_SET",
                    "DURATION" : "",
                    "EXCEPTION" : "",
                    "TRACE"    : ""

        }
        return test_result

    def _execute_fixture(self, test_obj, method_name, test_result):
        try:
            if not method_name.startswith("test"):
                if not self.config.options.ignore_device:
                    retries = 0
                    # while retries < 3:
                        # dev = self.aObjectRef[os.environ['device']]
                        # if dev.get_device_status():
                            # break
                        # retries = retries + 1
                    # if retries == 3:
                        # logger.info("*** DEVICE IS NOT UP. ABORTING TEST EXECUTION. ***")
                        # for dev in self.aObjectRef.values():
                            # dev.disconnect()
                        # import sys
                        # sys.exit(1)
                logger.info("**** Device is UP *****")
                logger.info("Executing test.%s" % method_name)
            if not self.config.options.fake_run:
                apply(getattr(test_obj, method_name))
        except AssertionError, e:
            if 'actual and expected running config' in str(e):
                self.error_type = 'EE'
            elif 'Post clean-up the device did not' in str(e):
                self.error_type = 'EP'
            else:
                self.error_type = 'UKN'
            test_result["RESULT"] = "FAIL"
            import traceback
            test_result["EXCEPTION"] = "AssertionError in Test %s: %s" % (method_name, str(e))
            test_result["TRACE"] = traceback.format_exc(e)
            test_obj.on_fail()
            return None            
        except Exception, e:
            self.error_type = 'UKN'
            test_result["RESULT"] = "ERROR"            
            import traceback
            test_result["EXCEPTION"] = "Exception in Test %s: %s" % (method_name, str(e))
            test_result["TRACE"] = traceback.format_exc(e)    
            test_obj.on_error()            
            return None
Пример #4
0
class TestRunner:
    '''
        Base runner Class for all Test Cases
        Add tests to run from Excel data file thus 
        providing Data Driven Support to the frame work
    '''
    def __init__(self, config):
        '''
            Base runner Class Initiator Sets Data files from where Data need
            to be retrieved and to be used for Building Tests.
        '''
        logger.info("Initializing TestRunner")
        # The keys are test link ids and the values are import information.
        self.__test_map = {}
        if not os.path.isdir(fwork.RESULTS_DIR):
            os.makedirs(fwork.RESULTS_DIR)
        self.reporter = TestReporter(config)
        self.config = config
        if not 'device' in os.environ:
            if not self.config.options.switch:
                os.environ['device'] = "IAP_1"
            else:
                os.environ['device'] = "Switch_1"
        self.config.global_vars = ConfigReader("global_vars")
        self.config.config_vars = ConfigReader("config_vars")
        self._test_discover_count = 0
        self.filter_input_files = False
        self.filter_testlink_ids = False
        self.filter_test_classes = False
        self.filter_test_methods = False
        self.filtered_test_id_list = []
        if self.config.options.input_files != "ALL":
            self.filter_input_files = True
            self.config.input_files = [
                i.strip().upper()
                for i in self.config.options.input_files.split(",")
            ]
        if self.config.options.testlink_ids != "ALL":
            self.filter_testlink_ids = True
            self.config.testlink_ids = [
                i.strip().upper()
                for i in self.config.options.testlink_ids.split(",")
            ]
        if self.config.options.test_classes != "ALL":
            self.filter_test_classes = True
            self.config.test_classes = [
                i.strip().upper()
                for i in self.config.options.test_classes.split(",")
            ]
        if self.config.options.test_methods != "ALL":
            self.filter_test_methods = True
            self.config.test_methods = [
                i.strip().upper()
                for i in self.config.options.test_methods.split(",")
            ]

        import re
        self.aClassRef = {}
        self.aObjectRef = {}
        if not self.config.options.ignore_device:
            from inspect import isclass
            from Device_Module import IAPDevice
            # from Device_Module import ClientDevice
            from Device_Module import SwitchDevice
            classes = [x for x in dir(devices) if isclass(getattr(devices, x))]
            print classes
            for clas in classes:
                if getattr(devices, clas).type == "IAP":
                    self.aClassRef[clas] = type(clas, (IAPDevice.IAPDevice, ),
                                                {})
                # elif getattr(devices, clas).type == "CLIENT":
                # self.aClassRef[clas] = type(clas, (ClientDevice.ClientDevice,), {})
                elif getattr(devices, clas).type == "SWITCH":
                    self.aClassRef[clas] = type(clas,
                                                (SwitchDevice.SwitchDevice, ),
                                                {})
                self.aObjectRef[clas] = self.aClassRef[clas]()
                print self.aObjectRef.values()
                val = self.aClassRef[clas]
                print clas
                print val
                for att in dir(getattr(devices, clas)):
                    if att != "__doc__" and att != "__module__":
                        setattr(val, att, getattr(getattr(devices, clas), att))
        self.error_type = None

    def _include_input_file(self, dd_file):
        logger.debug("Executing Input File Filter.")
        if not dd_file.endswith(".xls"):
            return False
        if self.filter_input_files:
            logger.debug("Check %s in %s" % (dd_file, self.config.input_files))
            file_name = dd_file.split(".")[0]
            if file_name.upper() in self.config.input_files:
                return True
            else:
                return False
        else:
            return True

    def _include_test_id_based_on_column_filter(self, test_info):
        filter = test_info.record.get("FILTER?", None)
        if (filter is None) or (filter == u''):
            return True
        filter = filter.upper().strip()
        if self.config.options.exclude_filter:
            if filter == "Y":
                return False
            else:
                return True
        else:
            if filter == "Y":
                return True
            else:
                return False

    def _include_test_id(self, test_info):
        include = False
        if self.filter_testlink_ids:
            logger.debug("Check filter: %s in %s" %
                         (test_info.id.upper(), self.config.testlink_ids))
            if test_info.id.upper() in self.config.testlink_ids:
                include = self._include_test_id_based_on_column_filter(
                    test_info)
            else:
                include = False
        else:
            include = self._include_test_id_based_on_column_filter(test_info)

        return include

    def _include_test_class(self, class_name):
        if self.filter_test_classes:
            if class_name.upper() in self.config.test_classes:
                return True
            else:
                return False
        else:
            return True

    def _include_test_method(self, method_name):
        if self.filter_test_methods:
            if method_name.upper() in self.config.test_methods:
                return True
            else:
                return False
        else:
            return True

    def _add_test(self, test_module_meta):
        '''
            Adds Test meta data and forms Test_Queue from data files
        '''
        Test = None
        exec("from %s import %s as Test" %
             (test_module_meta[0], test_module_meta[1]))
        test = Test(self.config)
        method_tuples = []
        logger.debug("Disovering Test Methods for Test Class: %s" %
                     test.__class__.__name__)
        for m in test.__class__.__dict__.keys():
            if m.startswith("test"):
                test_id = "-".join([i.upper() for i in m.split("_")[1:3]])
                if not self._include_test_method(m):
                    self.filtered_test_id_list.append(test_id)
                    continue
                self._test_discover_count += 1
                logger.debug(m)
                self.__test_map[test_id] = {
                    "PARENT": test,
                    "PARENT_IMPORT_PATH": test_module_meta[0],
                    "METHOD_NAME": m,
                }

    def discover(self, tests_filter=None):
        test_queue = []
        counter = 1
        for root, dirs, files in os.walk(self.config.options.tests_dir):
            index = root.find(self.config.options.tests_dir)
            import_path_prefix = ".".join([
                "athenataf", "tests",
                root[index + len(self.config.options.tests_dir) + 1:].replace(
                    "\\", ".").replace("/", "."), ""
            ])
            for test_module in files:
                if test_module.endswith(
                        ".py") and not test_module.startswith("_"):
                    test_module_name = test_module.split(".")[0]
                    if not self._include_test_class(test_module_name):
                        continue
                    test_module_import = import_path_prefix + test_module_name
                    include = True
                    if tests_filter is not None:
                        if test_module_name not in tests_filter:
                            include = False
                    if include:
                        test_queue.append(
                            [test_module_import, test_module_name])

        for test_module_meta in test_queue:
            self._add_test(test_module_meta)

        logger.info("TOTAL TESTS DISCOVERED IN CODE: %d" %
                    self._test_discover_count)
        logger.debug("Test ID Keys:")
        keys = self.__test_map.keys()
        keys.sort()
        for key in keys:
            logger.debug(key)

    def prepare(self):
        pass

    def setUp(self):
        self.reporter.setUp()
        if not self.config.options.ignore_device:
            retries = 0
            print(self.aObjectRef.values())
            for dev in self.aObjectRef.values():
                dev.connect()
            # while retries < 5:
            # dev = self.aObjectRef[os.environ['device']]
            # if dev.get_device_status():
            # break
            # retries = retries + 1
            # if retries == 5:
            # logger.info("*** DEVICE IS NOT UP. ABORTING TEST EXECUTION. ***")
            # for dev in self.aObjectRef.values():
            # dev.disconnect()
            # import sys
            # sys.exit(1)

    def _get_test_result_map(self, test_info):
        test_idea = test_info.record.get("TEST_IDEA", None)
        if (test_idea == u"") or (test_idea is None):
            test_idea = "Not Provided"
        test_result = {
            "TESTLINK_ID": test_info.record["TESTLINK_ID"],
            "TEST_IDEA": test_idea,
            "IMPORT_PATH": test_info.ipath,
            "TEST_CLASS": test_info.klass,
            "TEST_METHOD": test_info.method,
            "TEST_DATA": str(test_info.record),
            "RESULT": "NOT_SET",
            "DURATION": "",
            "EXCEPTION": "",
            "TRACE": ""
        }
        return test_result

    def _execute_fixture(self, test_obj, method_name, test_result):
        try:
            if not method_name.startswith("test"):
                if not self.config.options.ignore_device:
                    retries = 0
                    # while retries < 3:
                    # dev = self.aObjectRef[os.environ['device']]
                    # if dev.get_device_status():
                    # break
                    # retries = retries + 1
                    # if retries == 3:
                    # logger.info("*** DEVICE IS NOT UP. ABORTING TEST EXECUTION. ***")
                    # for dev in self.aObjectRef.values():
                    # dev.disconnect()
                    # import sys
                    # sys.exit(1)
                logger.info("**** Device is UP *****")
                logger.info("Executing test.%s" % method_name)
            if not self.config.options.fake_run:
                apply(getattr(test_obj, method_name))
        except AssertionError, e:
            if 'actual and expected running config' in str(e):
                self.error_type = 'EE'
            elif 'Post clean-up the device did not' in str(e):
                self.error_type = 'EP'
            else:
                self.error_type = 'UKN'
            test_result["RESULT"] = "FAIL"
            import traceback
            test_result["EXCEPTION"] = "AssertionError in Test %s: %s" % (
                method_name, str(e))
            test_result["TRACE"] = traceback.format_exc(e)
            test_obj.on_fail()
            return None
        except Exception, e:
            self.error_type = 'UKN'
            test_result["RESULT"] = "ERROR"
            import traceback
            test_result["EXCEPTION"] = "Exception in Test %s: %s" % (
                method_name, str(e))
            test_result["TRACE"] = traceback.format_exc(e)
            test_obj.on_error()
            return None