Example #1
0
    def setup(self, args):
        # Parse command line options
        parser = PassThroughOptionParser()
        parser.add_option('-a', '--virtual_config_file', default=None, type='string')
        parser.add_option('-b', '--browser', default=None, type='string')
        parser.add_option('-c', '--config_file', default=None, type='string')
        parser.add_option('-d', '--db', default="false", type='string')
        parser.add_option('-e', '--excluded', default=None, type='string')
        parser.add_option('-g', '--grid_address', default=None, type='string')
        parser.add_option('-i', '--override_settings', default=None, type='string') # override global settings with this comma-separated, key:value string
        parser.add_option('-l', '--logfile', default=None, type='string')
        parser.add_option('-m', '--modules', default=None, type='string')
        parser.add_option('-o', '--override_params', default=None, type='string') # override specific test params with this comma-separated, key:value string
        parser.add_option('-s', '--settings', default=None, type='string') 
        parser.add_option('-t', '--test', default=None, type='string')
        parser.add_option('-u', '--url', default=None, type='string')
        parser.add_option('-y', '--test_type', default=None, type='string')
        parser.add_option('-z', '--loadtest_settings', default=None,
                                                       type='string')    
        options, args_out = parser.parse_args(args)
        self.override_params = options.override_params
        self.override_settings = options.override_settings
        global_settings = options.settings
        global_settings_config = None
        config = {}
        sub_configs = []
        test_overrides = {}

        if options.config_file == None and options.virtual_config_file == None and options.settings == None:
            print '-c (--config_file) or -a (--virtual_config_file)  or -s (--settings) is required'
            sys.exit(2)

        if options.logfile != None:
            ''' redirect stdout to the logfile '''
            f = open(options.logfile, "a", 0)
            sys.stdout = f

        if options.excluded != None:
            ''' build a list of tests explicitly excluded from the
            command-line option '''
            excluded_list = options.excluded.split(',')
        else:
            excluded_list = None

        ''' load the config file '''
        # optional global settings config file
        # can be a comma-separated list, in which case the global_settings config is the first item
        # and any number of config files can be added to it, merging fields as needed
        # global settings override all sub_config settings, but are overridden by command-line -i overrides
        # tests['includes'] and tests['excludes'] in sub_configs are simply added together
        # an exclude in any sub_config will override any include of the same test elsewhere in the merged configs        
        if global_settings != None:           
            gsettings = global_settings.split(",")
            config_path = os.getenv('PYTAFHOME') + os.sep + "config" + os.sep
            #print gsettings
            # first import the "global" or top-most, settings file
            f = open("%s%s" % (config_path, gsettings[0]), 'r').read()
            global_settings_config = json.loads(f)
            test_overrides = global_settings_config.get('test_overrides', {})
            #print test_overrides
            #print global_settings_config
            for i in range(1, len(gsettings)):
                sub_configs.append(gsettings[i])
        
        # handle the virtual config file
        try:
            if options.virtual_config_file != None:
                #print options.virtual_config_file
                config = json.loads(options.virtual_config_file)
                db_config = {}
            elif options.config_file != None:
                config_path = os.getenv('PYTAF_HOME') + os.sep + "config" + os.sep
                f = open("%s%s" % (config_path,options.config_file), 'r').read()
                config = json.loads(f)           
            
            # allow for the possibility of nested config files
            new_settings = {}
            # allow for an import from override config files
            config_to_import = config.get('import', None)
            if config_to_import != None:
                sub_configs.append(config_to_import)
             
            if global_settings_config != None and len(sub_configs) > 0:
                top_config = global_settings_config
            else:
                top_config = config # {} at this point
            
            # import each sub_config and merge       
            if len(sub_configs) > 0:    
                for sub_config in sub_configs:
                    #print "sub_config file = %s" % sub_config
                    f = open("%s%s" % (config_path, sub_config), 'r').read()           
                    imported_config = json.loads(f)
                    # let any original config 'settings' override the imported config's settings by merging the dictionaries 
                    if imported_config.has_key('settings'):
                        new_settings = dict(imported_config['settings'].items() + top_config['settings'].items())
                    
                    # if there's a global settings config, it will trump all settings
                    if global_settings_config != None:
                        new_settings = dict(new_settings.items() + global_settings_config['settings'].items())
                    
                    # the new, merged config's settings
                    config['settings'] = new_settings
                    try:
                        config['settings']['config_file'] += "," + sub_config
                    except: # if not already there, initialize it
                        config['settings']['config_file'] = gsettings[0] + "," + sub_config
                    top_config['settings'] = config['settings'] # update this as well
                        
                    if imported_config.has_key('tests'):
                        #print "merge the tests includes sections"
                        if config.has_key('tests') == False:
                            config['tests'] = {}
                        try:
                            config['tests']['includes'] += imported_config['tests']['includes']
                        except:                          
                            config['tests']['includes'] = imported_config['tests']['includes']
                       
                        if imported_config['tests'].has_key('excludes'):  
                            for exclude_element in imported_config['tests']['excludes']:
                                for method in exclude_element['methods']:
                                    #print "adding to exclude list: %s" % method['name']
                                    excluded_list.append(method['name'])
    
                    # the params dictionary of individual tests can be overridden, e.g.
                    # "test_overrides":
                    #   {
                    #     "test_owner_add_private_channel_invalid_code": { "bad_code" : "AAAAACHECKITOUT" }
                    #   }
                    test_overrides = dict(list(test_overrides.items()) + list(imported_config.get("test_overrides", {}).items()))
                    #print test_overrides
                    
                    # additional excludes can be added to the base config file's excluded tests list, e.g:
                    #"additional_excludes":
                    #   [  
                    #      "test_owner_manage_subscriptions"
                    #   ]
                    #print "look for additional excludes"
                    additional_excludes = top_config.get('additional_excludes', [])
                    #print "additional excludes = %s" % additional_excludes
                    if len(excluded_list) == 0 and len(additional_excludes) > 0:
                        #print "set excluded_list to %s" % excluded_list
                        excluded_list = additional_excludes
                    else:
                        #print "else?"
                        for j in range (len(additional_excludes)):
                            excluded_list.append(additional_excludes[j]) 
                            #print "excluded_list now = %s" % excluded_list
                            
                    #print "config = %s" % config
                          
                else: # if there's no overrides config but there is a global_settings config
                    # if there's a global settings config, it will trump all settings
                    if global_settings_config != None:
                        #print 'global settings override'
                        #print config['settings']
                        if config.has_key('settings'):
                            new_settings = dict(config['settings'].items() + global_settings_config['settings'].items())
                            config['settings'] = new_settings     
                        else:
                            config['settings'] = global_settings_config['settings']
                    #print "config = %s" % config
                                   
        except:
            print pytaf_utils.formatExceptionInfo()
            if len(sub_configs) > 0:
                cf = gsettings
            else:
                cf = options.config_file
            print "JSON problem in a config file: %s" % cf
            sys.exit(2)   

        config['settings']['config_file'] = options.config_file
        
        try:  # try to open the default db_config file
            f2 = open("%s%s" % (config_path, "db_config.json"), 'r').read()
            db_config = json.loads(f2)
        except:
            db_config = {}

        # command-line -u overrides config file for url
        if options.url != None:
            config['settings']['url'] = options.url  # for browser tests

        # command-line -b overrides config file for browser
        # can be in the form of 'Firefox' or 'Firefox,10,WINDOWS'
        if options.browser != None:
            config['settings']['browser'] = options.browser

        # command-line -g overrides config file setting for test_host (and optionally test_port as well)
        # used for selenium_grid or local selenium server
        if options.grid_address != None:
            if options.grid_address.find(":") >= 0:
                g = options.grid_address.split(":")
                config['settings']['test_host'] = g[0]
                config['settings']['test_port'] = int(g[1])
            else:
                config['settings']['test_host'] = options.grid_address

        # reset the settings object for passing on to test methods
        settings = config['settings']
        
        # initialize the root logger
        self.setup_logger(settings)

        # dynamically import all modules found in the config file
        if options.modules == None:
            modules_array = pytaf_utils.get_all_modules(config)
        else:  # only load the specified module(s) from the config file
            modules_array = options.modules.split(",")
        logging.debug('modules: %s' % modules_array)
        mapped_modules = map(__import__, modules_array)

        passed = 0
        failed = 0

        if options.test_type == 'load':
            '''
             the command-line may override load_test_settings with
             -z --loadtest_settings in the form of
             duration:max_threads:ramp_steps:ramp_interval:throttle_rate
             e.g. 3600,500,10,30,1
             which would run the load test for 1 hour (3600 seconds)
             ramping up to a total of 500 threads in 10 steps
             (each step would add 50 threads (500/10))
             and these batches of threads would be added in
             30 second installments (approximately)
             the final value (throttle_rate=1) is used to
             brake the entire load test operation by sleeping for
             that amount (in seconds) between chunks of test case allocations
            '''
            if options.loadtest_settings != None:
                p = options.loadtest_settings.split(",")
                if len(p) == 5:
                    config['settings']['load_test_settings'] = \
                    {"duration": int(p[0]),
                     "max_threads": int(p[1]),
                     "ramp_steps": int(p[2]),
                     "ramp_interval": int(p[3]),
                     "throttle_rate": int(p[4])}
                else:
                    logging.fatal('load test settings are not complete.')
                    logging.fatal('they must be in the form of' \
                'duration:max_threads:ramp_steps:ramp_interval:throttle_rate')
                    sys.exit(-1)
            # now start the load test
            passed, failed = self.do_load_test(mapped_modules, config)
        # if --test is specified, try and get the params and run each one
        elif options.test != None:
            ts = options.test.split(",")
            for i in range(0, len(ts)):
                test = ts[i]
                if self.test_excluded(test, excluded_list) == False:
                    params = pytaf_utils.get_params(config, test)
                    if params == None:
                        logging.fatal("could not find params for test %s" % test)
                        sys.exit()
                    else:
                        #if test_overrides.get(test, None):
                        #    params = dict(params.items() + test_overrides[test].items())
                        params, settings = self.do_overrides(params, settings, test, test_overrides, self.override_settings, self.override_params)                                             
                        status = self.do_test(mapped_modules, settings,
                                              test, params)
                        if status == True:
                            passed = passed + 1
                        else:
                            failed = failed + 1
                else:
                    logging.info("%s is on the excluded list" % test)
        # if --test is not specified, collect and run all the
        # tests in the config file
        else:
            tests = pytaf_utils.get_all_tests(config, mapped_modules)
            for test in tests:
                if self.test_excluded(test, excluded_list) == False:
                    params = pytaf_utils.get_params(config, test)
                    if params != None:
                        #if test_overrides.get(test, None):
                        #    params = dict(params.items() + test_overrides[test].items())
                        params, settings = self.do_overrides(params, settings, test, test_overrides, self.override_settings, self.override_params)                             
                        status = self.do_test(mapped_modules, settings,
                                              test, params)
                        if status == True:
                            passed = passed + 1
                        else:
                            failed = failed + 1
                else:
                    logging.info("%s is on the excluded list" % test)

        logging.info("---------------")
        logging.info("Tests Run: %s" % (passed + failed))
        logging.info("Passed:    %s" % passed)
        logging.info("Failed:    %s" % failed)

        print_results = []
        for r in self.results:
            print_results.append(r['status'] + " " + r['test_method'])

        for r in sorted(print_results):
            logging.info(r)

        # post results to the database
        if pytaf_utils.str2bool(options.db) == True:
            pytaf_utils.post_results(self.results, settings,
                                     db_config, passed, failed)
Example #2
0
    def setup(self, args):
        # Parse command line options
        parser = optparse.OptionParser()
        parser.add_option("-b", "--browser", default=None, type="string")
        parser.add_option("-c", "--config_file", default=None, type="string")
        parser.add_option("-d", "--db", default="false", type="string")
        parser.add_option("-e", "--excluded", default=None, type="string")
        parser.add_option("-l", "--logfile", default=None, type="string")
        parser.add_option("-m", "--modules", default=None, type="string")
        parser.add_option("-s", "--selenium_server", default=None, type="string")
        parser.add_option("-t", "--test", default=None, type="string")
        parser.add_option("-u", "--url", default=None, type="string")
        parser.add_option("-y", "--test_type", default=None, type="string")
        parser.add_option("-z", "--loadtest_settings", default=None, type="string")
        options, args_out = parser.parse_args(args)

        if options.config_file == None:
            print("-c (--config_file) is required")
            sys.exit(-1)

        if options.logfile != None:
            """ redirect stdout to the logfile """
            the_file = open(options.logfile, "a", 0)
            sys.stdout = the_file

        if options.excluded != None:
            """ build a list of tests explicitly excluded from the
            command-line option """
            excluded_list = options.excluded.split(",")
        else:
            excluded_list = None

        """ load the config file """
        try:
            config_path = os.getenv("PYTAF_HOME") + os.sep + "config" + os.sep
            the_file = open("%s%s" % (config_path, options.config_file), "r").read()
            config = json.loads(the_file)
        except:
            print(pytaf_utils.formatExceptionInfo())
            print("problem with config file %s%s" % (config_path, options.config_file))
            sys.exit()

        try:  # try to open the default db_config file
            the_file = open("%s%s" % (config_path, "db_config.json"), "r").read()
            db_config = json.loads(the_file)
        except:
            db_config = {}

        # command-line -u overrides config file for url
        if options.url != None:
            config["settings"]["url"] = options.url  # for browser tests

        # command-line -b overrides config file for browser
        # can be in the form of 'Firefox' or 'Firefox,10,WINDOWS'
        if options.browser != None:
            config["settings"]["browser"] = options.browser

        # command-line -s option overrides the selenium_server
        # host and port settings
        if options.selenium_server != None:
            if options.selenium_server.find(":") >= 0:
                sel_list = options.selenium_server.split(":")
                config["settings"]["selenium_host"] = sel_list[0]
                config["settings"]["selenium_port"] = int(sel_list[1])
            else:
                config["settings"]["selenium_host"] = options.selenium_server

        # reset the settings object for passing on to test methods
        settings = config["settings"]

        # dynamically import all modules found in the config file
        if options.modules == None:
            modules_array = pytaf_utils.get_all_modules(config)
        else:  # only load the specified module(s) from the config file
            modules_array = options.modules.split(",")
        if DEBUG:
            print("modules: %s" % modules_array)
        mapped_modules = map(__import__, modules_array)

        passed = 0
        failed = 0

        if options.test_type == "load":
            """
             the command-line may override load_test_settings with
             -z --loadtest_settings in the form of
             duration:max_threads:ramp_steps:ramp_interval:throttle_rate
             e.g. 3600,500,10,30,1
             which would run the load test for 1 hour (3600 seconds)
             ramping up to a total of 500 threads in 10 steps
             (each step would add 50 threads (500/10))
             and these batches of threads would be added in
             30 second installments (approximately)
             the final value (throttle_rate=1) is used to
             brake the entire load test operation by sleeping for
             that amount (in seconds) between chunks of test case allocations
            """
            if options.loadtest_settings != None:
                load_list = options.loadtest_settings.split(",")
                if len(load_list) == 5:
                    config["settings"]["load_test_settings"] = {
                        "duration": int(load_list[0]),
                        "max_threads": int(load_list[1]),
                        "ramp_steps": int(load_list[2]),
                        "ramp_interval": int(load_list[3]),
                        "throttle_rate": int(load_list[4]),
                    }
                else:
                    print("load test settings are not complete.")
                    print("they must be in the form of" "duration:max_threads:ramp_steps:ramp_interval:throttle_rate")
                    sys.exit(-1)
            # now start the load test
            passed, failed = self.do_load_test(mapped_modules, config)
        # if --test is specified, try and get the params and run each one
        elif options.test != None:
            test_list = options.test.split(",")
            for i in range(0, len(test_list)):
                test = test_list[i]
                if self.test_excluded(test, excluded_list) == False:
                    params = pytaf_utils.get_params(config, test)
                    if params == None:
                        print("could not find params for test %s" % test)
                        sys.exit()
                    else:
                        status = self.do_test(mapped_modules, settings, test, params)
                        if status == True:
                            passed = passed + 1
                        else:
                            failed = failed + 1
                else:
                    print("%s is on the excluded list" % test)
        # if --test is not specified, collect and run all the
        # tests in the config file
        else:
            tests = pytaf_utils.get_all_tests(config, mapped_modules)
            for test in tests:
                if self.test_excluded(test, excluded_list) == False:
                    params = pytaf_utils.get_params(config, test)
                    if params != None:
                        status = self.do_test(mapped_modules, settings, test, params)
                        if status == True:
                            passed = passed + 1
                        else:
                            failed = failed + 1
                else:
                    print("%s is on the excluded list" % test)

        print("---------------")
        print("Tests Run: %s" % (passed + failed))
        print("Passed:    %s" % passed)
        print("Failed:    %s" % failed)

        print_results = []
        for result in self.results:
            print_results.append(result["status"] + " " + result["test_method"])

        for result in sorted(print_results):
            print(result)

        # post results to the database
        if pytaf_utils.str2bool(options.db) == True:
            pytaf_utils.post_results(self.results, settings, db_config, passed, failed)
Example #3
0
    def do_curl(
        self,
        url,
        u,
        request,
        settings={},
        curl_method="POST",
        https=True,
        return_type="json",
        ignore_certs=False,
        connection_timeout=60,
        do_log=True,
    ):
        response = {"data": "", "status": "", "reason": "", "headers": ""}
        rheaders = ""
        status = ""
        reason = ""

        # settings['httplib'] can override the use of pycurl and redirect to httlib, just as settings['webdriver'] can redirect selenium to webdriver for browser tests
        if pytaf_utils.str2bool(settings.get("httplib", "false")) == True:
            if curl_method.upper() == "POST":
                logging.info("using httplib to POST instead of pycurl due to settings['httplib']")
                return self.do_post(url, u, request, settings, https, connection_timeout, do_log)
            elif curl_method.upper() == "GET":
                logging.info("using httplib to GET instead of pycurl due to settings['httplib']")
                return self.do_get(url, u, settings, https, connection_timeout, do_log)

        try:
            import pycurl
        except:
            return response

        try:
            # handle json stuff
            if type(request) == dict:
                request = json.dumps(request)
            if len(request) > 0:
                if do_log:
                    logging.info("request = %s" % request)

            # setup response stuff
            import StringIO

            body = StringIO.StringIO()
            response_headers = StringIO.StringIO()
            c = pycurl.Curl()
            if https:
                if url.find("http") < 0:
                    the_url = "https://" + url + u
                else:
                    the_url = url + u
                if settings.get("use_ssl", False) == True:
                    c.setopt(c.SSLVERSION, c.SSLVERSION_SSLv3)
                else:
                    c.setopt(c.SSLVERSION, c.SSLVERSION_TLSv1)

                if settings.get("verify_peer", True) == False:
                    c.setopt(c.SSL_VERIFYPEER, False)
            else:
                if url.find("http") < 0:
                    the_url = "http://" + url + u
                else:
                    the_url = url + u
            c.setopt(c.WRITEFUNCTION, body.write)
            c.setopt(c.HEADERFUNCTION, response_headers.write)

            # optional bitrate limiting
            if settings.get("bitrate_limit", None) != None:
                c.setopt(c.MAX_RECV_SPEED_LARGE, int(settings.get("bitrate_limit")))

            # handle method types
            if curl_method == "POST":
                c.setopt(c.POST, 1)
                c.setopt(c.POSTFIELDS, str(request))
            elif curl_method == "PUT":
                c.setopt(c.CUSTOMREQUEST, "PUT")
                c.setopt(c.POSTFIELDS, str(request))
            elif curl_method == "DELETE":
                c.setopt(c.CUSTOMREQUEST, "DELETE")
                # c.setopt(c.POSTFIELDS, str(request))
            # else: # "GET"
            #    c.setopt(c.GET, 1) # no such setting. pyCurl defaults to GET

            # set basic cURL options
            c.setopt(c.URL, str(the_url))
            # c.VERBOSE is way too verbose - only enable for temporary debugging
            # if DEBUG: c.setopt(c.VERBOSE, 1)
            c.setopt(c.FOLLOWLOCATION, 1)
            c.setopt(c.MAXREDIRS, 5)
            c.setopt(c.CONNECTTIMEOUT, connection_timeout)
            # c.setopt(c.TIMEOUT, connection_timeout)

            # handle optional cert stuff
            CERT_FILE, CACERT, KEY_FILE = self.get_cert_paths(settings)
            if ignore_certs == False:
                if CERT_FILE:
                    c.setopt(c.SSLCERT, str(CERT_FILE))
                    c.setopt(c.SSLCERTTYPE, "PEM")

                    if CACERT:
                        c.setopt(c.CAINFO, str(CACERT))

                    if KEY_FILE:
                        c.setopt(c.SSLKEY, str(KEY_FILE))
                        c.setopt(c.SSLKEYTYPE, "PEM")
                else:
                    # logging.debug('ignore certs')
                    c.setopt(c.SSL_VERIFYHOST, False)
                    c.setopt(c.SSL_VERIFYPEER, False)
            else:
                c.setopt(c.SSL_VERIFYHOST, False)
                c.setopt(c.SSL_VERIFYPEER, False)

            # set headers
            headers = self.get_headers(url, request, settings)
            if do_log:
                logging.info("headers = %s:" % headers)
            c.setopt(pycurl.HTTPHEADER, headers)
            # display request params, if any
            if do_log:
                if (curl_method == "POST" or curl_method == "PUT") and len(request) > 0:
                    logging.critical(
                        "*** %s *** [%s] %s (%s)" % (curl_method, datetime.now().strftime("%H:%M.%S"), the_url, request)
                    )
                else:
                    logging.critical("*** %s *** [%s] %s" % (curl_method, datetime.now().strftime("%H:%M.%S"), the_url))
            # send it
            c.perform()

            # get the response
            if return_type != "raw":
                response = body.getvalue().decode("utf-8")
            else:
                response = body.getvalue()
            rheaders = response_headers.getvalue().decode("utf-8")
            rheaders_array = rheaders.split("\n")
            status = rheaders_array[0][rheaders_array[0].index(" ") : rheaders_array[0].index(" ") + 4].strip()
            reason = rheaders_array[0][rheaders_array[0].index(" ") + 4 :].strip()
            if do_log:
                logging.info("http status = %s, %s" % (status, reason))
            if do_log:
                if len(response) > 0:
                    logging.info("response = %s:" % response)
            c.close()
        except Exception as inst:
            logging.error(inst)

        if return_type == "json":
            return response
        if return_type == "raw":
            return {"data": response, "status": status, "reason": reason, "headers": rheaders}
        elif return_type == "xml":
            try:
                data = objectify.fromstring(response)
                return {"data": data, "status": status, "reason": reason, "headers": rheaders}
            except Exception as inst:
                logging.debug(inst)
                try:
                    data = objectify.fromstring(str(response))  # converts from unicode if needed
                    return {"data": data, "status": status, "reason": reason, "headers": rheaders}
                except Exception as inst2:
                    logging.debug(inst2)
                    return {"data": "", "status": status, "reason": reason, "headers": rheaders}
        else:
            return {"data": response, "status": status, "reason": reason, "headers": rheaders}