def setup(self, args):
# Parse command line options
parser = PassThroughOptionParser()
parser.add_option('-a', '--virtual_config_file', default=None, type='string')
parser.add_option('-b', '--browser', default=None, type='string')
parser.add_option('-c', '--config_file', default=None, type='string')
parser.add_option('-d', '--db', default="false", type='string')
parser.add_option('-e', '--excluded', default=None, type='string')
parser.add_option('-g', '--grid_address', default=None, type='string')
parser.add_option('-i', '--override_settings', default=None, type='string') # override global settings with this comma-separated, key:value string
parser.add_option('-l', '--logfile', default=None, type='string')
parser.add_option('-m', '--modules', default=None, type='string')
parser.add_option('-o', '--override_params', default=None, type='string') # override specific test params with this comma-separated, key:value string
parser.add_option('-s', '--settings', default=None, type='string')
parser.add_option('-t', '--test', default=None, type='string')
parser.add_option('-u', '--url', default=None, type='string')
parser.add_option('-y', '--test_type', default=None, type='string')
parser.add_option('-z', '--loadtest_settings', default=None,
type='string')
options, args_out = parser.parse_args(args)
self.override_params = options.override_params
self.override_settings = options.override_settings
global_settings = options.settings
global_settings_config = None
config = {}
sub_configs = []
test_overrides = {}
if options.config_file == None and options.virtual_config_file == None and options.settings == None:
print '-c (--config_file) or -a (--virtual_config_file) or -s (--settings) is required'
sys.exit(2)
if options.logfile != None:
''' redirect stdout to the logfile '''
f = open(options.logfile, "a", 0)
sys.stdout = f
if options.excluded != None:
''' build a list of tests explicitly excluded from the
command-line option '''
excluded_list = options.excluded.split(',')
else:
excluded_list = None
''' load the config file '''
# optional global settings config file
# can be a comma-separated list, in which case the global_settings config is the first item
# and any number of config files can be added to it, merging fields as needed
# global settings override all sub_config settings, but are overridden by command-line -i overrides
# tests['includes'] and tests['excludes'] in sub_configs are simply added together
# an exclude in any sub_config will override any include of the same test elsewhere in the merged configs
if global_settings != None:
gsettings = global_settings.split(",")
config_path = os.getenv('PYTAFHOME') + os.sep + "config" + os.sep
#print gsettings
# first import the "global" or top-most, settings file
f = open("%s%s" % (config_path, gsettings[0]), 'r').read()
global_settings_config = json.loads(f)
test_overrides = global_settings_config.get('test_overrides', {})
#print test_overrides
#print global_settings_config
for i in range(1, len(gsettings)):
sub_configs.append(gsettings[i])
# handle the virtual config file
try:
if options.virtual_config_file != None:
#print options.virtual_config_file
config = json.loads(options.virtual_config_file)
db_config = {}
elif options.config_file != None:
config_path = os.getenv('PYTAF_HOME') + os.sep + "config" + os.sep
f = open("%s%s" % (config_path,options.config_file), 'r').read()
config = json.loads(f)
# allow for the possibility of nested config files
new_settings = {}
# allow for an import from override config files
config_to_import = config.get('import', None)
if config_to_import != None:
sub_configs.append(config_to_import)
if global_settings_config != None and len(sub_configs) > 0:
top_config = global_settings_config
else:
top_config = config # {} at this point
# import each sub_config and merge
if len(sub_configs) > 0:
for sub_config in sub_configs:
#print "sub_config file = %s" % sub_config
f = open("%s%s" % (config_path, sub_config), 'r').read()
imported_config = json.loads(f)
# let any original config 'settings' override the imported config's settings by merging the dictionaries
if imported_config.has_key('settings'):
new_settings = dict(imported_config['settings'].items() + top_config['settings'].items())
# if there's a global settings config, it will trump all settings
if global_settings_config != None:
new_settings = dict(new_settings.items() + global_settings_config['settings'].items())
# the new, merged config's settings
config['settings'] = new_settings
try:
config['settings']['config_file'] += "," + sub_config
except: # if not already there, initialize it
config['settings']['config_file'] = gsettings[0] + "," + sub_config
top_config['settings'] = config['settings'] # update this as well
if imported_config.has_key('tests'):
#print "merge the tests includes sections"
if config.has_key('tests') == False:
config['tests'] = {}
try:
config['tests']['includes'] += imported_config['tests']['includes']
except:
config['tests']['includes'] = imported_config['tests']['includes']
if imported_config['tests'].has_key('excludes'):
for exclude_element in imported_config['tests']['excludes']:
for method in exclude_element['methods']:
#print "adding to exclude list: %s" % method['name']
excluded_list.append(method['name'])
# the params dictionary of individual tests can be overridden, e.g.
# "test_overrides":
# {
# "test_owner_add_private_channel_invalid_code": { "bad_code" : "AAAAACHECKITOUT" }
# }
test_overrides = dict(list(test_overrides.items()) + list(imported_config.get("test_overrides", {}).items()))
#print test_overrides
# additional excludes can be added to the base config file's excluded tests list, e.g:
#"additional_excludes":
# [
# "test_owner_manage_subscriptions"
# ]
#print "look for additional excludes"
additional_excludes = top_config.get('additional_excludes', [])
#print "additional excludes = %s" % additional_excludes
if len(excluded_list) == 0 and len(additional_excludes) > 0:
#print "set excluded_list to %s" % excluded_list
excluded_list = additional_excludes
else:
#print "else?"
for j in range (len(additional_excludes)):
excluded_list.append(additional_excludes[j])
#print "excluded_list now = %s" % excluded_list
#print "config = %s" % config
else: # if there's no overrides config but there is a global_settings config
# if there's a global settings config, it will trump all settings
if global_settings_config != None:
#print 'global settings override'
#print config['settings']
if config.has_key('settings'):
new_settings = dict(config['settings'].items() + global_settings_config['settings'].items())
config['settings'] = new_settings
else:
config['settings'] = global_settings_config['settings']
#print "config = %s" % config
except:
print pytaf_utils.formatExceptionInfo()
if len(sub_configs) > 0:
cf = gsettings
else:
cf = options.config_file
print "JSON problem in a config file: %s" % cf
sys.exit(2)
config['settings']['config_file'] = options.config_file
try: # try to open the default db_config file
f2 = open("%s%s" % (config_path, "db_config.json"), 'r').read()
db_config = json.loads(f2)
except:
db_config = {}
# command-line -u overrides config file for url
if options.url != None:
config['settings']['url'] = options.url # for browser tests
# command-line -b overrides config file for browser
# can be in the form of 'Firefox' or 'Firefox,10,WINDOWS'
if options.browser != None:
config['settings']['browser'] = options.browser
# command-line -g overrides config file setting for test_host (and optionally test_port as well)
# used for selenium_grid or local selenium server
if options.grid_address != None:
if options.grid_address.find(":") >= 0:
g = options.grid_address.split(":")
config['settings']['test_host'] = g[0]
config['settings']['test_port'] = int(g[1])
else:
config['settings']['test_host'] = options.grid_address
# reset the settings object for passing on to test methods
settings = config['settings']
# initialize the root logger
self.setup_logger(settings)
# dynamically import all modules found in the config file
if options.modules == None:
modules_array = pytaf_utils.get_all_modules(config)
else: # only load the specified module(s) from the config file
modules_array = options.modules.split(",")
logging.debug('modules: %s' % modules_array)
mapped_modules = map(__import__, modules_array)
passed = 0
failed = 0
if options.test_type == 'load':
'''
the command-line may override load_test_settings with
-z --loadtest_settings in the form of
duration:max_threads:ramp_steps:ramp_interval:throttle_rate
e.g. 3600,500,10,30,1
which would run the load test for 1 hour (3600 seconds)
ramping up to a total of 500 threads in 10 steps
(each step would add 50 threads (500/10))
and these batches of threads would be added in
30 second installments (approximately)
the final value (throttle_rate=1) is used to
brake the entire load test operation by sleeping for
that amount (in seconds) between chunks of test case allocations
'''
if options.loadtest_settings != None:
p = options.loadtest_settings.split(",")
if len(p) == 5:
config['settings']['load_test_settings'] = \
{"duration": int(p[0]),
"max_threads": int(p[1]),
"ramp_steps": int(p[2]),
"ramp_interval": int(p[3]),
"throttle_rate": int(p[4])}
else:
logging.fatal('load test settings are not complete.')
logging.fatal('they must be in the form of' \
'duration:max_threads:ramp_steps:ramp_interval:throttle_rate')
sys.exit(-1)
# now start the load test
passed, failed = self.do_load_test(mapped_modules, config)
# if --test is specified, try and get the params and run each one
elif options.test != None:
ts = options.test.split(",")
for i in range(0, len(ts)):
test = ts[i]
if self.test_excluded(test, excluded_list) == False:
params = pytaf_utils.get_params(config, test)
if params == None:
logging.fatal("could not find params for test %s" % test)
sys.exit()
else:
#if test_overrides.get(test, None):
# params = dict(params.items() + test_overrides[test].items())
params, settings = self.do_overrides(params, settings, test, test_overrides, self.override_settings, self.override_params)
status = self.do_test(mapped_modules, settings,
test, params)
if status == True:
passed = passed + 1
else:
failed = failed + 1
else:
logging.info("%s is on the excluded list" % test)
# if --test is not specified, collect and run all the
# tests in the config file
else:
tests = pytaf_utils.get_all_tests(config, mapped_modules)
for test in tests:
if self.test_excluded(test, excluded_list) == False:
params = pytaf_utils.get_params(config, test)
if params != None:
#if test_overrides.get(test, None):
# params = dict(params.items() + test_overrides[test].items())
params, settings = self.do_overrides(params, settings, test, test_overrides, self.override_settings, self.override_params)
status = self.do_test(mapped_modules, settings,
test, params)
if status == True:
passed = passed + 1
else:
failed = failed + 1
else:
logging.info("%s is on the excluded list" % test)
logging.info("---------------")
logging.info("Tests Run: %s" % (passed + failed))
logging.info("Passed: %s" % passed)
logging.info("Failed: %s" % failed)
print_results = []
for r in self.results:
print_results.append(r['status'] + " " + r['test_method'])
for r in sorted(print_results):
logging.info(r)
# post results to the database
if pytaf_utils.str2bool(options.db) == True:
pytaf_utils.post_results(self.results, settings,
db_config, passed, failed)
def setup(self, args):
# Parse command line options
parser = optparse.OptionParser()
parser.add_option("-b", "--browser", default=None, type="string")
parser.add_option("-c", "--config_file", default=None, type="string")
parser.add_option("-d", "--db", default="false", type="string")
parser.add_option("-e", "--excluded", default=None, type="string")
parser.add_option("-l", "--logfile", default=None, type="string")
parser.add_option("-m", "--modules", default=None, type="string")
parser.add_option("-s", "--selenium_server", default=None, type="string")
parser.add_option("-t", "--test", default=None, type="string")
parser.add_option("-u", "--url", default=None, type="string")
parser.add_option("-y", "--test_type", default=None, type="string")
parser.add_option("-z", "--loadtest_settings", default=None, type="string")
options, args_out = parser.parse_args(args)
if options.config_file == None:
print("-c (--config_file) is required")
sys.exit(-1)
if options.logfile != None:
""" redirect stdout to the logfile """
the_file = open(options.logfile, "a", 0)
sys.stdout = the_file
if options.excluded != None:
""" build a list of tests explicitly excluded from the
command-line option """
excluded_list = options.excluded.split(",")
else:
excluded_list = None
""" load the config file """
try:
config_path = os.getenv("PYTAF_HOME") + os.sep + "config" + os.sep
the_file = open("%s%s" % (config_path, options.config_file), "r").read()
config = json.loads(the_file)
except:
print(pytaf_utils.formatExceptionInfo())
print("problem with config file %s%s" % (config_path, options.config_file))
sys.exit()
try: # try to open the default db_config file
the_file = open("%s%s" % (config_path, "db_config.json"), "r").read()
db_config = json.loads(the_file)
except:
db_config = {}
# command-line -u overrides config file for url
if options.url != None:
config["settings"]["url"] = options.url # for browser tests
# command-line -b overrides config file for browser
# can be in the form of 'Firefox' or 'Firefox,10,WINDOWS'
if options.browser != None:
config["settings"]["browser"] = options.browser
# command-line -s option overrides the selenium_server
# host and port settings
if options.selenium_server != None:
if options.selenium_server.find(":") >= 0:
sel_list = options.selenium_server.split(":")
config["settings"]["selenium_host"] = sel_list[0]
config["settings"]["selenium_port"] = int(sel_list[1])
else:
config["settings"]["selenium_host"] = options.selenium_server
# reset the settings object for passing on to test methods
settings = config["settings"]
# dynamically import all modules found in the config file
if options.modules == None:
modules_array = pytaf_utils.get_all_modules(config)
else: # only load the specified module(s) from the config file
modules_array = options.modules.split(",")
if DEBUG:
print("modules: %s" % modules_array)
mapped_modules = map(__import__, modules_array)
passed = 0
failed = 0
if options.test_type == "load":
"""
the command-line may override load_test_settings with
-z --loadtest_settings in the form of
duration:max_threads:ramp_steps:ramp_interval:throttle_rate
e.g. 3600,500,10,30,1
which would run the load test for 1 hour (3600 seconds)
ramping up to a total of 500 threads in 10 steps
(each step would add 50 threads (500/10))
and these batches of threads would be added in
30 second installments (approximately)
the final value (throttle_rate=1) is used to
brake the entire load test operation by sleeping for
that amount (in seconds) between chunks of test case allocations
"""
if options.loadtest_settings != None:
load_list = options.loadtest_settings.split(",")
if len(load_list) == 5:
config["settings"]["load_test_settings"] = {
"duration": int(load_list[0]),
"max_threads": int(load_list[1]),
"ramp_steps": int(load_list[2]),
"ramp_interval": int(load_list[3]),
"throttle_rate": int(load_list[4]),
}
else:
print("load test settings are not complete.")
print("they must be in the form of" "duration:max_threads:ramp_steps:ramp_interval:throttle_rate")
sys.exit(-1)
# now start the load test
passed, failed = self.do_load_test(mapped_modules, config)
# if --test is specified, try and get the params and run each one
elif options.test != None:
test_list = options.test.split(",")
for i in range(0, len(test_list)):
test = test_list[i]
if self.test_excluded(test, excluded_list) == False:
params = pytaf_utils.get_params(config, test)
if params == None:
print("could not find params for test %s" % test)
sys.exit()
else:
status = self.do_test(mapped_modules, settings, test, params)
if status == True:
passed = passed + 1
else:
failed = failed + 1
else:
print("%s is on the excluded list" % test)
# if --test is not specified, collect and run all the
# tests in the config file
else:
tests = pytaf_utils.get_all_tests(config, mapped_modules)
for test in tests:
if self.test_excluded(test, excluded_list) == False:
params = pytaf_utils.get_params(config, test)
if params != None:
status = self.do_test(mapped_modules, settings, test, params)
if status == True:
passed = passed + 1
else:
failed = failed + 1
else:
print("%s is on the excluded list" % test)
print("---------------")
print("Tests Run: %s" % (passed + failed))
print("Passed: %s" % passed)
print("Failed: %s" % failed)
print_results = []
for result in self.results:
print_results.append(result["status"] + " " + result["test_method"])
for result in sorted(print_results):
print(result)
# post results to the database
if pytaf_utils.str2bool(options.db) == True:
pytaf_utils.post_results(self.results, settings, db_config, passed, failed)
def do_curl(
self,
url,
u,
request,
settings={},
curl_method="POST",
https=True,
return_type="json",
ignore_certs=False,
connection_timeout=60,
do_log=True,
):
response = {"data": "", "status": "", "reason": "", "headers": ""}
rheaders = ""
status = ""
reason = ""
# settings['httplib'] can override the use of pycurl and redirect to httlib, just as settings['webdriver'] can redirect selenium to webdriver for browser tests
if pytaf_utils.str2bool(settings.get("httplib", "false")) == True:
if curl_method.upper() == "POST":
logging.info("using httplib to POST instead of pycurl due to settings['httplib']")
return self.do_post(url, u, request, settings, https, connection_timeout, do_log)
elif curl_method.upper() == "GET":
logging.info("using httplib to GET instead of pycurl due to settings['httplib']")
return self.do_get(url, u, settings, https, connection_timeout, do_log)
try:
import pycurl
except:
return response
try:
# handle json stuff
if type(request) == dict:
request = json.dumps(request)
if len(request) > 0:
if do_log:
logging.info("request = %s" % request)
# setup response stuff
import StringIO
body = StringIO.StringIO()
response_headers = StringIO.StringIO()
c = pycurl.Curl()
if https:
if url.find("http") < 0:
the_url = "https://" + url + u
else:
the_url = url + u
if settings.get("use_ssl", False) == True:
c.setopt(c.SSLVERSION, c.SSLVERSION_SSLv3)
else:
c.setopt(c.SSLVERSION, c.SSLVERSION_TLSv1)
if settings.get("verify_peer", True) == False:
c.setopt(c.SSL_VERIFYPEER, False)
else:
if url.find("http") < 0:
the_url = "http://" + url + u
else:
the_url = url + u
c.setopt(c.WRITEFUNCTION, body.write)
c.setopt(c.HEADERFUNCTION, response_headers.write)
# optional bitrate limiting
if settings.get("bitrate_limit", None) != None:
c.setopt(c.MAX_RECV_SPEED_LARGE, int(settings.get("bitrate_limit")))
# handle method types
if curl_method == "POST":
c.setopt(c.POST, 1)
c.setopt(c.POSTFIELDS, str(request))
elif curl_method == "PUT":
c.setopt(c.CUSTOMREQUEST, "PUT")
c.setopt(c.POSTFIELDS, str(request))
elif curl_method == "DELETE":
c.setopt(c.CUSTOMREQUEST, "DELETE")
# c.setopt(c.POSTFIELDS, str(request))
# else: # "GET"
# c.setopt(c.GET, 1) # no such setting. pyCurl defaults to GET
# set basic cURL options
c.setopt(c.URL, str(the_url))
# c.VERBOSE is way too verbose - only enable for temporary debugging
# if DEBUG: c.setopt(c.VERBOSE, 1)
c.setopt(c.FOLLOWLOCATION, 1)
c.setopt(c.MAXREDIRS, 5)
c.setopt(c.CONNECTTIMEOUT, connection_timeout)
# c.setopt(c.TIMEOUT, connection_timeout)
# handle optional cert stuff
CERT_FILE, CACERT, KEY_FILE = self.get_cert_paths(settings)
if ignore_certs == False:
if CERT_FILE:
c.setopt(c.SSLCERT, str(CERT_FILE))
c.setopt(c.SSLCERTTYPE, "PEM")
if CACERT:
c.setopt(c.CAINFO, str(CACERT))
if KEY_FILE:
c.setopt(c.SSLKEY, str(KEY_FILE))
c.setopt(c.SSLKEYTYPE, "PEM")
else:
# logging.debug('ignore certs')
c.setopt(c.SSL_VERIFYHOST, False)
c.setopt(c.SSL_VERIFYPEER, False)
else:
c.setopt(c.SSL_VERIFYHOST, False)
c.setopt(c.SSL_VERIFYPEER, False)
# set headers
headers = self.get_headers(url, request, settings)
if do_log:
logging.info("headers = %s:" % headers)
c.setopt(pycurl.HTTPHEADER, headers)
# display request params, if any
if do_log:
if (curl_method == "POST" or curl_method == "PUT") and len(request) > 0:
logging.critical(
"*** %s *** [%s] %s (%s)" % (curl_method, datetime.now().strftime("%H:%M.%S"), the_url, request)
)
else:
logging.critical("*** %s *** [%s] %s" % (curl_method, datetime.now().strftime("%H:%M.%S"), the_url))
# send it
c.perform()
# get the response
if return_type != "raw":
response = body.getvalue().decode("utf-8")
else:
response = body.getvalue()
rheaders = response_headers.getvalue().decode("utf-8")
rheaders_array = rheaders.split("\n")
status = rheaders_array[0][rheaders_array[0].index(" ") : rheaders_array[0].index(" ") + 4].strip()
reason = rheaders_array[0][rheaders_array[0].index(" ") + 4 :].strip()
if do_log:
logging.info("http status = %s, %s" % (status, reason))
if do_log:
if len(response) > 0:
logging.info("response = %s:" % response)
c.close()
except Exception as inst:
logging.error(inst)
if return_type == "json":
return response
if return_type == "raw":
return {"data": response, "status": status, "reason": reason, "headers": rheaders}
elif return_type == "xml":
try:
data = objectify.fromstring(response)
return {"data": data, "status": status, "reason": reason, "headers": rheaders}
except Exception as inst:
logging.debug(inst)
try:
data = objectify.fromstring(str(response)) # converts from unicode if needed
return {"data": data, "status": status, "reason": reason, "headers": rheaders}
except Exception as inst2:
logging.debug(inst2)
return {"data": "", "status": status, "reason": reason, "headers": rheaders}
else:
return {"data": response, "status": status, "reason": reason, "headers": rheaders}