def generate_test_cases(self, context_factory, parameterizations=None):
"""Creates the set of test cases that this test represents.
The parameterizations argument should contain a parameterizations registry
(keyed by parameterization name) containing values that are instances of
a checkers.Parameterization class.
Args:
context_factory: Callable to create a context instance given a TestCase.
parameterizations: (Registry) Parameterizations used to create test cases.
Returns:
list(TestCase): List of test cases (aka test closures).
"""
test_cases = registry.AutoKeyRegistry(lambda tc: tc.full_name)
if not parameterizations:
test_case = TestCase(self, context_factory,
description=self.description)
test_cases.register(test_case)
return test_cases
# It is a parameterized test, so we need to generate multiple test cases;
# one for each parameterization.
for suffix, param in parameterizations.iteritems():
name = '%s_%s' % (self.name, suffix)
full_name = '%s_%s' % (self.full_name, suffix)
test_case = TestCase(self, context_factory, name=name,
full_name=full_name, description=self.description)
for key, value in param.variables.iteritems():
test_case.context.variables.register(key, value)
for suite_name in param.suites:
test_case.test_suites.register(TestSuite(suite_name))
test_cases.register(test_case)
return test_cases
def get():
tests = []
diversed_whitelist = [
'node_Linear'
]
for gen in TESTS:
category = gen.dirname
name = gen.filename
test_name = 'elichika_%s_%s' % (category, name)
kwargs = {}
diversed = False
for substr in diversed_whitelist:
if substr in test_name:
diversed = True
break
test_dirs = glob.glob('out/%s' % test_name)
test_dirs += glob.glob('out/%s_*' % test_name)
for d in test_dirs:
name = os.path.basename(d)
test_dir = os.path.join('out', name)
tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
if diversed:
tests.append(TestCase(name=name + '_diversed',
test_dir=test_dir,
backend='xcvm_test',
**kwargs))
return tests
def __init__(self, test_path='', test_config=None):
"""Class that handles tests involving two-party bridges.
There are three Asterisk instances used for this test.
0 : the unit under test "UUT"
1 : the unit from which calls originate, also known as "Alice"
2 : the unit where calls terminate, also known as "Bob"
"""
TestCase.__init__(self, test_path)
self.test_runs = []
self.current_run = 0
self.ami_uut = None
self.ami_alice = None
self.ami_bob = None
self.call_end_observers = []
self.feature_start_observers = []
self.feature_end_observers = []
self.instances = 3
self.connections = 0
self.features = []
self.alice_channel = None
self.bob_channel = None
self.uut_alice_channel = None
self.uut_bob_channel = None
self.uut_bridge_id = None
self.alice_hungup = False
self.bob_hungup = False
self.uut_alice_hungup = False
self.uut_bob_hungup = False
self.current_feature = 0
self.infeatures = False
self.issue_hangups_on_bridged = False
self.bridged = False
self.audio_detected = False
self.feature_success = False
msg = ("BridgeTestCase hasn't raised the flag to indicate completion "
"of all expected calls.")
self.bridge_fail_token = self.create_fail_token(msg)
if test_config is None:
LOGGER.error("No configuration provided. Bailing.")
raise Exception
# Just a quick sanity check so we can die early if
# the tests are badly misconfigured
for test_run in test_config['test-runs']:
if not 'originate_channel' in test_run:
LOGGER.error("No configured originate channel in test run")
raise Exception
self.test_runs.append(test_run)
if 'asterisk-instances' in test_config:
self.instances = test_config['asterisk-instances']
self.create_asterisk(self.instances, "%s/configs/bridge" % os.getcwd())
LOGGER.info("Bridge test initialized")
def test_thread(param):
tests = create_test_case(param.file, param.step, param.r, param.f)
t1 = threading.Thread(target=run, args=(tests, test, 1))
t2 = threading.Thread(target=run, args=(tests, online, 2))
t1.setDaemon(True)
t2.setDaemon(True)
t1.start()
t2.start()
t1.join()
t2.join()
for tc in tests:
tc.check_response()
TestCase.show_reslut()
def get(target_opsets=[]):
tests = []
if len(target_opsets) is 0:
target_opsets = [10]
for opset in target_opsets:
for test_dir in sorted(glob.glob('out/opset{}/*'.format(opset))):
name = 'onnx_chainer_{}_{}'.format(
os.path.basename(test_dir), opset)
fail = ('dilated' in name or
'hard_sigmoid' in name or
'pad_edge' in name or
'pad_reflect' in name or
'prod_axis' in name or
'roipooling2d' in name or
'prelu' in name or
'tile' in name or
'resizeimages' in name)
equal_nan = ('powvarvar' in name or
'arcsin' in name or
'arccos' in name)
tests.append(TestCase(name=name,
test_dir=test_dir,
equal_nan=equal_nan,
fail=fail,
opset_version=opset))
return tests
def get_tests_for_suite(self, selected_suite_name):
"""Gets the test list for the suite"""
test_list = []
for test_instance in self.discover():
for test_method in test_instance.runnable_test_methods():
if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
test_list.append(test_method)
return test_list
def get_tests_for_suite(self, selected_suite_name):
"""Gets the test list for the suite"""
test_list = []
for test_instance in self.discover():
for test_method in test_instance.runnable_test_methods():
if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
test_list.append(test_method)
return test_list
def send(self, ignoreSend):
self._genHtmlCtn()
self._getTextCtn()
notifyFile = open("%s/notify.html" % TestCase.getWorkDir(), 'w')
notifyFile.write(self.html)
notifyFile.close()
if ignoreSend or not TestCase.checkParam('email.enable', 'True'):
logger.info("No email notification")
return
if TestCase.isForceSkip():
emailTo = self.emailFrom
else:
emailTo = TestCase.getParamEval('email.toList')
if emailTo is None:
emailTo = [self.emailFrom]
else:
emailTo.append(self.emailFrom)
msg = MIMEMultipart('alternative')
msg['Subject'] = "[Test Result] %s" % TestCase.testCtx.desc
msg['From'] = self.emailFrom
msg['To'] = ';'.join(emailTo)
part1 = MIMEText(self.text, 'plain')
part2 = MIMEText(self.html, 'html')
msg.attach(part1)
msg.attach(part2)
try:
server = smtplib.SMTP(self.stmpSvr) # connect, no login step
except:
logger.error('Failed to connect the email server, %s',
self.stmpSvr)
return False
failed = server.sendmail(self.emailFrom, emailTo, msg.as_string())
server.quit()
if failed: # smtplib may raise exceptions
logger.error("Failed to send mail due to %s", failed)
return False
else:
logger.info("Send test result email to %s", emailTo)
return True
def list_tests(self, selected_suite_name=None):
"""Lists all tests, optionally scoped to a single suite."""
test_list = []
for test_instance in self.discover():
for test_method in test_instance.runnable_test_methods():
if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
test_list.append(test_method)
pp = pprint.PrettyPrinter(indent=2)
print(pp.pformat([self.get_test_method_name(test) for test in test_list]))
def get_test_cases(file_url):
with open(file_url) as f:
test_cases_dicts = yaml.safe_load_all(f)
test_cases = []
for test_case_dict in test_cases_dicts.next():
test_case = TestCase.from_dict(test_case_dict)
test_cases.append(test_case)
return test_cases
def get():
tests = []
skip_shape_inference_blacklist = [
# A bug of ONNX's shape inference?
# TODO(hamaji): Investigate.
'node_SwapAxes',
]
diversed_whitelist = ['node_Linear']
for category, names in [('model', MODEL_TESTS), ('node', NODE_TESTS),
('syntax', SYNTAX_TESTS)]:
for name in names:
test_name = 'ch2o_%s_%s' % (category, name)
kwargs = {}
for substr in skip_shape_inference_blacklist:
if substr in test_name:
kwargs['skip_shape_inference'] = True
break
diversed = False
for substr in diversed_whitelist:
if substr in test_name:
diversed = True
break
test_dirs = glob.glob('out/%s' % test_name)
test_dirs += glob.glob('out/%s_*' % test_name)
for d in test_dirs:
name = os.path.basename(d)
test_dir = os.path.join('out', name)
tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
if diversed:
tests.append(
TestCase(name=name + '_diversed',
test_dir=test_dir,
backend='xcvm_test',
**kwargs))
return tests
def get():
tests = []
diversed_whitelist = [
'node_Linear'
]
for gen in TESTS:
category = gen.category
name = gen.filename
test_name = 'elichika_%s_%s' % (category, name)
kwargs = {}
if gen.fail:
kwargs['fail'] = True
# TODO(hamaji): Remove this once type inference is improved.
if ('EspNet_Decoder' in test_name or
'EspNet_E2E' in test_name):
kwargs['skip_runtime_type_check'] = True
diversed = False
for substr in diversed_whitelist:
if substr in test_name:
diversed = True
break
test_dirs = glob.glob('out/%s' % test_name)
test_dirs += glob.glob('out/%s_*' % test_name)
assert test_dirs, 'No tests found for %s' % test_name
for d in test_dirs:
name = os.path.basename(d)
test_dir = os.path.join('out', name)
tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
if diversed:
tests.append(TestCase(name=name + '_diversed',
test_dir=test_dir,
backend='chxvm_test',
**kwargs))
return tests
def get_udp_data(self, step=100, r=0, f=None):
print "read cap file :" + self.cappath
pcap = dpkt.pcap.Reader(f)
post_data = []
start_time = int(time.time() * 1000)
i = r*step
for ts, buf in pcap:
if i >= (r+1)*step:
break
try:
eth = dpkt.ethernet.Ethernet(buf)
ip = eth.data
udp = ip.data
except Exception, e:
print Exception, ":", e
continue
try:
test = TestCase()
test.id = i
test.data = udp.data
test.seq = self.get_seq(test.data)
test.request = self.get_request(test.data)
post_data.append(test)
i += 1
except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
continue
def list_tests(self, selected_suite_name=None):
"""Lists all tests, optionally scoped to a single suite."""
test_list = []
for test_instance in self.discover():
for test_method in test_instance.runnable_test_methods():
if not selected_suite_name or TestCase.in_suite(
test_method, selected_suite_name):
test_list.append(test_method)
pp = pprint.PrettyPrinter(indent=2)
print(
pp.pformat([self.get_test_method_name(test)
for test in test_list]))
def get():
tests = []
for test_dir in sorted(glob.glob('out/opset10/*')):
name = 'onnx_chainer_' + os.path.basename(test_dir)
fail = ('dilated' in name or 'hard_sigmoid' in name
or 'pad_edge' in name or 'pad_reflect' in name
or 'prod' in name or 'roipooling2d' in name or 'prelu' in name
or 'tile' in name or 'group3' in name
or 'resizeimages' in name)
tests.append(TestCase(name=name, test_dir=test_dir, fail=fail))
return tests
def _getInfoSec(self):
while True:
infos = TestCase.getParamEval('email.info')
bench = TestCase.getParamExactEval('bench')
if infos is None and bench is None:
break
if bench is not None:
if isinstance(bench, basestring):
bench = [bench]
if infos is None:
infos = ['Test Bench', bench['desc']]
else:
infos.append(['Test Bench', bench['desc']])
for info in infos:
self.html += '<h1>%s</h1><ul style="list-style-type:square">' % info[
0]
for item in info[1]:
self.html += '<li>%s</li>' % item
self.html += '</ul>\n'
break
def list_tests(self, selected_suite_name=None):
"""Lists all tests, optionally scoped to a single suite."""
test_list = []
for test_case_class in self.test_case_classes:
test_instance = test_case_class(
suites_include=self.suites_include,
suites_exclude=self.suites_exclude,
suites_require=self.suites_require)
for test_method in test_instance.runnable_test_methods():
if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
test_list.append(test_method)
pp = pprint.PrettyPrinter(indent=2)
print(pp.pformat([self.get_test_method_name(test) for test in test_list]))
def _append_relevant_results_and_log_relevant_failures(result):
"""Log the results of test methods."""
if not test_case.is_fixture_method(result.test_method):
if not test_case.method_excluded(result.test_method):
self.logger.report_test_result(result)
results.append(result)
elif result.test_method._fixture_type == 'class_teardown' and (result.failure or result.error):
# For a class_teardown failure, log the name too (since it wouldn't have
# already been logged by on_run_test_method).
self.logger.report_test_name(result.test_method)
self.logger.report_test_result(result)
results.append(result)
if not result.success and not TestCase.in_suite(result.test_method, 'expected-failure'):
self.logger.failure(result)
def get():
tests = []
for data_json in glob.glob('onnx/onnx/backend/test/data/real/*/data.json'):
with open(data_json) as f:
json_content = f.read()
json_data = json.loads(json_content)
downloader = _Downloader(json_content, json_data)
rtol = None
if downloader.name == 'densenet121':
rtol = 1e-3
# TODO(hamaji): Investigate the fairly large error.
if downloader.name == 'inception_v1':
rtol = 2
tests.append(
TestCase('out',
downloader.test_name(),
prepare_func=downloader.prepare,
want_gpu=True,
rtol=rtol))
return tests
def read_program(status_path, traces_path, instrument_path):
statuses = read_file(status_path).split('\n')
traces = read_file(traces_path).split('\n')
if len(statuses) == 0:
print 'status length 0'
if len(traces) == 0:
print 'traces length 0'
if len(statuses) != len(traces):
print 'length of traces and status are different'
testcases = list([])
for i in xrange(0, len(statuses)):
status = True if 0 == statuses[i].split(':')[1] else False
trace = traces[i].split(':')[1]
testcase = TestCase(trace, status)
testcases.append(testcase)
instrument = read_file(instrument_path).split('\n')
if len(instrument) < 2:
print 'can not read instrument info'
fault_line = instrument[0].split(':')[1].split(',')
fault = [int(e) for e in fault_line]
total = int(instrument[1].split(':')[1])
return Program(testcases, fault, total)
def __init__(self, name): self.wasRun = None self.wasSetUp = 1 self.name = name self.log = "setUp " TestCase.__init__(self, name)
def discover(self, path):
test_suite = TestSuite()
for api_file in self.collect(path):
test_case = TestCase(api_file)
test_suite.add_case(test_case)
return test_suite
def run(self):
"""Override of TestCase.run"""
TestCase.run(self)
self.create_ami_factory(self.instances)
def run(self, project_id, id, browser):
try:
logger.info('正在查询项目[ID：%s]相关信息' % project_id)
result = test_platform_db.select_one_record(
'SELECT home_page, environment_id, valid_flag '
'FROM `website_ui_project_setting` WHERE id = %s',
(project_id, ))
if result[0] and result[1]:
home_page, environment_id, valid_flag = result[1]
logger.info('正在查询与项目关联的数据库信息')
result = test_platform_db.select_many_record(
"SELECT db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd "
"FROM `website_database_setting` "
"WHERE locate('UI%s', project_id) != 0 AND environment_id= '%s'"
% (project_id, environment_id))
if result[0] and result[1]:
for record in result[1]:
db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd = record
if db_type == 'MySQL':
mydb = MyDB(db_name=db_name,
db_host=db_host,
port=db_port,
user=db_user,
password=db_passwd,
charset='utf8')
db_related_to_project_dic[db_alias] = mydb
elif not result[0]:
logger.error('查询项目相关的数据库配置信息出错：%s' % result[1])
return [False, result[1]]
logger.info('正在查询与项目关联的全局变量')
result = test_platform_db.select_many_record(
"SELECT `name`, `value` "
"FROM `website_global_variable_setting` "
"WHERE project_type='UI项目' AND locate('%s', project_id) != 0 AND locate('%s', env_id) != 0 "
% (self.project_id, environment_id))
if result[0] and result[1]:
for record in result[1]:
name, value = record
name = name
global_variable_dic[name] = value
elif not result[0]:
logger.error('查询项目相关的全局变量配置信息出错：%s' % result[1])
return [False, result[1]]
host_port = re.findall(
'[https|http]+://[^/]+', home_page
) # 获取http地址 形如 http://www.baidu.com, http://www.baidu.com:8080
if host_port:
host_port = host_port[0]
else:
return [False, '项目主页填写错误']
logger.info('正在查询输入ID标识的用例(套件)相关信息')
query = 'SELECT id, text FROM `website_ui_case_tree` WHERE project_id = %s AND id = %s' % (
project_id, id)
result = test_platform_db.select_one_record(query)
logger.info('正在获取浏览器驱动')
result_get = selenium_util.set_driver(browser)
if not result_get[0]:
logger.error('获取浏览器驱动出错，退出')
exit()
browser_driver = selenium_util.get_driver()
logger.info('正在打开项目主页：%s' % home_page)
selenium_util.maximize_window()
selenium_util.get(home_page)
selenium_util.implicitly_wait(20)
try:
if result[0] and result[1]:
record = result[1]
case_id, case_name = record
execution_num = str(int(time.time())) # 执行编号
query = 'SELECT id, text FROM `website_ui_case_tree` WHERE project_id = %s AND parent_id = %s ' \
'AND id NOT IN (SELECT parent_id FROM `website_ui_case_tree` WHERE project_id=%s)' \
'ORDER BY `order` ASC' % (project_id, id, project_id)
result = test_platform_db.select_many_record(query)
if result[0] and result[1]:
logger.info('输入ID标识的是测试套件')
records = result[1]
for record in records:
case_id, case_name = record
test_case = TestCase(execution_num, 0, case_id,
'--', case_name,
host_port)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(True)
if not result[0]:
return [
False,
'用例（ID:%s 名称：%s）运行出错：%s' %
(case_id, case_name, result[2])
]
elif result[0] and not result[1]:
logger.info('输入ID标识的是测试用例，开始执行用例')
test_case = TestCase(execution_num, 0, case_id,
'--', case_name, host_port)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(True)
if not result[0]:
return [
False,
'用例（ID:%s 名称：%s）运行出错：%s' %
(case_id, case_name, result[2])
]
else:
logger.error('查询出错:%s' % result[1])
return [False, result[1]]
elif result[0] and not result[1]:
reason = '未查找到相关信息，请检查配置的项目ID(%s)，用例(套件)标识ID(%s)是否正确'
logger.warn(reason)
return [False, reason]
else:
logger.error('查找相关信息失败：%s' % result[1])
return [False, '查找相关信息失败：%s' % result[1]]
except Exception as e:
logger.error('运行出错：%s' % e)
finally:
browser_driver.quit()
elif result[0] and not result[1]:
logger.error('未查询到项目相关的信息')
return [False, '未查询到项目相关的信息']
else:
logger.error('查询项目相关信息失败：%s' % result[1])
return [False, '查询项目相关信息失败：%s' % result[1]]
except Exception as e:
logger.error('%s' % e)
return [False, '%s' % e]
finally:
logger.info('正在释放资源')
logger.info('正在断开与项目关联的数据库连接')
# 关闭数据库
for key, db in db_related_to_project_dic.copy().items():
db.close()
del db_related_to_project_dic[key]
logger.info('正在清理与项目关联的全局变量')
global_variable_dic.clear()
help='Force setting --computation_order flag')
parser.add_argument('--verbose',
action='store_true',
help='Run tests with --verbose flag')
args = parser.parse_args()
GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
RESET = '\033[0m'
ONNX_TEST_DATA = 'third_party/onnx/onnx/backend/test/data'
NODE_TEST = os.path.join(ONNX_TEST_DATA, 'node')
TEST_CASES = [
TestCase(NODE_TEST, 'test_identity'),
TestCase(NODE_TEST, 'test_add'),
TestCase(NODE_TEST, 'test_add_bcast'),
TestCase(NODE_TEST, 'test_sub'),
TestCase(NODE_TEST, 'test_sub_bcast'),
TestCase(NODE_TEST, 'test_sub_example'),
TestCase(NODE_TEST, 'test_mul'),
TestCase(NODE_TEST, 'test_mul_bcast'),
TestCase(NODE_TEST, 'test_mul_example'),
TestCase(NODE_TEST, 'test_div'),
TestCase(NODE_TEST, 'test_div_bcast'),
TestCase(NODE_TEST, 'test_div_example'),
TestCase(NODE_TEST, 'test_pow'),
TestCase(NODE_TEST, 'test_pow_bcast_array'),
TestCase(NODE_TEST, 'test_pow_bcast_scalar'),
TestCase(NODE_TEST, 'test_pow_example'),
def __add_test_case__(self, name, input_tup, output_tup):
self.append(TestCase(name, input_tup, output_tup))
def __init__(self, name):
TestCase.__init__(self, name)
def run(self, debug):
try:
# 获取开始运行时间
timestamp_for_start = time.time()
start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
logger.info('正在查询测试计划关联的测试用例')
query = 'SELECT node_id, node_path, node_name FROM `website_api_case_tree_test_plan` WHERE plan_id = %s AND sub_node_num = 0 ORDER BY `order` ASC'
data = (self.plan_id, )
result = test_platform_db.select_many_record(query, data)
if result[0] and result[1]:
records = result[1]
execution_num = str(int(time.time())) # 执行编号
if not debug:
data = (execution_num, self.project_id, self.plan_id,
self.project_name, self.plan_name, start_time, '',
'', 0, 0, 0, 0, '')
logger.info('正在往测试报告-测试概况插入计划执行概要记录')
test_reporter.insert_report_for_summary(data)
flag = False
remark = ''
for record in records:
plan_id = self.plan_id
case_id, case_path, case_name = record
test_case = TestCase(execution_num, plan_id, case_id,
case_path, case_name, self.protocol,
self.host, self.port,
self.global_headers)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(debug)
if not result[0]:
flag = True # 有运行出错的用例
remark = '存在运行失败、被阻塞的用例'
if not debug:
end_time = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime()) # 结束运行时间
# 记录运行截止时间
timestamp_for_end = time.time()
logger.info('测试用例执行完毕，正在更新测试报告-测试概况表')
time_took = int(timestamp_for_end - timestamp_for_start)
days, hours, minutes, seconds = str(
time_took // 86400), str(
(time_took % 86400) // 3600), str(
((time_took % 86400) % 3600) // 60), str(
((time_took % 86400) % 3600) % 60)
time_took = days + '天 ' + hours + '小时 ' + minutes + '分 ' + seconds + '秒' # 运行耗时
case_pass_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '成功') # 运行成功用例数
case_fail_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '失败') # 运行失败用例数
case_block_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '阻塞') # 运行被阻塞用例数
case_total_num = case_block_num + case_fail_num + case_pass_num
data = (end_time, time_took, case_total_num, case_pass_num,
case_fail_num, case_block_num, remark,
execution_num, self.plan_id)
test_reporter.update_report_for_summary(data)
if flag:
return [False, '存在运行失败、被阻塞的用例']
else:
return [True, '执行成功']
elif result[0] and not result[1]:
reason = '未查找到同测试计划关联的用例'
logger.warn(reason)
return [False, reason]
else:
reason = '查找同测试计划[名称：%s, ID：%s]关联的用例失败：%s' % (
self.plan_name, self.plan_id, result[1])
logger.error(reason)
return [False, reason]
except Exception as e:
logger.error('%s' % e)
return [False, '%s' % e]
def setUp(self):
TestCase.setUp(self)
self.hidden_hilite = gfm.HiddenHiliteExtension([])
def __init__(self, name):
self.wasRun = None
TestCase.__init__(self, name)
def run(self, debug):
try:
# 获取开始运行时间
timestamp_for_start = time.time()
start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
logger.info('计划运行的浏览器有：%s' % (self.broswers))
logger.info('正在查询测试计划关联的测试用例')
query = 'SELECT node_id, node_path, node_name FROM `website_ui_case_tree_test_plan` WHERE plan_id = %s AND sub_node_num = 0 ORDER BY `order` ASC'
data = (self.plan_id, )
result = test_platform_db.select_many_record(query, data)
if result[0] and result[1]:
records = result[1]
browser_list_for_failure = []
mark = False # 用于标记是否有运行用例出错的浏览器
host_port = re.findall(
'[https|http]+://[^/]+', self.home_page
) # 获取http地址 形如 http://www.baidu.com, http://www.baidu.com:8080
if host_port:
host_port = host_port[0]
else:
return [False, '项目主页填写错误']
for browser in self.broswers:
execution_num = str(int(time.time())) # 执行编号
if not debug:
data = (execution_num, self.project_id, self.plan_id,
self.project_name, self.plan_name, browser,
start_time, '', '', 0, 0, 0, 0, '')
logger.info('正在往测试报告-测试概况插入计划执行概要记录')
test_reporter.insert_report_for_summary(data)
logger.info('正在获取浏览器驱动')
result = selenium_util.set_driver(browser)
if not result[0] and not debug: # 设置驱动出错
logger.error('获取浏览器驱动出错，跳过该浏览器的执行')
data = ('', '', 0, 0, 0, 0, result[1], execution_num,
self.plan_id)
test_reporter.update_report_for_summary(data)
continue
try:
browser_driver = selenium_util.get_driver()
selenium_util.maximize_window()
logger.info('正在打开项目主页：%s' % self.home_page)
selenium_util.get(self.home_page)
selenium_util.implicitly_wait(20)
flag = False # 用于标记是在每个浏览器下运行时，是否出现了运行出错的用例
logger.info(
'======================正在【%s】浏览器下运行测试用例======================'
% browser)
for record in records:
case_id, case_path, case_name = record
test_case = TestCase(execution_num, self.plan_id,
case_id, case_path, case_name,
host_port)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(debug)
if not result[0]:
flag = True # 有运行出错的用例
if flag:
mark = True
remark = '部分用例运行出错'
browser_list_for_failure.append(browser)
else:
remark = ''
except Exception as e:
logger.error('运行出错：%s' % e)
remark = '%s' % e
mark = True
finally:
browser_driver.close()
browser_driver.quit()
if not debug:
end_time = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime()) # 结束运行时间
# 记录运行截止时间
timestamp_for_end = time.time()
time_took = int(timestamp_for_end -
timestamp_for_start)
logger.info('测试用例执行完毕，正在更新测试报告-测试概况表')
days, hours, minutes, seconds = str(
time_took //
86400), str((time_took % 86400) // 3600), str(
((time_took % 86400) % 3600) // 60), str(
((time_took % 86400) % 3600) % 60)
time_took = days + '天 ' + hours + '小时 ' + minutes + '分 ' + seconds + '秒' # 运行耗时
case_pass_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '成功') # 运行成功用例数
case_fail_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '失败') # 运行失败用例数
case_block_num = test_reporter.get_case_num_by_run_result(
execution_num, self.plan_id, '阻塞') # 运行被阻塞用例数
case_total_num = case_block_num + case_fail_num + case_pass_num
data = (end_time, time_took, case_total_num,
case_pass_num, case_fail_num,
case_block_num, remark, execution_num,
self.plan_id)
test_reporter.update_report_for_summary(data)
if mark:
return [
False,
'测试计划在浏览器%s下运行失败' % str(browser_list_for_failure)
]
else:
return [True, '执行成功']
elif result[0] and not result[1]:
reason = '未查找到同测试计划关联的用例'
logger.warn(reason)
return [False, reason]
else:
reason = '查找同测试计划[名称：%s, ID：%s]关联的用例失败：%s' % (
self.plan_name, self.plan_id, result[1])
logger.error(reason)
return [False, reason]
except Exception as e:
logger.error('%s' % e)
return [False, '%s' % e]
def FromProto(proto):
test_suite = TestSuite(str(proto.name), str(proto.description))
for test_case_proto in proto.test_case:
test_case = TestCase.FromProto(test_case_proto)
test_suite.AddTestCase(test_case)
return test_suite
def main():
parser = argparse.ArgumentParser(
description="PyTorch Object Detection Training")
parser.add_argument(
"--config-file",
default="",
metavar="FILE",
help="path to config file",
type=str,
)
parser.add_argument("--local_rank", type=int, default=0)
parser.add_argument("--sync_bn", action="store_true")
parser.add_argument(
"--skip-test",
dest="skip_test",
help="Do not test the final model",
action="store_true",
)
parser.add_argument(
"opts",
help="Modify config options using the command-line",
default=None,
nargs=argparse.REMAINDER,
)
args = parser.parse_args()
num_gpus = int(
os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
args.distributed = num_gpus > 1
assert num_gpus == 2, "unittest only for 2 gpus"
if args.distributed:
torch.cuda.set_device(args.local_rank)
torch.distributed.init_process_group(backend="nccl",
init_method="env://")
device = torch.device('cuda')
model = DistributedSyncBN(3).to(device)
nn.init.constant_(model.weight, 1)
if args.distributed:
model = torch.nn.parallel.DistributedDataParallel(
model,
device_ids=[args.local_rank],
output_device=args.local_rank,
# this should be removed if we update BatchNorm stats
broadcast_buffers=False,
)
# check train
np.random.seed(args.local_rank)
x = torch.from_numpy(np.random.rand(2, 3, 2, 2)).float()
x.to(device)
y = model(x)
z = F.relu(y)
z = z.sum()
z.backward()
np.random.seed(0)
x1 = np.random.rand(2, 3, 2, 2)
np.random.seed(1)
x2 = np.random.rand(2, 3, 2, 2)
x = np.concatenate((x1, x2), axis=0).astype(float)
xv = x.reshape(4, 3, -1)
mean = np.mean(np.mean(xv, axis=0, keepdims=True), axis=2, keepdims=True)
a, b, c = xv.shape
var = (np.var(np.transpose(xv, [0, 2, 1]).reshape((a * c, b)),
axis=0,
ddof=1).reshape((1, b, 1)))
sd = np.sqrt(var + model.module.eps)
y_expect = (xv - mean) / sd
y_expect = y_expect.reshape(x.shape)
test_case = TestCase()
if dist.get_rank() == 0:
test_case.assertTensorClose(y_expect[:2],
y.detach().cpu().numpy(),
max_err=5e-6)
else:
test_case.assertTensorClose(y_expect[2:],
y.detach().cpu().numpy(),
max_err=5e-6)
def set_test_suite(self):
for api_file in self.api_file_list:
test_case = TestCase(api_file)
self.test_suite.add_case(test_case)
def run_test(path, using_eva):
try:
test = TestCase(path)
test.setup(using_eva)
test.exercise()
test.verify()
if not test.has_passed or True:
suffix = "-EVA" if using_eva else "-CIL"
dirname = os.path.join(result_dir, test.name) + suffix
os.mkdir(dirname)
test.store_all_data(dirname)
if using_eva:
results_eva[test.status] += 1
else:
results_cil[test.status] +=1
test.print_result(using_eva)
except NoSpecification:
print_err(f"Test case {test.name} missing specification")
def setUp(self):
TestCase.setUp(self)
self.hidden_hilite = gfm.HiddenHiliteExtension([])
def run(self, project_id, id):
try:
logger.info('正在查询项目[ID：%s]相关信息' % project_id)
result = test_platform_db.select_one_record(
'SELECT protocol, host, port, environment_id, valid_flag '
'FROM `website_api_project_setting` WHERE id = %s',
(project_id, ))
if result[0] and result[1]:
protocol, host, port, environment_id, valid_flag = result[1]
logger.info('正在查询与项目关联的数据库信息')
result = test_platform_db.select_many_record(
"SELECT db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd "
"FROM `website_database_setting` "
"WHERE locate('API%s', project_id) != 0 AND environment_id= '%s'"
% (project_id, environment_id))
if result[0] and result[1]:
for record in result[1]:
db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd = record
if db_type == 'MySQL':
mydb = MyDB(db_name=db_name,
db_host=db_host,
port=db_port,
user=db_user,
password=db_passwd,
charset='utf8')
db_related_to_project_dic[db_alias] = mydb
elif db_type == 'Redis':
if not db_passwd.strip():
db_passwd = None
if db_name.strip() == '':
db_name = '0'
myredis = RedisClient(host=db_host,
port=db_port,
password=db_passwd,
db=db_name,
charset='utf-8')
redis_related_to_project_dic[db_alias] = myredis
elif not result[0]:
logger.error('查询项目相关的数据库配置信息出错：%s' % result[1])
return [False, result[1]]
logger.info('正在查询与项目关联的全局变量')
result = test_platform_db.select_many_record(
"SELECT `name`, `value` "
"FROM `website_global_variable_setting` "
"WHERE project_type='API项目' AND locate('%s', project_id) != 0 AND locate('%s', env_id) != 0 "
% (project_id, environment_id))
if result[0] and result[1]:
for record in result[1]:
name, value = record
name = name
global_variable_dic[name] = value
elif not result[0]:
logger.error('查询项目相关的全局变量配置信息出错：%s' % result[1])
return [False, result[1]]
try:
if 'global_headers' in global_variable_dic.keys():
global_headers = global_variable_dic['global_headers']
# 防止用户输入了中文冒号，替换为英文冒号,不然经过global_headers.encode("utf-8").decode("latin1")这样编码转换，
# 会把"key"：中的中文冒号解码为非英文冒号，导致执行json loads函数时会报错；
# 另外，请求头从数据库读取，可能涉及到换行符，需要去掉
global_headers = global_headers.replace('：',
':').replace(
'\t', '')
global_headers = json.loads(
global_headers, object_pairs_hook=OrderedDict)
else:
global_headers = {}
except Exception as e:
logger.error('%s' % e)
return [False, '%s' % e]
logger.info('正在查询输入ID标识的用例(套件)相关信息')
query = 'SELECT id, text FROM `website_api_case_tree` WHERE project_id = %s AND id = %s' % (
project_id, id)
result = test_platform_db.select_one_record(query)
if result[0] and result[1]:
record = result[1]
case_id, case_name = record
execution_num = str(int(time.time())) # 执行编号
query = 'SELECT id, text FROM `website_api_case_tree` WHERE project_id = %s AND parent_id = %s ' \
'AND id NOT IN (SELECT parent_id FROM `website_api_case_tree` WHERE project_id=%s)' \
'ORDER BY `order` ASC' % (project_id, id, project_id)
result = test_platform_db.select_many_record(query)
if result[0] and result[1]:
logger.info('输入ID标识的是测试套件')
records = result[1]
for record in records:
case_id, case_name = record
test_case = TestCase(execution_num, 0, case_id,
'--', case_name, protocol,
host, port, global_headers)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(True)
if not result[0]:
return [
False,
'用例（ID:%s 名称：%s）运行出错：%s' %
(case_id, case_name, result[2])
]
elif result[0] and not result[1]:
logger.info('输入ID标识的是测试用例，开始执行用例')
test_case = TestCase(execution_num, 0, case_id, '--',
case_name, protocol, host, port,
global_headers)
logger.info(
'======================开始运行测试用例[名称：%s, ID:%s]======================'
% (case_name, case_id))
result = test_case.run(True)
if not result[0]:
return [
False,
'用例（ID:%s 名称：%s）运行出错：%s' %
(case_id, case_name, result[2])
]
else:
logger.error('查询出错:%s' % result[1])
return [False, result[1]]
elif result[0] and not result[1]:
reason = '未查找到相关信息，请检查配置的项目ID(%s)，用例(套件)标识ID(%s)是否正确'
logger.warn(reason)
return [False, reason]
else:
logger.error('查找相关信息失败：%s' % result[1])
return [False, '查找相关信息失败：%s' % result[1]]
elif result[0] and not result[1]:
logger.error('未查询到项目相关的信息')
return [False, '未查询到项目相关的信息']
else:
logger.error('查询项目相关信息失败：%s' % result[1])
return [False, '查询项目相关信息失败：%s' % result[1]]
except Exception as e:
logger.error('%s' % e)
return [False, '%s' % e]
finally:
logger.info('正在释放资源')
logger.info('正在断开与项目关联的数据库连接')
# 关闭数据库
for key, db in db_related_to_project_dic.copy().items():
db.close()
del db_related_to_project_dic[key]
logger.info('正在清理与项目关联的全局变量')
global_variable_dic.clear()
