Ejemplo n.º 1
0
  def generate_test_cases(self, context_factory, parameterizations=None):
    """Creates the set of test cases that this test represents.

    The parameterizations argument should contain a parameterizations registry
    (keyed by parameterization name) containing values that are instances of
    a checkers.Parameterization class.

    Args:
      context_factory: Callable to create a context instance given a TestCase.
      parameterizations: (Registry) Parameterizations used to create test cases.

    Returns:
      list(TestCase): List of test cases (aka test closures).
    """
    test_cases = registry.AutoKeyRegistry(lambda tc: tc.full_name)
    if not parameterizations:
      test_case = TestCase(self, context_factory,
                           description=self.description)
      test_cases.register(test_case)
      return test_cases
    # It is a parameterized test, so we need to generate multiple test cases;
    # one for each parameterization.
    for suffix, param in parameterizations.iteritems():
      name = '%s_%s' % (self.name, suffix)
      full_name = '%s_%s' % (self.full_name, suffix)
      test_case = TestCase(self, context_factory, name=name,
                           full_name=full_name, description=self.description)
      for key, value in param.variables.iteritems():
        test_case.context.variables.register(key, value)
      for suite_name in param.suites:
        test_case.test_suites.register(TestSuite(suite_name))
      test_cases.register(test_case)
    return test_cases
Ejemplo n.º 2
0
def get():
    tests = []

    diversed_whitelist = [
        'node_Linear'
    ]

    for gen in TESTS:
        category = gen.dirname
        name = gen.filename
        test_name = 'elichika_%s_%s' % (category, name)
        kwargs = {}

        diversed = False
        for substr in diversed_whitelist:
            if substr in test_name:
                diversed = True
                break

        test_dirs = glob.glob('out/%s' % test_name)
        test_dirs += glob.glob('out/%s_*' % test_name)
        for d in test_dirs:
            name = os.path.basename(d)
            test_dir = os.path.join('out', name)
            tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
            if diversed:
                tests.append(TestCase(name=name + '_diversed',
                                      test_dir=test_dir,
                                      backend='xcvm_test',
                                      **kwargs))

    return tests
    def __init__(self, test_path='', test_config=None):
        """Class that handles tests involving two-party bridges.

        There are three Asterisk instances used for this test.
        0 : the unit under test "UUT"
        1 : the unit from which calls originate, also known as "Alice"
        2 : the unit where calls terminate, also known as "Bob"
        """
        TestCase.__init__(self, test_path)
        self.test_runs = []
        self.current_run = 0
        self.ami_uut = None
        self.ami_alice = None
        self.ami_bob = None
        self.call_end_observers = []
        self.feature_start_observers = []
        self.feature_end_observers = []
        self.instances = 3
        self.connections = 0
        self.features = []
        self.alice_channel = None
        self.bob_channel = None
        self.uut_alice_channel = None
        self.uut_bob_channel = None
        self.uut_bridge_id = None
        self.alice_hungup = False
        self.bob_hungup = False
        self.uut_alice_hungup = False
        self.uut_bob_hungup = False
        self.current_feature = 0
        self.infeatures = False
        self.issue_hangups_on_bridged = False
        self.bridged = False
        self.audio_detected = False
        self.feature_success = False

        msg = ("BridgeTestCase hasn't raised the flag to indicate completion "
               "of all expected calls.")
        self.bridge_fail_token = self.create_fail_token(msg)

        if test_config is None:
            LOGGER.error("No configuration provided. Bailing.")
            raise Exception

        # Just a quick sanity check so we can die early if
        # the tests are badly misconfigured
        for test_run in test_config['test-runs']:
            if not 'originate_channel' in test_run:
                LOGGER.error("No configured originate channel in test run")
                raise Exception

            self.test_runs.append(test_run)

        if 'asterisk-instances' in test_config:
            self.instances = test_config['asterisk-instances']
        self.create_asterisk(self.instances, "%s/configs/bridge" % os.getcwd())
        LOGGER.info("Bridge test initialized")
Ejemplo n.º 4
0
def test_thread(param):
    tests = create_test_case(param.file, param.step, param.r, param.f)
    t1 = threading.Thread(target=run, args=(tests, test, 1))
    t2 = threading.Thread(target=run, args=(tests, online, 2))
    t1.setDaemon(True)
    t2.setDaemon(True)
    t1.start()
    t2.start()
    t1.join()
    t2.join()
    for tc in tests:
        tc.check_response()
    TestCase.show_reslut()
def get(target_opsets=[]):
    tests = []

    if len(target_opsets) is 0:
        target_opsets = [10]

    for opset in target_opsets:
        for test_dir in sorted(glob.glob('out/opset{}/*'.format(opset))):
            name = 'onnx_chainer_{}_{}'.format(
                os.path.basename(test_dir), opset)
            fail = ('dilated' in name or
                    'hard_sigmoid' in name or
                    'pad_edge' in name or
                    'pad_reflect' in name or
                    'prod_axis' in name or
                    'roipooling2d' in name or
                    'prelu' in name or
                    'tile' in name or
                    'resizeimages' in name)
            equal_nan = ('powvarvar' in name or
                         'arcsin' in name or
                         'arccos' in name)
            tests.append(TestCase(name=name,
                                  test_dir=test_dir,
                                  equal_nan=equal_nan,
                                  fail=fail,
                                  opset_version=opset))

    return tests
Ejemplo n.º 6
0
 def get_tests_for_suite(self, selected_suite_name):
     """Gets the test list for the suite"""
     test_list = []
     for test_instance in self.discover():
         for test_method in test_instance.runnable_test_methods():
             if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
                 test_list.append(test_method)
     return test_list
Ejemplo n.º 7
0
 def get_tests_for_suite(self, selected_suite_name):
     """Gets the test list for the suite"""
     test_list = []
     for test_instance in self.discover():
         for test_method in test_instance.runnable_test_methods():
             if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
                 test_list.append(test_method)
     return test_list
Ejemplo n.º 8
0
    def send(self, ignoreSend):
        self._genHtmlCtn()
        self._getTextCtn()
        notifyFile = open("%s/notify.html" % TestCase.getWorkDir(), 'w')
        notifyFile.write(self.html)
        notifyFile.close()

        if ignoreSend or not TestCase.checkParam('email.enable', 'True'):
            logger.info("No email notification")
            return

        if TestCase.isForceSkip():
            emailTo = self.emailFrom
        else:
            emailTo = TestCase.getParamEval('email.toList')

        if emailTo is None:
            emailTo = [self.emailFrom]
        else:
            emailTo.append(self.emailFrom)

        msg = MIMEMultipart('alternative')
        msg['Subject'] = "[Test Result] %s" % TestCase.testCtx.desc
        msg['From'] = self.emailFrom
        msg['To'] = ';'.join(emailTo)

        part1 = MIMEText(self.text, 'plain')
        part2 = MIMEText(self.html, 'html')

        msg.attach(part1)
        msg.attach(part2)

        try:
            server = smtplib.SMTP(self.stmpSvr)  # connect, no login step
        except:
            logger.error('Failed to connect the email server, %s',
                         self.stmpSvr)
            return False
        failed = server.sendmail(self.emailFrom, emailTo, msg.as_string())
        server.quit()
        if failed:  # smtplib may raise exceptions
            logger.error("Failed to send mail due to %s", failed)
            return False
        else:
            logger.info("Send test result email to %s", emailTo)
            return True
Ejemplo n.º 9
0
    def list_tests(self, selected_suite_name=None):
        """Lists all tests, optionally scoped to a single suite."""
        test_list = []
        for test_instance in self.discover():
            for test_method in test_instance.runnable_test_methods():
                if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
                    test_list.append(test_method)

        pp = pprint.PrettyPrinter(indent=2)
        print(pp.pformat([self.get_test_method_name(test) for test in test_list]))
def get_test_cases(file_url):
    with open(file_url) as f:
        test_cases_dicts = yaml.safe_load_all(f)

        test_cases = []
        for test_case_dict in test_cases_dicts.next():
            test_case = TestCase.from_dict(test_case_dict)
            test_cases.append(test_case)

        return test_cases
Ejemplo n.º 11
0
def get():
    tests = []

    skip_shape_inference_blacklist = [
        # A bug of ONNX's shape inference?
        # TODO(hamaji): Investigate.
        'node_SwapAxes',
    ]

    diversed_whitelist = ['node_Linear']

    for category, names in [('model', MODEL_TESTS), ('node', NODE_TESTS),
                            ('syntax', SYNTAX_TESTS)]:
        for name in names:
            test_name = 'ch2o_%s_%s' % (category, name)
            kwargs = {}
            for substr in skip_shape_inference_blacklist:
                if substr in test_name:
                    kwargs['skip_shape_inference'] = True
                    break

            diversed = False
            for substr in diversed_whitelist:
                if substr in test_name:
                    diversed = True
                    break

            test_dirs = glob.glob('out/%s' % test_name)
            test_dirs += glob.glob('out/%s_*' % test_name)
            for d in test_dirs:
                name = os.path.basename(d)
                test_dir = os.path.join('out', name)
                tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
                if diversed:
                    tests.append(
                        TestCase(name=name + '_diversed',
                                 test_dir=test_dir,
                                 backend='xcvm_test',
                                 **kwargs))

    return tests
Ejemplo n.º 12
0
def get():
    tests = []

    diversed_whitelist = [
        'node_Linear'
    ]

    for gen in TESTS:
        category = gen.category
        name = gen.filename
        test_name = 'elichika_%s_%s' % (category, name)
        kwargs = {}

        if gen.fail:
            kwargs['fail'] = True

        # TODO(hamaji): Remove this once type inference is improved.
        if ('EspNet_Decoder' in test_name or
            'EspNet_E2E' in test_name):
            kwargs['skip_runtime_type_check'] = True

        diversed = False
        for substr in diversed_whitelist:
            if substr in test_name:
                diversed = True
                break

        test_dirs = glob.glob('out/%s' % test_name)
        test_dirs += glob.glob('out/%s_*' % test_name)
        assert test_dirs, 'No tests found for %s' % test_name
        for d in test_dirs:
            name = os.path.basename(d)
            test_dir = os.path.join('out', name)
            tests.append(TestCase(name=name, test_dir=test_dir, **kwargs))
            if diversed:
                tests.append(TestCase(name=name + '_diversed',
                                      test_dir=test_dir,
                                      backend='chxvm_test',
                                      **kwargs))

    return tests
Ejemplo n.º 13
0
 def get_udp_data(self, step=100, r=0, f=None):
     print "read cap file :" + self.cappath
     pcap = dpkt.pcap.Reader(f)
     post_data = []
     start_time = int(time.time() * 1000)
     i = r*step
     for ts, buf in pcap:
         if i >= (r+1)*step:
             break
         try:
             eth = dpkt.ethernet.Ethernet(buf)
             ip = eth.data
             udp = ip.data
         except Exception, e:
             print Exception, ":", e
             continue
         try:
             test = TestCase()
             test.id = i
             test.data = udp.data
             test.seq = self.get_seq(test.data)
             test.request = self.get_request(test.data)
             post_data.append(test)
             i += 1
         except (dpkt.dpkt.NeedData, dpkt.dpkt.UnpackError):
             continue
Ejemplo n.º 14
0
    def list_tests(self, selected_suite_name=None):
        """Lists all tests, optionally scoped to a single suite."""
        test_list = []
        for test_instance in self.discover():
            for test_method in test_instance.runnable_test_methods():
                if not selected_suite_name or TestCase.in_suite(
                        test_method, selected_suite_name):
                    test_list.append(test_method)

        pp = pprint.PrettyPrinter(indent=2)
        print(
            pp.pformat([self.get_test_method_name(test)
                        for test in test_list]))
Ejemplo n.º 15
0
def get():
    tests = []

    for test_dir in sorted(glob.glob('out/opset10/*')):
        name = 'onnx_chainer_' + os.path.basename(test_dir)
        fail = ('dilated' in name or 'hard_sigmoid' in name
                or 'pad_edge' in name or 'pad_reflect' in name
                or 'prod' in name or 'roipooling2d' in name or 'prelu' in name
                or 'tile' in name or 'group3' in name
                or 'resizeimages' in name)
        tests.append(TestCase(name=name, test_dir=test_dir, fail=fail))

    return tests
Ejemplo n.º 16
0
    def _getInfoSec(self):
        while True:
            infos = TestCase.getParamEval('email.info')
            bench = TestCase.getParamExactEval('bench')
            if infos is None and bench is None:
                break

            if bench is not None:
                if isinstance(bench, basestring):
                    bench = [bench]

                if infos is None:
                    infos = ['Test Bench', bench['desc']]
                else:
                    infos.append(['Test Bench', bench['desc']])

            for info in infos:
                self.html += '<h1>%s</h1><ul style="list-style-type:square">' % info[
                    0]
                for item in info[1]:
                    self.html += '<li>%s</li>' % item
                self.html += '</ul>\n'
            break
Ejemplo n.º 17
0
    def list_tests(self, selected_suite_name=None):
        """Lists all tests, optionally scoped to a single suite."""
        test_list = []
        for test_case_class in self.test_case_classes:
            test_instance = test_case_class(
                suites_include=self.suites_include,
                suites_exclude=self.suites_exclude,
                suites_require=self.suites_require)
            for test_method in test_instance.runnable_test_methods():
                if not selected_suite_name or TestCase.in_suite(test_method, selected_suite_name):
                    test_list.append(test_method)

        pp = pprint.PrettyPrinter(indent=2)
        print(pp.pformat([self.get_test_method_name(test) for test in test_list]))
Ejemplo n.º 18
0
 def _append_relevant_results_and_log_relevant_failures(result):
     """Log the results of test methods."""
     if not test_case.is_fixture_method(result.test_method):
         if not test_case.method_excluded(result.test_method):
             self.logger.report_test_result(result)
         results.append(result)
     elif result.test_method._fixture_type == 'class_teardown' and (result.failure or result.error):
         # For a class_teardown failure, log the name too (since it wouldn't have 
         # already been logged by on_run_test_method).
         self.logger.report_test_name(result.test_method)
         self.logger.report_test_result(result)
         results.append(result)
     if not result.success and not TestCase.in_suite(result.test_method, 'expected-failure'):
         self.logger.failure(result)
def get():
    tests = []
    for data_json in glob.glob('onnx/onnx/backend/test/data/real/*/data.json'):
        with open(data_json) as f:
            json_content = f.read()
            json_data = json.loads(json_content)

        downloader = _Downloader(json_content, json_data)
        rtol = None
        if downloader.name == 'densenet121':
            rtol = 1e-3
        # TODO(hamaji): Investigate the fairly large error.
        if downloader.name == 'inception_v1':
            rtol = 2
        tests.append(
            TestCase('out',
                     downloader.test_name(),
                     prepare_func=downloader.prepare,
                     want_gpu=True,
                     rtol=rtol))
    return tests
def read_program(status_path, traces_path, instrument_path):
    statuses = read_file(status_path).split('\n')
    traces = read_file(traces_path).split('\n')
    if len(statuses) == 0:
        print 'status length 0'
    if len(traces) == 0:
        print 'traces length 0'
    if len(statuses) != len(traces):
        print 'length of traces and status are different'
    testcases = list([])
    for i in xrange(0, len(statuses)):
        status = True if 0 == statuses[i].split(':')[1] else False
        trace = traces[i].split(':')[1]
        testcase = TestCase(trace, status)
        testcases.append(testcase)
    instrument = read_file(instrument_path).split('\n')
    if len(instrument) < 2:
        print 'can not read instrument info'
    fault_line = instrument[0].split(':')[1].split(',')
    fault = [int(e) for e in fault_line]
    total = int(instrument[1].split(':')[1])
    return Program(testcases, fault, total)
Ejemplo n.º 21
0
 def __init__(self, name):
   self.wasRun   = None
   self.wasSetUp = 1
   self.name     = name
   self.log      = "setUp "
   TestCase.__init__(self, name)
Ejemplo n.º 22
0
 def discover(self, path):
     test_suite = TestSuite()
     for api_file in self.collect(path):
         test_case = TestCase(api_file)
         test_suite.add_case(test_case)
     return test_suite
 def run(self):
     """Override of TestCase.run"""
     TestCase.run(self)
     self.create_ami_factory(self.instances)
Ejemplo n.º 24
0
    def run(self, project_id, id, browser):
        try:
            logger.info('正在查询项目[ID:%s]相关信息' % project_id)
            result = test_platform_db.select_one_record(
                'SELECT home_page, environment_id, valid_flag '
                'FROM `website_ui_project_setting` WHERE id = %s',
                (project_id, ))
            if result[0] and result[1]:
                home_page, environment_id, valid_flag = result[1]

                logger.info('正在查询与项目关联的数据库信息')
                result = test_platform_db.select_many_record(
                    "SELECT db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd "
                    "FROM `website_database_setting` "
                    "WHERE  locate('UI%s', project_id) != 0  AND environment_id= '%s'"
                    % (project_id, environment_id))

                if result[0] and result[1]:
                    for record in result[1]:
                        db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd = record
                        if db_type == 'MySQL':
                            mydb = MyDB(db_name=db_name,
                                        db_host=db_host,
                                        port=db_port,
                                        user=db_user,
                                        password=db_passwd,
                                        charset='utf8')
                            db_related_to_project_dic[db_alias] = mydb
                elif not result[0]:
                    logger.error('查询项目相关的数据库配置信息出错:%s' % result[1])
                    return [False, result[1]]

                logger.info('正在查询与项目关联的全局变量')
                result = test_platform_db.select_many_record(
                    "SELECT `name`, `value` "
                    "FROM `website_global_variable_setting` "
                    "WHERE  project_type='UI项目' AND locate('%s', project_id) != 0 AND locate('%s', env_id) != 0 "
                    % (self.project_id, environment_id))

                if result[0] and result[1]:
                    for record in result[1]:
                        name, value = record
                        name = name
                        global_variable_dic[name] = value
                elif not result[0]:
                    logger.error('查询项目相关的全局变量配置信息出错:%s' % result[1])
                    return [False, result[1]]

                host_port = re.findall(
                    '[https|http]+://[^/]+', home_page
                )  # 获取http地址 形如 http://www.baidu.com, http://www.baidu.com:8080
                if host_port:
                    host_port = host_port[0]
                else:
                    return [False, '项目主页填写错误']

                logger.info('正在查询输入ID标识的用例(套件)相关信息')
                query = 'SELECT id, text FROM `website_ui_case_tree` WHERE project_id = %s AND id = %s' % (
                    project_id, id)
                result = test_platform_db.select_one_record(query)

                logger.info('正在获取浏览器驱动')
                result_get = selenium_util.set_driver(browser)
                if not result_get[0]:
                    logger.error('获取浏览器驱动出错,退出')
                    exit()
                browser_driver = selenium_util.get_driver()

                logger.info('正在打开项目主页:%s' % home_page)
                selenium_util.maximize_window()
                selenium_util.get(home_page)
                selenium_util.implicitly_wait(20)
                try:
                    if result[0] and result[1]:
                        record = result[1]
                        case_id, case_name = record
                        execution_num = str(int(time.time()))  # 执行编号

                        query = 'SELECT id, text FROM `website_ui_case_tree` WHERE project_id = %s AND parent_id = %s  ' \
                                'AND id NOT IN (SELECT parent_id FROM `website_ui_case_tree` WHERE project_id=%s)' \
                                'ORDER BY `order` ASC' % (project_id, id, project_id)
                        result = test_platform_db.select_many_record(query)

                        if result[0] and result[1]:
                            logger.info('输入ID标识的是测试套件')
                            records = result[1]

                            for record in records:
                                case_id, case_name = record
                                test_case = TestCase(execution_num, 0, case_id,
                                                     '--', case_name,
                                                     host_port)
                                logger.info(
                                    '======================开始运行测试用例[名称:%s, ID:%s]======================'
                                    % (case_name, case_id))
                                result = test_case.run(True)
                                if not result[0]:
                                    return [
                                        False,
                                        '用例(ID:%s 名称:%s)运行出错:%s' %
                                        (case_id, case_name, result[2])
                                    ]
                        elif result[0] and not result[1]:
                            logger.info('输入ID标识的是测试用例,开始执行用例')
                            test_case = TestCase(execution_num, 0, case_id,
                                                 '--', case_name, host_port)
                            logger.info(
                                '======================开始运行测试用例[名称:%s, ID:%s]======================'
                                % (case_name, case_id))
                            result = test_case.run(True)
                            if not result[0]:
                                return [
                                    False,
                                    '用例(ID:%s 名称:%s)运行出错:%s' %
                                    (case_id, case_name, result[2])
                                ]
                        else:
                            logger.error('查询出错:%s' % result[1])
                            return [False, result[1]]
                    elif result[0] and not result[1]:
                        reason = '未查找到相关信息,请检查配置的项目ID(%s),用例(套件)标识ID(%s)是否正确'
                        logger.warn(reason)
                        return [False, reason]
                    else:
                        logger.error('查找相关信息失败:%s' % result[1])
                        return [False, '查找相关信息失败:%s' % result[1]]
                except Exception as e:
                    logger.error('运行出错:%s' % e)
                finally:
                    browser_driver.quit()
            elif result[0] and not result[1]:
                logger.error('未查询到项目相关的信息')
                return [False, '未查询到项目相关的信息']
            else:
                logger.error('查询项目相关信息失败:%s' % result[1])
                return [False, '查询项目相关信息失败:%s' % result[1]]
        except Exception as e:
            logger.error('%s' % e)
            return [False, '%s' % e]
        finally:
            logger.info('正在释放资源')
            logger.info('正在断开与项目关联的数据库连接')
            # 关闭数据库
            for key, db in db_related_to_project_dic.copy().items():
                db.close()
                del db_related_to_project_dic[key]

            logger.info('正在清理与项目关联的全局变量')
            global_variable_dic.clear()
Ejemplo n.º 25
0
                    help='Force setting --computation_order flag')
parser.add_argument('--verbose',
                    action='store_true',
                    help='Run tests with --verbose flag')
args = parser.parse_args()

GREEN = '\033[92m'
YELLOW = '\033[93m'
RED = '\033[91m'
RESET = '\033[0m'

ONNX_TEST_DATA = 'third_party/onnx/onnx/backend/test/data'
NODE_TEST = os.path.join(ONNX_TEST_DATA, 'node')

TEST_CASES = [
    TestCase(NODE_TEST, 'test_identity'),
    TestCase(NODE_TEST, 'test_add'),
    TestCase(NODE_TEST, 'test_add_bcast'),
    TestCase(NODE_TEST, 'test_sub'),
    TestCase(NODE_TEST, 'test_sub_bcast'),
    TestCase(NODE_TEST, 'test_sub_example'),
    TestCase(NODE_TEST, 'test_mul'),
    TestCase(NODE_TEST, 'test_mul_bcast'),
    TestCase(NODE_TEST, 'test_mul_example'),
    TestCase(NODE_TEST, 'test_div'),
    TestCase(NODE_TEST, 'test_div_bcast'),
    TestCase(NODE_TEST, 'test_div_example'),
    TestCase(NODE_TEST, 'test_pow'),
    TestCase(NODE_TEST, 'test_pow_bcast_array'),
    TestCase(NODE_TEST, 'test_pow_bcast_scalar'),
    TestCase(NODE_TEST, 'test_pow_example'),
Ejemplo n.º 26
0
 def __add_test_case__(self, name, input_tup, output_tup):
     self.append(TestCase(name, input_tup, output_tup))
Ejemplo n.º 27
0
 def __init__(self, name):
     TestCase.__init__(self, name)
Ejemplo n.º 28
0
    def run(self, debug):
        try:
            # 获取开始运行时间
            timestamp_for_start = time.time()
            start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())
            logger.info('正在查询测试计划关联的测试用例')
            query = 'SELECT node_id, node_path, node_name FROM `website_api_case_tree_test_plan` WHERE plan_id = %s AND sub_node_num = 0 ORDER BY `order` ASC'
            data = (self.plan_id, )
            result = test_platform_db.select_many_record(query, data)
            if result[0] and result[1]:
                records = result[1]
                execution_num = str(int(time.time()))  # 执行编号

                if not debug:
                    data = (execution_num, self.project_id, self.plan_id,
                            self.project_name, self.plan_name, start_time, '',
                            '', 0, 0, 0, 0, '')
                    logger.info('正在往测试报告-测试概况插入计划执行概要记录')
                    test_reporter.insert_report_for_summary(data)

                flag = False
                remark = ''
                for record in records:
                    plan_id = self.plan_id
                    case_id, case_path, case_name = record
                    test_case = TestCase(execution_num, plan_id, case_id,
                                         case_path, case_name, self.protocol,
                                         self.host, self.port,
                                         self.global_headers)
                    logger.info(
                        '======================开始运行测试用例[名称:%s, ID:%s]======================'
                        % (case_name, case_id))
                    result = test_case.run(debug)
                    if not result[0]:
                        flag = True  # 有运行出错的用例
                        remark = '存在运行失败、被阻塞的用例'

                if not debug:
                    end_time = time.strftime('%Y-%m-%d %H:%M:%S',
                                             time.localtime())  # 结束运行时间
                    # 记录运行截止时间
                    timestamp_for_end = time.time()
                    logger.info('测试用例执行完毕,正在更新测试报告-测试概况表')
                    time_took = int(timestamp_for_end - timestamp_for_start)
                    days, hours, minutes, seconds = str(
                        time_took // 86400), str(
                            (time_took % 86400) // 3600), str(
                                ((time_took % 86400) % 3600) // 60), str(
                                    ((time_took % 86400) % 3600) % 60)
                    time_took = days + '天 ' + hours + '小时 ' + minutes + '分 ' + seconds + '秒'  # 运行耗时

                    case_pass_num = test_reporter.get_case_num_by_run_result(
                        execution_num, self.plan_id, '成功')  # 运行成功用例数
                    case_fail_num = test_reporter.get_case_num_by_run_result(
                        execution_num, self.plan_id, '失败')  # 运行失败用例数
                    case_block_num = test_reporter.get_case_num_by_run_result(
                        execution_num, self.plan_id, '阻塞')  # 运行被阻塞用例数
                    case_total_num = case_block_num + case_fail_num + case_pass_num

                    data = (end_time, time_took, case_total_num, case_pass_num,
                            case_fail_num, case_block_num, remark,
                            execution_num, self.plan_id)
                    test_reporter.update_report_for_summary(data)

                if flag:
                    return [False, '存在运行失败、被阻塞的用例']
                else:
                    return [True, '执行成功']
            elif result[0] and not result[1]:
                reason = '未查找到同测试计划关联的用例'
                logger.warn(reason)
                return [False, reason]
            else:
                reason = '查找同测试计划[名称:%s, ID:%s]关联的用例失败:%s' % (
                    self.plan_name, self.plan_id, result[1])
                logger.error(reason)
                return [False, reason]
        except Exception as e:
            logger.error('%s' % e)
            return [False, '%s' % e]
Ejemplo n.º 29
0
 def setUp(self):
     TestCase.setUp(self)
     self.hidden_hilite = gfm.HiddenHiliteExtension([])
Ejemplo n.º 30
0
 def __init__(self, name):
     self.wasRun = None
     TestCase.__init__(self, name)
Ejemplo n.º 31
0
    def run(self, debug):
        try:
            # 获取开始运行时间
            timestamp_for_start = time.time()
            start_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())

            logger.info('计划运行的浏览器有:%s' % (self.broswers))

            logger.info('正在查询测试计划关联的测试用例')
            query = 'SELECT node_id, node_path, node_name FROM `website_ui_case_tree_test_plan` WHERE plan_id = %s AND sub_node_num = 0 ORDER BY `order` ASC'
            data = (self.plan_id, )
            result = test_platform_db.select_many_record(query, data)
            if result[0] and result[1]:
                records = result[1]

                browser_list_for_failure = []
                mark = False  # 用于标记是否有运行用例出错的浏览器
                host_port = re.findall(
                    '[https|http]+://[^/]+', self.home_page
                )  # 获取http地址 形如 http://www.baidu.com, http://www.baidu.com:8080
                if host_port:
                    host_port = host_port[0]
                else:
                    return [False, '项目主页填写错误']
                for browser in self.broswers:
                    execution_num = str(int(time.time()))  # 执行编号

                    if not debug:
                        data = (execution_num, self.project_id, self.plan_id,
                                self.project_name, self.plan_name, browser,
                                start_time, '', '', 0, 0, 0, 0, '')
                        logger.info('正在往测试报告-测试概况插入计划执行概要记录')
                        test_reporter.insert_report_for_summary(data)

                    logger.info('正在获取浏览器驱动')
                    result = selenium_util.set_driver(browser)
                    if not result[0] and not debug:  # 设置驱动出错
                        logger.error('获取浏览器驱动出错,跳过该浏览器的执行')
                        data = ('', '', 0, 0, 0, 0, result[1], execution_num,
                                self.plan_id)
                        test_reporter.update_report_for_summary(data)
                        continue
                    try:
                        browser_driver = selenium_util.get_driver()
                        selenium_util.maximize_window()
                        logger.info('正在打开项目主页:%s' % self.home_page)
                        selenium_util.get(self.home_page)
                        selenium_util.implicitly_wait(20)

                        flag = False  # 用于标记是在每个浏览器下运行时,是否出现了运行出错的用例
                        logger.info(
                            '======================正在【%s】浏览器下运行测试用例======================'
                            % browser)
                        for record in records:
                            case_id, case_path, case_name = record
                            test_case = TestCase(execution_num, self.plan_id,
                                                 case_id, case_path, case_name,
                                                 host_port)
                            logger.info(
                                '======================开始运行测试用例[名称:%s, ID:%s]======================'
                                % (case_name, case_id))
                            result = test_case.run(debug)
                            if not result[0]:
                                flag = True  # 有运行出错的用例

                        if flag:
                            mark = True
                            remark = '部分用例运行出错'
                            browser_list_for_failure.append(browser)
                        else:
                            remark = ''
                    except Exception as e:
                        logger.error('运行出错:%s' % e)
                        remark = '%s' % e
                        mark = True
                    finally:
                        browser_driver.close()
                        browser_driver.quit()

                        if not debug:
                            end_time = time.strftime(
                                '%Y-%m-%d %H:%M:%S',
                                time.localtime())  # 结束运行时间

                            # 记录运行截止时间
                            timestamp_for_end = time.time()
                            time_took = int(timestamp_for_end -
                                            timestamp_for_start)
                            logger.info('测试用例执行完毕,正在更新测试报告-测试概况表')
                            days, hours, minutes, seconds = str(
                                time_took //
                                86400), str((time_took % 86400) // 3600), str(
                                    ((time_took % 86400) % 3600) // 60), str(
                                        ((time_took % 86400) % 3600) % 60)
                            time_took = days + '天 ' + hours + '小时 ' + minutes + '分 ' + seconds + '秒'  # 运行耗时

                            case_pass_num = test_reporter.get_case_num_by_run_result(
                                execution_num, self.plan_id, '成功')  # 运行成功用例数
                            case_fail_num = test_reporter.get_case_num_by_run_result(
                                execution_num, self.plan_id, '失败')  # 运行失败用例数
                            case_block_num = test_reporter.get_case_num_by_run_result(
                                execution_num, self.plan_id, '阻塞')  # 运行被阻塞用例数
                            case_total_num = case_block_num + case_fail_num + case_pass_num

                            data = (end_time, time_took, case_total_num,
                                    case_pass_num, case_fail_num,
                                    case_block_num, remark, execution_num,
                                    self.plan_id)
                            test_reporter.update_report_for_summary(data)
                if mark:
                    return [
                        False,
                        '测试计划在浏览器%s下运行失败' % str(browser_list_for_failure)
                    ]
                else:
                    return [True, '执行成功']
            elif result[0] and not result[1]:
                reason = '未查找到同测试计划关联的用例'
                logger.warn(reason)
                return [False, reason]
            else:
                reason = '查找同测试计划[名称:%s, ID:%s]关联的用例失败:%s' % (
                    self.plan_name, self.plan_id, result[1])
                logger.error(reason)
                return [False, reason]
        except Exception as e:
            logger.error('%s' % e)
            return [False, '%s' % e]
Ejemplo n.º 32
0
 def FromProto(proto):
   test_suite = TestSuite(str(proto.name), str(proto.description))
   for test_case_proto in proto.test_case:
     test_case = TestCase.FromProto(test_case_proto)
     test_suite.AddTestCase(test_case)
   return test_suite
Ejemplo n.º 33
0
def main():
    parser = argparse.ArgumentParser(
        description="PyTorch Object Detection Training")
    parser.add_argument(
        "--config-file",
        default="",
        metavar="FILE",
        help="path to config file",
        type=str,
    )
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument("--sync_bn", action="store_true")
    parser.add_argument(
        "--skip-test",
        dest="skip_test",
        help="Do not test the final model",
        action="store_true",
    )
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )

    args = parser.parse_args()

    num_gpus = int(
        os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
    args.distributed = num_gpus > 1

    assert num_gpus == 2, "unittest only for 2 gpus"

    if args.distributed:
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(backend="nccl",
                                             init_method="env://")

    device = torch.device('cuda')
    model = DistributedSyncBN(3).to(device)
    nn.init.constant_(model.weight, 1)

    if args.distributed:
        model = torch.nn.parallel.DistributedDataParallel(
            model,
            device_ids=[args.local_rank],
            output_device=args.local_rank,
            # this should be removed if we update BatchNorm stats
            broadcast_buffers=False,
        )

    # check train
    np.random.seed(args.local_rank)
    x = torch.from_numpy(np.random.rand(2, 3, 2, 2)).float()
    x.to(device)
    y = model(x)
    z = F.relu(y)
    z = z.sum()
    z.backward()

    np.random.seed(0)
    x1 = np.random.rand(2, 3, 2, 2)
    np.random.seed(1)
    x2 = np.random.rand(2, 3, 2, 2)
    x = np.concatenate((x1, x2), axis=0).astype(float)

    xv = x.reshape(4, 3, -1)
    mean = np.mean(np.mean(xv, axis=0, keepdims=True), axis=2, keepdims=True)
    a, b, c = xv.shape
    var = (np.var(np.transpose(xv, [0, 2, 1]).reshape((a * c, b)),
                  axis=0,
                  ddof=1).reshape((1, b, 1)))

    sd = np.sqrt(var + model.module.eps)

    y_expect = (xv - mean) / sd
    y_expect = y_expect.reshape(x.shape)

    test_case = TestCase()

    if dist.get_rank() == 0:
        test_case.assertTensorClose(y_expect[:2],
                                    y.detach().cpu().numpy(),
                                    max_err=5e-6)
    else:
        test_case.assertTensorClose(y_expect[2:],
                                    y.detach().cpu().numpy(),
                                    max_err=5e-6)
Ejemplo n.º 34
0
 def set_test_suite(self):
     for api_file in self.api_file_list:
         test_case = TestCase(api_file)
         self.test_suite.add_case(test_case)
Ejemplo n.º 35
0
def run_test(path, using_eva):
  try:
      test = TestCase(path)
      
      test.setup(using_eva)
      test.exercise()
      test.verify()

      if not test.has_passed or True:
          suffix = "-EVA" if using_eva else "-CIL"
          dirname = os.path.join(result_dir, test.name) + suffix
          os.mkdir(dirname)
          test.store_all_data(dirname)
    
      if using_eva: 
          results_eva[test.status] += 1
      else: 
          results_cil[test.status] +=1
      
      test.print_result(using_eva)

  except NoSpecification:
      print_err(f"Test case {test.name} missing specification")
Ejemplo n.º 36
0
 def setUp(self):
     TestCase.setUp(self)
     self.hidden_hilite = gfm.HiddenHiliteExtension([])
Ejemplo n.º 37
0
    def run(self, project_id, id):
        try:
            logger.info('正在查询项目[ID:%s]相关信息' % project_id)
            result = test_platform_db.select_one_record(
                'SELECT protocol, host, port, environment_id, valid_flag '
                'FROM `website_api_project_setting` WHERE id = %s',
                (project_id, ))
            if result[0] and result[1]:
                protocol, host, port, environment_id, valid_flag = result[1]

                logger.info('正在查询与项目关联的数据库信息')
                result = test_platform_db.select_many_record(
                    "SELECT db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd "
                    "FROM `website_database_setting` "
                    "WHERE  locate('API%s', project_id) != 0 AND environment_id= '%s'"
                    % (project_id, environment_id))
                if result[0] and result[1]:
                    for record in result[1]:
                        db_type, db_alias, db_name, db_host, db_port, db_user, db_passwd = record
                        if db_type == 'MySQL':
                            mydb = MyDB(db_name=db_name,
                                        db_host=db_host,
                                        port=db_port,
                                        user=db_user,
                                        password=db_passwd,
                                        charset='utf8')
                            db_related_to_project_dic[db_alias] = mydb
                        elif db_type == 'Redis':
                            if not db_passwd.strip():
                                db_passwd = None
                            if db_name.strip() == '':
                                db_name = '0'
                            myredis = RedisClient(host=db_host,
                                                  port=db_port,
                                                  password=db_passwd,
                                                  db=db_name,
                                                  charset='utf-8')
                            redis_related_to_project_dic[db_alias] = myredis

                elif not result[0]:
                    logger.error('查询项目相关的数据库配置信息出错:%s' % result[1])
                    return [False, result[1]]

                logger.info('正在查询与项目关联的全局变量')
                result = test_platform_db.select_many_record(
                    "SELECT `name`, `value` "
                    "FROM `website_global_variable_setting` "
                    "WHERE  project_type='API项目' AND locate('%s', project_id) != 0 AND locate('%s', env_id) != 0 "
                    % (project_id, environment_id))
                if result[0] and result[1]:
                    for record in result[1]:
                        name, value = record
                        name = name
                        global_variable_dic[name] = value
                elif not result[0]:
                    logger.error('查询项目相关的全局变量配置信息出错:%s' % result[1])
                    return [False, result[1]]

                try:
                    if 'global_headers' in global_variable_dic.keys():
                        global_headers = global_variable_dic['global_headers']
                        # 防止用户输入了中文冒号,替换为英文冒号,不然经过global_headers.encode("utf-8").decode("latin1")这样编码转换,
                        # 会把"key":中的中文冒号解码为非英文冒号,导致执行json loads函数时会报错;
                        # 另外,请求头从数据库读取,可能涉及到换行符,需要去掉
                        global_headers = global_headers.replace(':',
                                                                ':').replace(
                                                                    '\t', '')
                        global_headers = json.loads(
                            global_headers, object_pairs_hook=OrderedDict)
                    else:
                        global_headers = {}
                except Exception as e:
                    logger.error('%s' % e)
                    return [False, '%s' % e]

                logger.info('正在查询输入ID标识的用例(套件)相关信息')
                query = 'SELECT id, text FROM `website_api_case_tree` WHERE project_id = %s AND id = %s' % (
                    project_id, id)
                result = test_platform_db.select_one_record(query)

                if result[0] and result[1]:
                    record = result[1]
                    case_id, case_name = record
                    execution_num = str(int(time.time()))  # 执行编号

                    query = 'SELECT id, text FROM `website_api_case_tree` WHERE project_id = %s AND parent_id = %s  ' \
                            'AND id NOT IN (SELECT parent_id FROM `website_api_case_tree` WHERE project_id=%s)' \
                            'ORDER BY `order` ASC' % (project_id, id, project_id)
                    result = test_platform_db.select_many_record(query)
                    if result[0] and result[1]:
                        logger.info('输入ID标识的是测试套件')
                        records = result[1]
                        for record in records:
                            case_id, case_name = record
                            test_case = TestCase(execution_num, 0, case_id,
                                                 '--', case_name, protocol,
                                                 host, port, global_headers)
                            logger.info(
                                '======================开始运行测试用例[名称:%s, ID:%s]======================'
                                % (case_name, case_id))
                            result = test_case.run(True)
                            if not result[0]:
                                return [
                                    False,
                                    '用例(ID:%s 名称:%s)运行出错:%s' %
                                    (case_id, case_name, result[2])
                                ]

                    elif result[0] and not result[1]:
                        logger.info('输入ID标识的是测试用例,开始执行用例')
                        test_case = TestCase(execution_num, 0, case_id, '--',
                                             case_name, protocol, host, port,
                                             global_headers)
                        logger.info(
                            '======================开始运行测试用例[名称:%s, ID:%s]======================'
                            % (case_name, case_id))
                        result = test_case.run(True)
                        if not result[0]:
                            return [
                                False,
                                '用例(ID:%s 名称:%s)运行出错:%s' %
                                (case_id, case_name, result[2])
                            ]
                    else:
                        logger.error('查询出错:%s' % result[1])
                        return [False, result[1]]
                elif result[0] and not result[1]:
                    reason = '未查找到相关信息,请检查配置的项目ID(%s),用例(套件)标识ID(%s)是否正确'
                    logger.warn(reason)
                    return [False, reason]
                else:
                    logger.error('查找相关信息失败:%s' % result[1])
                    return [False, '查找相关信息失败:%s' % result[1]]

            elif result[0] and not result[1]:
                logger.error('未查询到项目相关的信息')
                return [False, '未查询到项目相关的信息']
            else:
                logger.error('查询项目相关信息失败:%s' % result[1])
                return [False, '查询项目相关信息失败:%s' % result[1]]
        except Exception as e:
            logger.error('%s' % e)
            return [False, '%s' % e]
        finally:
            logger.info('正在释放资源')
            logger.info('正在断开与项目关联的数据库连接')
            # 关闭数据库
            for key, db in db_related_to_project_dic.copy().items():
                db.close()
                del db_related_to_project_dic[key]

            logger.info('正在清理与项目关联的全局变量')
            global_variable_dic.clear()