Esempio n. 1
0
def cli(args=sys.argv[1:]):
    worker_map = {
        'rq': rq_worker,
        'sqs': busy_wait_worker,
        'mongo': burst_worker,
    }

    parser = argparse.ArgumentParser()
    parser.add_argument('queue',
                        choices=worker_map.keys(),
                        help='The work queue the workers should grab jobs from.')
    parser.add_argument('-j',
                        dest='num_workers',
                        type=int,
                        default=1,
                        help='The number of worker processes to spawn.')
    # setup logging args
    commandline.log_formatters = { k: v for k, v in commandline.log_formatters.iteritems() if k in ('raw', 'mach') }
    commandline.add_logging_group(parser)

    args = vars(parser.parse_args(args))
    commandline.setup_logging("catalog-worker", args)

    qname = args['queue']
    process_class = Process
    if qname == 'rq':
        process_class = Thread

    for _ in range(args['num_workers']):
        worker = process_class(target=worker_map[qname], args=(qname,))
        worker.start()
Esempio n. 2
0
def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from marionette.runtests import (
        MarionetteTestRunner,
        MarionetteHarness
    )

    parser = setup_marionette_argument_parser()

    if not tests:
        tests = [os.path.join(topsrcdir,
                 'testing/marionette/harness/marionette/tests/unit-tests.ini')]

    args = argparse.Namespace(tests=tests)

    args.binary = binary

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests",
                                            args,
                                            {"mach": sys.stdout})
    failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 3
0
def run_marionette(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from marionette.runtests import (
        MarionetteTestRunner,
        BaseMarionetteArguments,
        MarionetteHarness
    )

    parser = BaseMarionetteArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [os.path.join(topsrcdir,
                 'testing/marionette/harness/marionette/tests/unit-tests.ini')]

    args = parser.parse_args(args=tests)

    args.binary = binary
    path, exe = os.path.split(args.binary)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests",
                                            args,
                                            {"mach": sys.stdout})
    failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 4
0
def run_external_media_test(tests, testtype=None, topsrcdir=None, **kwargs):
    from external_media_harness.runtests import (
        FirefoxMediaHarness,
        MediaTestArguments,
        MediaTestRunner,
        mn_cli,
    )

    from mozlog.structured import commandline

    parser = MediaTestArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [os.path.join(topsrcdir,
                 'dom/media/test/external/external_media_tests/manifest.ini')]

    args = parser.parse_args(args=tests)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Firefox External Media Tests",
                                            args,
                                            {"mach": sys.stdout})
    failed = mn_cli(MediaTestRunner, MediaTestArguments, FirefoxMediaHarness,
                    args=vars(args))

    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 5
0
 def __init__(self, options):
     self.options = options
     self.logger = commandline.setup_logging("mozregression",
                                             self.options,
                                             {"mach": sys.stdout})
     self.action = None
     self.fetch_config = None
Esempio n. 6
0
def main():
    parser = argparse.ArgumentParser()

    dsn = config.get('database')

    parser.add_argument('--dsn', default=dsn,
                        help="Postgresql DSN connection string")
    commandline.add_logging_group(parser)
    args = parser.parse_args()

    logging.basicConfig()
    logger = commandline.setup_logging('autoland', vars(args), {})
    logger.info('starting autoland')

    dbconn = get_dbconn(args.dsn)
    last_error_msg = None
    while True:
        try:
            handle_pending_mozreview_pullrequests(logger, dbconn)
            handle_pending_transplants(logger, dbconn)
            handle_pending_github_updates(logger, dbconn)
            handle_pending_mozreview_updates(logger, dbconn)
            time.sleep(0.25)
        except KeyboardInterrupt:
            break
        except psycopg2.InterfaceError:
            dbconn = get_dbconn(args.dsn)
        except:
            # If things go really badly, we might see the same exception
            # thousands of times in a row. There's not really any point in
            # logging it more than once.
            error_msg = traceback.format_exc()
            if error_msg != last_error_msg:
                logger.error(error_msg)
                last_error_msg = error_msg
Esempio n. 7
0
    def run_cppunit_test(self, **params):
        import mozinfo
        from mozlog.structured import commandline
        import runcppunittests as cppunittests

        log = commandline.setup_logging("cppunittest", {},
                                        {"tbpl": sys.stdout})

        if len(params['test_files']) == 0:
            testdir = os.path.join(self.distdir, 'cppunittests')
            tests = cppunittests.extract_unittests_from_args([testdir],
                                                             mozinfo.info)
        else:
            tests = cppunittests.extract_unittests_from_args(
                params['test_files'], mozinfo.info)

        # See if we have crash symbols
        symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
        if not os.path.isdir(symbols_path):
            symbols_path = None

        tester = cppunittests.CPPUnitTests()
        try:
            result = tester.run_tests(tests,
                                      self.bindir,
                                      symbols_path,
                                      interactive=True)
        except Exception as e:
            log.error("Caught exception running cpp unit tests: %s" % str(e))
            result = False

        return 0 if result else 1
Esempio n. 8
0
def run_firefox_ui_test(tests, testtype=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline
    from firefox_ui_harness import cli_functional
    from firefox_ui_harness.arguments import FirefoxUIArguments

    parser = FirefoxUIArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [
            os.path.join(
                topsrcdir,
                'testing/firefox-ui/tests/firefox_ui_tests/manifest.ini')
        ]

    args = parser.parse_args(args=tests)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Firefox UI - Functional Tests",
                                            args, {"mach": sys.stdout})
    failed = cli_functional.mn_cli(cli_functional.FirefoxUITestRunner,
                                   args=args)
    if failed > 0:
        return 1
    else:
        return 0
def run_marionette(context, **kwargs):
    from marionette.runtests import (
        MarionetteTestRunner,
        MarionetteHarness
    )
    from mozlog.structured import commandline


    args = argparse.Namespace(**kwargs)

    if not args.binary:
        args.binary = context.find_firefox()

    test_root = os.path.join(context.package_root, 'marionette', 'tests')
    if not args.tests:
        args.tests = [os.path.join(test_root, 'testing', 'marionette', 'harness',
                                   'marionette', 'tests', 'unit-tests.ini')]

    normalize = partial(context.normalize_test_path, test_root)
    args.tests = map(normalize, args.tests)

    commandline.add_logging_group(parser)
    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests",
                                            args,
                                            {"mach": sys.stdout})
    status = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    return 1 if status else 0
Esempio n. 10
0
def run_marionette(context, **kwargs):
    from marionette.runtests import MarionetteTestRunner, MarionetteHarness
    from mozlog.structured import commandline

    args = argparse.Namespace(**kwargs)
    args.binary = args.binary or context.firefox_bin

    test_root = os.path.join(context.package_root, "marionette", "tests")
    if not args.tests:
        args.tests = [
            os.path.join(
                test_root,
                "testing",
                "marionette",
                "harness",
                "marionette_harness",
                "tests",
                "unit-tests.ini",
            )
        ]

    normalize = partial(context.normalize_test_path, test_root)
    args.tests = list(map(normalize, args.tests))

    commandline.add_logging_group(parser)
    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests", args,
                                            {"mach": sys.stdout})
    status = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    return 1 if status else 0
Esempio n. 11
0
def run_session(tests, testtype=None, address=None, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from marionette.runtests import MarionetteHarness

    from session.runtests import SessionTestRunner, BaseSessionArguments, SessionArguments, SessionTestCase

    parser = BaseSessionArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [os.path.join(topsrcdir, "testing/marionette/harness/session/tests/unit-tests.ini")]

    args = parser.parse_args(args=tests)

    args.binary = binary

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Session Unit Tests", args, {"mach": sys.stdout})
    failed = MarionetteHarness(
        runner_class=SessionTestRunner, parser_class=SessionArguments, testcase_class=SessionTestCase, args=vars(args)
    ).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 12
0
 def test_setup_logging_optparse(self):
     parser = optparse.OptionParser()
     commandline.add_logging_group(parser)
     args, _ = parser.parse_args(["--log-raw=-"])
     logger = commandline.setup_logging("test_optparse", args, {})
     self.assertEqual(len(logger.handlers), 1)
     self.assertIsInstance(logger.handlers[0], handlers.StreamHandler)
Esempio n. 13
0
def main(argv):
    parser = get_parser()
    args = parser.parse_args(argv[1:])
    logger = commandline.setup_logging("webapi", args, {"mach": sys.stdout})

    test_loader = TestLoader()
    tests = []
    if len(args.tests) >= 1 and args.tests[0] == "discover":
        start_dir = args.tests[1] if len(args.tests) > 1 else "."
        tests = test_loader.discover(start_dir, args.pattern or "test_*.py")
    else:
        tests = None
        if len(args.tests) > 0:
            tests = test_loader.loadTestsFromNames(args.tests, None)
        else:
            tests = unittest.TestSuite()

    results = run(
        tests,
        spawn_browser=not args.no_browser,
        verbosity=2 if args.verbose else 1,
        failfast=args.failfast,
        catch_break=args.catch,
        buffer=args.buffer,
        logger=logger,
    )
    sys.exit(not results.wasSuccessful())
Esempio n. 14
0
 def test_setup_logging_optparse(self):
     parser = optparse.OptionParser()
     commandline.add_logging_group(parser)
     args, _ = parser.parse_args(["--log-raw=-"])
     logger = commandline.setup_logging("test_optparse", args, {})
     self.assertEqual(len(logger.handlers), 1)
     self.assertIsInstance(logger.handlers[0], handlers.StreamHandler)
Esempio n. 15
0
def run_external_media_test(tests, testtype=None, topsrcdir=None, **kwargs):
    from external_media_harness.runtests import (
        FirefoxMediaHarness,
        MediaTestArguments,
        MediaTestRunner,
        mn_cli,
    )

    from mozlog.structured import commandline

    parser = MediaTestArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [os.path.join(topsrcdir,
                 'dom/media/test/external/external_media_tests/manifest.ini')]

    args = parser.parse_args(args=tests)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Firefox External Media Tests",
                                            args,
                                            {"mach": sys.stdout})
    failed = mn_cli(MediaTestRunner, MediaTestArguments, FirefoxMediaHarness,
                    args=args)

    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 16
0
    def __init__(self, options):
        self.options = options
        self.logger = commandline.setup_logging("mozregression",
                                                self.options,
                                                {"mach": sys.stdout})
        # allow to filter process output based on the user option
        if options.process_output is None:
            # process_output not user defined
            log_process_output = options.build_type != 'opt'
        else:
            log_process_output = options.process_output == 'stdout'
        get_default_logger("process").component_filter = \
            lambda data: data if log_process_output else None

        # filter some mozversion log lines
        re_ignore_mozversion_line = re.compile(
            r"^(platform_.+|application_vendor|application_remotingname"
            r"|application_id|application_display_name): .+"
        )
        get_default_logger("mozversion").component_filter = lambda data: (
            None if re_ignore_mozversion_line.match(data['message']) else data
        )

        self.action = None
        self.fetch_config = None
Esempio n. 17
0
def main():
    parser = argparse.ArgumentParser()
    dsn = 'dbname=autoland user=autoland host=localhost password=autoland'
    parser.add_argument('--dsn', default=dsn,
                        help="Postgresql DSN connection string")
    commandline.add_logging_group(parser)
    args = parser.parse_args()

    logging.basicConfig()
    logger = commandline.setup_logging('autoland', vars(args), {})
    logger.info('starting autoland')

    auth = selfserve.read_credentials()
    if auth is None:
        logger.critical('could not read selfserve credentials. aborting')
        return

    dbconn = psycopg2.connect(args.dsn)

    # this should exceed the stable delay in buildapi
    stable_delay = datetime.timedelta(minutes=5)
    old_job = datetime.timedelta(minutes=30)

    while True:
        try:
            cursor = dbconn.cursor()
            now = datetime.datetime.now()

            # handle potentially finished autoland jobs
            query = """select tree,revision from Autoland
                       where pending=0 and running=0 and last_updated<=%(time)s
                       and can_be_landed is null"""
            cursor.execute(query, ({'time': now - stable_delay}))
            for row in cursor.fetchall():
                tree, rev = row
                handle_autoland_request(logger, auth, dbconn, tree, rev)

            # we also look for any older jobs - maybe something got missed
            # in pulse
            query = """select tree,revision from Autoland
                       where last_updated<=%(time)s
                       and can_be_landed is null"""
            cursor.execute(query, ({'time': now - old_job}))
            for row in cursor.fetchall():
                tree, rev = row
                handle_autoland_request(logger, auth, dbconn, tree, rev)

            #
            handle_pending_bugzilla_comments(logger, dbconn)

            #
            handle_pending_transplants(logger, dbconn)

            time.sleep(30)

        except KeyboardInterrupt:
            break
        except:
            t, v, tb = sys.exc_info()
            logger.error('\n'.join(traceback.format_exception(t, v, tb)))
Esempio n. 18
0
def cli(args=sys.argv[1:]):
    worker_map = {
        'sqs': busy_wait_worker,
        'mongo': burst_worker,
    }

    parser = argparse.ArgumentParser()
    parser.add_argument(
        'queue',
        choices=worker_map.keys(),
        help='The work queue the worker should grab jobs from.')

    # setup logging args
    commandline.log_formatters = {
        k: v
        for k, v in commandline.log_formatters.iteritems()
        if k in ('raw', 'mach')
    }
    commandline.add_logging_group(parser)

    args = vars(parser.parse_args(args))
    global logger
    logger = commandline.setup_logging("catalog-worker", args)

    qname = args['queue']
    logger.info("Starting a '{}' worker".format(qname))
    logger.info("Using '{}' as the storage backend".format(
        settings['datastore']))
    q = all_queues[qname]()
    return worker_map[qname](q)
Esempio n. 19
0
def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from marionette_harness.runtests import (MarionetteTestRunner,
                                             MarionetteHarness)

    parser = create_parser_tests()

    if not tests:
        tests = [
            os.path.join(
                topsrcdir,
                "testing/marionette/harness/marionette_harness/tests/unit-tests.ini"
            )
        ]

    args = argparse.Namespace(tests=tests)

    args.binary = binary

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests", args,
                                            {"mach": sys.stdout})
    failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 20
0
def main():
    parser = B2GOptions()
    structured.commandline.add_logging_group(parser)
    options, args = parser.parse_args()
    log = commandline.setup_logging("Remote XPCShell", options,
                                    {"tbpl": sys.stdout})
    run_remote_xpcshell(parser, options, args, log)
Esempio n. 21
0
def main(argv):
    parser = get_parser()
    args = parser.parse_args(argv[1:])
    logger = commandline.setup_logging("webapi", args, {"mach": sys.stdout})

    test_loader = TestLoader()
    tests = []
    if len(args.tests) >= 1 and args.tests[0] == "discover":
        start_dir = args.tests[1] if len(args.tests) > 1 else "."
        tests = test_loader.discover(start_dir, args.pattern or "test_*.py")
    else:
        tests = None
        if len(args.tests) > 0:
            tests = test_loader.loadTestsFromNames(args.tests, None)
        else:
            tests = unittest.TestSuite()

    results = run(tests,
                  spawn_browser=not args.no_browser,
                  verbosity=2 if args.verbose else 1,
                  failfast=args.failfast,
                  catch_break=args.catch,
                  buffer=args.buffer,
                  logger=logger)
    sys.exit(not results.wasSuccessful())
Esempio n. 22
0
def run_marionette(context, **kwargs):
    from marionette.runtests import (MarionetteTestRunner, MarionetteHarness)
    from mozlog.structured import commandline

    args = argparse.Namespace(**kwargs)
    args.binary = args.binary or context.firefox_bin
    args.e10s = context.mozharness_config.get('e10s', args.e10s)

    test_root = os.path.join(context.package_root, 'marionette', 'tests')
    if not args.tests:
        args.tests = [
            os.path.join(test_root, 'testing', 'marionette', 'harness',
                         'marionette', 'tests', 'unit-tests.ini')
        ]

    normalize = partial(context.normalize_test_path, test_root)
    args.tests = map(normalize, args.tests)

    commandline.add_logging_group(parser)
    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Marionette Unit Tests", args,
                                            {"mach": sys.stdout})
    status = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    return 1 if status else 0
Esempio n. 23
0
def run_marionette(tests, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from marionette_harness.runtests import MarionetteTestRunner, MarionetteHarness

    parser = create_parser_tests()

    args = argparse.Namespace(tests=tests)

    args.binary = binary
    args.logger = kwargs.pop("log", None)

    for k, v in iteritems(kwargs):
        setattr(args, k, v)

    parser.verify_usage(args)

    if not args.logger:
        args.logger = commandline.setup_logging("Marionette Unit Tests", args,
                                                {"mach": sys.stdout})
    failed = MarionetteHarness(MarionetteTestRunner, args=vars(args)).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 24
0
def run_telemetry(tests, binary=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline

    from telemetry_harness.runtests import TelemetryTestRunner

    from marionette_harness.runtests import MarionetteHarness

    parser = create_parser_tests()

    if not tests:
        tests = [
            os.path.join(
                topsrcdir,
                "toolkit/components/telemetry/tests/marionette/tests/manifest.ini",
            )
        ]

    args = argparse.Namespace(tests=tests)

    args.binary = binary
    args.logger = kwargs.pop("log", None)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    if not args.logger:
        args.logger = commandline.setup_logging("Telemetry Client Tests", args,
                                                {"mach": sys.stdout})
    failed = MarionetteHarness(TelemetryTestRunner, args=vars(args)).run()
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 25
0
    def run_cppunit_test(self, **params):
        import mozinfo
        from mozlog.structured import commandline
        import runcppunittests as cppunittests

        log = commandline.setup_logging("cppunittest",
                                        {},
                                        {"tbpl": sys.stdout})

        if len(params['test_files']) == 0:
            testdir = os.path.join(self.distdir, 'cppunittests')
            tests = cppunittests.extract_unittests_from_args([testdir], mozinfo.info)
        else:
            tests = cppunittests.extract_unittests_from_args(params['test_files'], mozinfo.info)

        # See if we have crash symbols
        symbols_path = os.path.join(self.distdir, 'crashreporter-symbols')
        if not os.path.isdir(symbols_path):
            symbols_path = None

        tester = cppunittests.CPPUnitTests()
        try:
            result = tester.run_tests(tests, self.bindir, symbols_path, interactive=True)
        except Exception as e:
            log.error("Caught exception running cpp unit tests: %s" % str(e))
            result = False

        return 0 if result else 1
Esempio n. 26
0
def run_firefox_ui_test(tests, testtype=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline
    from firefox_ui_harness import cli_functional
    from firefox_ui_harness.arguments import FirefoxUIArguments

    parser = FirefoxUIArguments()
    commandline.add_logging_group(parser)

    if not tests:
        tests = [os.path.join(topsrcdir,
                 'testing/firefox-ui/tests/firefox_ui_tests/manifest.ini')]

    args = parser.parse_args(args=tests)

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    args.logger = commandline.setup_logging("Firefox UI - Functional Tests",
                                            args,
                                            {"mach": sys.stdout})
    failed = cli_functional.mn_cli(cli_functional.FirefoxUITestRunner, args=args)
    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 27
0
def run_firefox_ui_test(testtype=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline
    from argparse import Namespace
    import firefox_ui_harness

    if testtype == 'functional':
        parser = setup_argument_parser_functional()
    else:
        parser = setup_argument_parser_update()

    test_types = {
        'functional': {
            'default_tests': [
                os.path.join('puppeteer', 'manifest.ini'),
                os.path.join('functional', 'manifest.ini'),
            ],
            'cli_module': firefox_ui_harness.cli_functional,
        },
        'update': {
            'default_tests': [
                os.path.join('update', 'manifest.ini'),
            ],
            'cli_module': firefox_ui_harness.cli_update,
        }
    }

    fxui_dir = os.path.join(topsrcdir, 'testing', 'firefox-ui')

    # Set the resources path which is used to serve test data via wptserve
    if not kwargs['server_root']:
        kwargs['server_root'] = os.path.join(fxui_dir, 'resources')

    # If called via "mach test" a dictionary of tests is passed in
    if 'test_objects' in kwargs:
        tests = []
        for obj in kwargs['test_objects']:
            tests.append(obj['file_relpath'])
        kwargs['tests'] = tests
    elif not kwargs.get('tests'):
        # If no tests have been selected, set default ones
        kwargs['tests'] = [os.path.join(fxui_dir, 'tests', test)
                           for test in test_types[testtype]['default_tests']]

    kwargs['logger'] = commandline.setup_logging('Firefox UI - {} Tests'.format(testtype),
                                                 {"mach": sys.stdout})

    args = Namespace()

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    failed = test_types[testtype]['cli_module'].cli(args=vars(args))

    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 28
0
def main():
    parser = B2GOptions()
    structured.commandline.add_logging_group(parser)
    options, args = parser.parse_args()
    log = commandline.setup_logging("Remote XPCShell",
                                    options,
                                    {"tbpl": sys.stdout})
    run_remote_xpcshell(parser, options, args, log)
Esempio n. 29
0
def cli():
    global webapi_results
    global webapi_results_embed_app

    parser = argparse.ArgumentParser()
    parser.add_argument("--version",
                        help="version of FxOS under test",
                        default="1.3",
                        action="store")
    parser.add_argument("--debug",
                        help="enable debug logging",
                        action="store_true")
    parser.add_argument("--list-test-groups",
                        help="print test groups available to run",
                        action="store_true")
    parser.add_argument("--include",
                        metavar="TEST-GROUP",
                        help="include this test group",
                        action="append")
    parser.add_argument("--result-file",
                        help="absolute file path to store the resulting json." \
                             "Defaults to results.json on your current path",
                        action="store")
    parser.add_argument("--generate-reference",
                        help="Generate expected result files",
                        action="store_true")
    commandline.add_logging_group(parser)

    args = parser.parse_args()

    test_groups = [
        'omni-analyzer',
        'permissions',
        'webapi',
        ]
    if args.list_test_groups:
        for t in test_groups:
            print t
        return 0

    test_groups = set(args.include if args.include else test_groups)
    report = {'buildprops': {}}

    logging.basicConfig()
    if not args.debug:
        logging.disable(logging.ERROR)

    logger = commandline.setup_logging("certsuite", vars(args), {})

    # Step 1: Get device information
    try:
        dm = mozdevice.DeviceManagerADB()
    except mozdevice.DMError, e:
        print "Error connecting to device via adb (error: %s). Please be " \
            "sure device is connected and 'remote debugging' is enabled." % \
            e.msg
        logger.error("Error connecting to device: %s" % e.msg)
        sys.exit(1)
Esempio n. 30
0
def main():
    parser = argparse.ArgumentParser(
        description="Runner for guided Web API tests.")
    parser.add_argument("-l",
                        "--list-test-groups",
                        action="store_true",
                        help="List all logical test groups")
    parser.add_argument("-a",
                        "--list-all-tests",
                        action="store_true",
                        help="List all tests")
    parser.add_argument(
        "-i",
        "--include",
        metavar="GROUP",
        action="append",
        default=[],
        help="Only include specified group(s) in run, include several "
        "groups by repeating flag")
    parser.add_argument(
        "-n",
        "--no-browser",
        action="store_true",
        help="Don't start a browser but wait for manual connection")
    parser.add_argument("-v",
                        dest="verbose",
                        action="store_true",
                        help="Verbose output")
    commandline.add_logging_group(parser)
    args = parser.parse_args(sys.argv[1:])
    logger = commandline.setup_logging("webapi", vars(args),
                                       {"raw": sys.stdout})
    if args.list_test_groups and len(args.include) > 0:
        print >> sys.stderr("%s: error: cannot list and include test "
                            "groups at the same time" % sys.argv[0])
        parser.print_usage()
        sys.exit(1)

    testgen = iter_tests(os.path.dirname(__file__))
    if args.list_test_groups:
        for group, _ in testgen:
            print(group)
        return 0
    elif args.list_all_tests:
        for group, tests in testgen:
            for test in tests:
                print("%s.%s" % (group, test))
        return 0

    test_loader = semiauto.TestLoader()
    tests = test_loader.loadTestsFromNames(
        map(lambda t: "webapi_tests.%s" % t, args.include
            or [g for g, _ in testgen]), None)
    results = semiauto.run(tests,
                           logger=logger,
                           spawn_browser=not args.no_browser,
                           verbosity=2 if args.verbose else 1)
    return 0 if results.wasSuccessful() else 1
Esempio n. 31
0
def run_marionette(tests,
                   b2g_path=None,
                   emulator=None,
                   testtype=None,
                   address=None,
                   binary=None,
                   topsrcdir=None,
                   **kwargs):

    # Import the harness directly and under a different name here to avoid
    # "marionette" being importable from two locations when "testing/marionette/client"
    # is on sys.path.
    # See bug 1050511 and bug 1114474. This needs to be removed with the
    # resolution of bug 1109183.
    clientdir = os.path.join(topsrcdir, 'testing/marionette/client')
    if clientdir in sys.path:
        sys.path.remove(clientdir)
    path = os.path.join(topsrcdir,
                        'testing/marionette/client/marionette/runtests.py')
    with open(path, 'r') as fh:
        imp.load_module('marionetteharness', fh, path,
                        ('.py', 'r', imp.PY_SOURCE))

    from marionetteharness import (MarionetteTestRunner, BaseMarionetteOptions,
                                   startTestRunner)

    parser = BaseMarionetteOptions()
    commandline.add_logging_group(parser)
    options, args = parser.parse_args()

    if not tests:
        tests = [
            os.path.join(
                topsrcdir,
                'testing/marionette/client/marionette/tests/unit-tests.ini')
        ]

    if b2g_path:
        options.homedir = b2g_path
        if emulator:
            options.emulator = emulator
    else:
        options.binary = binary
        path, exe = os.path.split(options.binary)

    for k, v in kwargs.iteritems():
        setattr(options, k, v)

    parser.verify_usage(options, tests)

    options.logger = commandline.setup_logging("Marionette Unit Tests",
                                               options, {"mach": sys.stdout})

    runner = startTestRunner(MarionetteTestRunner, options, tests)
    if runner.failed > 0:
        return 1

    return 0
Esempio n. 32
0
def setup(args, defaults):
    logger = commandline.setup_logging("web-platform-tests", args, defaults)
    setup_stdlib_logger()

    for name in args.keys():
        if name.startswith("log_"):
            args.pop(name)

    return logger
Esempio n. 33
0
def cli(argv=None):
    """
    main entry point of mozregression command line.
    """
    options = parse_args(argv)
    logger = commandline.setup_logging("mozregression",
                                       options,
                                       {"mach": sys.stdout})
    check_mozregression_version(logger)

    if options.list_releases:
        print(formatted_valid_release_dates())
        sys.exit()

    cache_session = limitedfilecache.get_cache(
        options.http_cache_dir, limitedfilecache.ONE_GIGABYTE,
        logger=get_default_logger('Limited File Cache'))
    set_http_cache_session(cache_session,
                           get_defaults={"timeout": options.http_timeout})

    fetch_config = create_config(options.app, mozinfo.os, options.bits)

    if options.command is None:
        launcher_kwargs = dict(
            addons=options.addons,
            profile=options.profile,
            cmdargs=options.cmdargs,
            preferences=preference(options.prefs_files, options.prefs),
        )
        test_runner = ManualTestRunner(launcher_kwargs=launcher_kwargs)
    else:
        test_runner = CommandTestRunner(options.command)

    runner = ResumeInfoBisectRunner(fetch_config, test_runner, options)

    if fetch_config.is_inbound():
        # this can be useful for both inbound and nightly, because we
        # can go to inbound from nightly.
        fetch_config.set_inbound_branch(options.inbound_branch)

    # bisect inbound if last good revision or first bad revision are set
    if options.first_bad_revision or options.last_good_revision:
        bisect = bisect_inbound
    else:
        bisect = bisect_nightlies

    try:
        launcher_class = APP_REGISTRY.get(fetch_config.app_name)
        launcher_class.check_is_runnable()

        sys.exit(bisect(runner, logger))
    except KeyboardInterrupt:
        sys.exit("\nInterrupted.")
    except UnavailableRelease as exc:
        sys.exit("%s\n%s" % (exc, formatted_valid_release_dates()))
    except (MozRegressionError, RequestException) as exc:
        sys.exit(str(exc))
Esempio n. 34
0
def setup_logging(args, defaults):
    global logger
    logger = commandline.setup_logging("web-platform-tests-update", args, defaults)

    for name in args.keys():
        if name.startswith("log_"):
            args.pop(name)

    return logger
Esempio n. 35
0
def cli(argv=None):
    """
    main entry point of mozregression command line.
    """
    options = parse_args(argv)
    logger = commandline.setup_logging("mozregression", options,
                                       {"mach": sys.stdout})
    check_mozregression_version(logger)

    if options.list_releases:
        print(formatted_valid_release_dates())
        sys.exit()

    cache_session = limitedfilecache.get_cache(
        options.http_cache_dir,
        limitedfilecache.ONE_GIGABYTE,
        logger=get_default_logger('Limited File Cache'))
    set_http_cache_session(cache_session,
                           get_defaults={"timeout": options.http_timeout})

    fetch_config = create_config(options.app, mozinfo.os, options.bits)

    if options.command is None:
        launcher_kwargs = dict(
            addons=options.addons,
            profile=options.profile,
            cmdargs=options.cmdargs,
            preferences=preference(options.prefs_files, options.prefs),
        )
        test_runner = ManualTestRunner(launcher_kwargs=launcher_kwargs)
    else:
        test_runner = CommandTestRunner(options.command)

    runner = ResumeInfoBisectRunner(fetch_config, test_runner, options)

    if fetch_config.is_inbound():
        # this can be useful for both inbound and nightly, because we
        # can go to inbound from nightly.
        fetch_config.set_inbound_branch(options.inbound_branch)

    # bisect inbound if last good revision or first bad revision are set
    if options.first_bad_revision or options.last_good_revision:
        bisect = bisect_inbound
    else:
        bisect = bisect_nightlies

    try:
        launcher_class = APP_REGISTRY.get(fetch_config.app_name)
        launcher_class.check_is_runnable()

        sys.exit(bisect(runner, logger))
    except KeyboardInterrupt:
        sys.exit("\nInterrupted.")
    except UnavailableRelease as exc:
        sys.exit("%s\n%s" % (exc, formatted_valid_release_dates()))
    except (MozRegressionError, RequestException) as exc:
        sys.exit(str(exc))
Esempio n. 36
0
def setup(args, defaults):
    logger = commandline.setup_logging("web-platform-tests", args, defaults)
    setup_stdlib_logger()

    for name in args.keys():
        if name.startswith("log_"):
            args.pop(name)

    return logger
Esempio n. 37
0
def main():

    if sys.version_info < (2,7):
        print >>sys.stderr, "Error: You must use python version 2.7 or newer but less than 3.0"
        sys.exit(1)

    parser = RemoteXPCShellOptions()
    structured.commandline.add_logging_group(parser)
    options, args = parser.parse_args()
    if not options.localAPK:
        for file in os.listdir(os.path.join(options.objdir, "dist")):
            if (file.endswith(".apk") and file.startswith("fennec")):
                options.localAPK = os.path.join(options.objdir, "dist")
                options.localAPK = os.path.join(options.localAPK, file)
                print >>sys.stderr, "using APK: " + options.localAPK
                break
        else:
            print >>sys.stderr, "Error: please specify an APK"
            sys.exit(1)

    options = parser.verifyRemoteOptions(options)
    log = commandline.setup_logging("Remote XPCShell",
                                    options,
                                    {"tbpl": sys.stdout})

    if len(args) < 1 and options.manifest is None:
        print >>sys.stderr, """Usage: %s <test dirs>
             or: %s --manifest=test.manifest """ % (sys.argv[0], sys.argv[0])
        sys.exit(1)

    if options.dm_trans == "adb":
        if options.deviceIP:
            dm = mozdevice.DroidADB(options.deviceIP, options.devicePort, packageName=None, deviceRoot=options.remoteTestRoot)
        else:
            dm = mozdevice.DroidADB(packageName=None, deviceRoot=options.remoteTestRoot)
    else:
        if not options.deviceIP:
            print "Error: you must provide a device IP to connect to via the --device option"
            sys.exit(1)
        dm = mozdevice.DroidSUT(options.deviceIP, options.devicePort, deviceRoot=options.remoteTestRoot)

    if options.interactive and not options.testPath:
        print >>sys.stderr, "Error: You must specify a test filename in interactive mode!"
        sys.exit(1)

    xpcsh = XPCShellRemote(dm, options, args, log)

    # we don't run concurrent tests on mobile
    options.sequential = True

    if not xpcsh.runTests(xpcshell='xpcshell',
                          testClass=RemoteXPCShellTestThread,
                          testdirs=args[0:],
                          mobileArgs=xpcsh.mobileArgs,
                          **options.__dict__):
        sys.exit(1)
Esempio n. 38
0
def get_logger():
    logger = get_default_logger('test-runtimes')
    if logger:
        return logger

    log = {
        'log_{}'.format(k.replace('-', '_')): v
        for k, v in dict(config.log).iteritems()
    }
    return setup_logging('test-runtimes', log)
Esempio n. 39
0
def setup_logging(args, defaults):
    global logger
    logger = commandline.setup_logging("web-platform-tests-unstable", args, defaults)
    wptrunner.setup_stdlib_logger()

    for name in args.keys():
        if name.startswith("log_"):
            args.pop(name)

    return logger
Esempio n. 40
0
def cli():
    global logger
    global webapi_results
    global webapi_results_embed_app

    reload(sys)
    sys.setdefaultencoding('utf-8')

    parser = argparse.ArgumentParser()
    parser.add_argument("--version",
                        help="version of FxOS under test",
                        default="2.2",
                        action="store")
    parser.add_argument("--debug",
                        help="enable debug logging",
                        action="store_true")
    parser.add_argument("--list-test-groups",
                        help="print test groups available to run",
                        action="store_true")
    parser.add_argument("--include",
                        metavar="TEST-GROUP",
                        help="include this test group",
                        action="append")
    parser.add_argument("--result-file",
                        help="absolute file path to store the resulting json." \
                             "Defaults to results.json on your current path",
                        action="store")
    parser.add_argument("--html-result-file",
                        help="absolute file path to store the resulting html.",
                        action="store")
    parser.add_argument("--generate-reference",
                        help="Generate expected result files",
                        action="store_true")
    parser.add_argument(
        '-p',
        "--device-profile",
        action="store",
        type=os.path.abspath,
        help=
        "specify the device profile file path which could include skipped test case information"
    )
    commandline.add_logging_group(parser)

    args = parser.parse_args()

    if not args.debug:
        logging.disable(logging.ERROR)

    logger = commandline.setup_logging("certsuite", vars(args), {})

    try:
        _run(args, logger)
    except:
        logger.critical(traceback.format_exc())
        raise
Esempio n. 41
0
def run_firefox_ui_test(testtype=None, topsrcdir=None, **kwargs):
    from mozlog.structured import commandline
    from argparse import Namespace
    import firefox_ui_harness

    if testtype == "functional":
        parser = setup_argument_parser_functional()

    test_types = {
        "functional": {
            "default_tests": [
                os.path.join("functional", "manifest.ini"),
            ],
            "cli_module": firefox_ui_harness.cli_functional,
        },
    }

    fxui_dir = os.path.join(topsrcdir, "testing", "firefox-ui")

    # Set the resources path which is used to serve test data via wptserve
    if not kwargs["server_root"]:
        kwargs["server_root"] = os.path.join(fxui_dir, "resources")

    # If called via "mach test" a dictionary of tests is passed in
    if "test_objects" in kwargs:
        tests = []
        for obj in kwargs["test_objects"]:
            tests.append(obj["file_relpath"])
        kwargs["tests"] = tests
    elif not kwargs.get("tests"):
        # If no tests have been selected, set default ones
        kwargs["tests"] = [
            os.path.join(fxui_dir, "tests", test)
            for test in test_types[testtype]["default_tests"]
        ]

    kwargs["logger"] = kwargs.pop("log", None)
    if not kwargs["logger"]:
        kwargs["logger"] = commandline.setup_logging(
            "Firefox UI - {} Tests".format(testtype), {"mach": sys.stdout}
        )

    args = Namespace()

    for k, v in kwargs.iteritems():
        setattr(args, k, v)

    parser.verify_usage(args)

    failed = test_types[testtype]["cli_module"].cli(args=vars(args))

    if failed > 0:
        return 1
    else:
        return 0
Esempio n. 42
0
    def test_logging_defaultlevel(self):
        parser = argparse.ArgumentParser()
        commandline.add_logging_group(parser)

        args = parser.parse_args(["--log-tbpl=%s" % self.logfile.name])
        logger = commandline.setup_logging("test_fmtopts", args, {})
        logger.info("INFO message")
        logger.debug("DEBUG message")
        logger.error("ERROR message")
        # The debug level is not logged by default.
        self.assertEqual(["INFO message", "ERROR message"], self.loglines)
Esempio n. 43
0
def setup_logging(args, defaults):
    """Use the command line arguments to set up the logger.

    :param args: Dictionary of command line arguments.
    :param defaults: Dictionary of {formatter_name: stream} to use if
                     no command line logging is specified"""
    logger = commandline.setup_logging("web-platform-tests-update", args, defaults)

    remove_logging_args(args)

    return logger
Esempio n. 44
0
def setup_logging(args, defaults):
    global logger
    logger = commandline.setup_logging("web-platform-tests-unstable", args,
                                       defaults)
    wptrunner.setup_stdlib_logger()

    for name in args.keys():
        if name.startswith("log_"):
            args.pop(name)

    return logger
Esempio n. 45
0
def setup_logging(args, defaults):
    """Use the command line arguments to set up the logger.

    :param args: Dictionary of command line arguments.
    :param defaults: Dictionary of {formatter_name: stream} to use if
                     no command line logging is specified"""
    logger = commandline.setup_logging("web-platform-tests-update", args, defaults)

    remove_logging_args(args)

    return logger
Esempio n. 46
0
 def test_logging_debuglevel(self):
     parser = argparse.ArgumentParser()
     commandline.add_logging_group(parser)
     args = parser.parse_args(
         ["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=debug"])
     logger = commandline.setup_logging("test_fmtopts", args, {})
     logger.info("INFO message")
     logger.debug("DEBUG message")
     logger.error("ERROR message")
     # Requesting a lower log level than default works as expected.
     self.assertEqual(["INFO message", "DEBUG message", "ERROR message"],
                      self.loglines)
Esempio n. 47
0
    def test_logging_errorlevel(self):
        parser = argparse.ArgumentParser()
        commandline.add_logging_group(parser)
        args = parser.parse_args(
            ["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=error"])
        logger = commandline.setup_logging("test_fmtopts", args, {})
        logger.info("INFO message")
        logger.debug("DEBUG message")
        logger.error("ERROR message")

        # Only the error level and above were requested.
        self.assertEqual(["ERROR message"], self.loglines)
Esempio n. 48
0
    def test_logging_errorlevel(self):
        parser = argparse.ArgumentParser()
        commandline.add_logging_group(parser)
        args = parser.parse_args(["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=error"])
        logger = commandline.setup_logging("test_fmtopts", args, {})
        logger.info("INFO message")
        logger.debug("DEBUG message")
        logger.error("ERROR message")

        # Only the error level and above were requested.
        self.assertEqual(["ERROR message"],
                         self.loglines)
Esempio n. 49
0
def run_marionette(tests, b2g_path=None, emulator=None, testtype=None,
    address=None, binary=None, topsrcdir=None, **kwargs):

    # Import the harness directly and under a different name here to avoid
    # "marionette" being importable from two locations when "testing/marionette/client"
    # is on sys.path.
    # See bug 1050511 and bug 1114474. This needs to be removed with the
    # resolution of bug 1109183.
    clientdir = os.path.join(topsrcdir, 'testing/marionette/client')
    if clientdir in sys.path:
        sys.path.remove(clientdir)
    path = os.path.join(topsrcdir, 'testing/marionette/client/marionette/runtests.py')
    with open(path, 'r') as fh:
        imp.load_module('marionetteharness', fh, path,
                        ('.py', 'r', imp.PY_SOURCE))

    from marionetteharness import (
        MarionetteTestRunner,
        BaseMarionetteOptions,
        startTestRunner
    )

    parser = BaseMarionetteOptions()
    commandline.add_logging_group(parser)
    options, args = parser.parse_args()

    if not tests:
        tests = [os.path.join(topsrcdir,
                    'testing/marionette/client/marionette/tests/unit-tests.ini')]

    if b2g_path:
        options.homedir = b2g_path
        if emulator:
            options.emulator = emulator
    else:
        options.binary = binary
        path, exe = os.path.split(options.binary)

    for k, v in kwargs.iteritems():
        setattr(options, k, v)

    parser.verify_usage(options, tests)

    options.logger = commandline.setup_logging("Marionette Unit Tests",
                                               options,
                                               {"mach": sys.stdout})

    runner = startTestRunner(MarionetteTestRunner, options, tests)
    if runner.failed > 0:
        return 1

    return 0
Esempio n. 50
0
def main():
    parser = get_parser()
    commandline.add_logging_group(parser)

    args = parser.parse_args()

    logger = commandline.setup_logging("structured-example", args, {"raw": sys.stdout})

    runner = TestRunner()
    try:
        runner.run()
    except:
        logger.critical("Error during test run:\n%s" % traceback.format_exc())
Esempio n. 51
0
 def test_logging_debuglevel(self):
     parser = argparse.ArgumentParser()
     commandline.add_logging_group(parser)
     args = parser.parse_args(["--log-tbpl=%s" % self.logfile.name, "--log-tbpl-level=debug"])
     logger = commandline.setup_logging("test_fmtopts", args, {})
     logger.info("INFO message")
     logger.debug("DEBUG message")
     logger.error("ERROR message")
     # Requesting a lower log level than default works as expected.
     self.assertEqual(["INFO message",
                       "DEBUG message",
                       "ERROR message"],
                      self.loglines)
Esempio n. 52
0
    def test_logging_defaultlevel(self):
        parser = argparse.ArgumentParser()
        commandline.add_logging_group(parser)

        args = parser.parse_args(["--log-tbpl=%s" % self.logfile.name])
        logger = commandline.setup_logging("test_fmtopts", args, {})
        logger.info("INFO message")
        logger.debug("DEBUG message")
        logger.error("ERROR message")
        # The debug level is not logged by default.
        self.assertEqual(["INFO message",
                          "ERROR message"],
                         self.loglines)
Esempio n. 53
0
    def __init__(self, logfile=None, out=sys.stdout):
        self.starttime = time.clock()
        self.ready = False
        self.args = {}
        self.argparser = argparse.ArgumentParser(
            description='Run automated benchmark suite, optionally adding \
                         datapoints to a sqlite database')
        self.arglist = {}
        self.out = out
        self.modules = {}
        self.logfile = None
        self.buildtime = None
        self.buildname = None
        self.sqlite = False
        self.errors = []
        self.warnings = []

        # Default to outputing 'mach' style to stdout.
        log_args = {'log_mach': ['-']}
        if logfile:
            # If a logfile is requested we also output in a raw structured log
            # format to the requested file.
            log_args.update({'log_raw': [logfile]})

        self.logger = commandline.setup_logging("AwsyTest", log_args)

        # These can be passed to setup() like so:
        #   mytester.setup({'binary': 'blah', 'buildname': 'blee'})
        # OR you can call mytester.parseArgs() on a command-line formatted arg
        #   list (sys.argv) to extract them as needed.
        self.add_argument('-b', '--binary',
                          help='The binary (either in the current PATH or a full path) to test')
        self.add_argument('--buildname',
                          help='The name of this firefox build. If omitted, attempts to use the \
                                commit id from the mercurial respository the binary resides \
                                in')
        self.add_argument('--buildtime',
                          help='The unix timestamp to assign to this build \
                                build. If omitted, attempts to use the commit timestamp \
                                from the mercurial repository the binary resides in')
        self.add_argument('--test-module', '-m',
                          help='Load the specified test module (from libs). You must load at \
                                least one module to have tests',
                          action='append')
        self.add_argument('-l', '--logfile',
                          help='Log to given file')
        self.add_argument('-s', '--sqlitedb',
                          help='Merge datapoint into specified sqlite database')

        self.info("BenchTester instantiated")
Esempio n. 54
0
def main():
    parser = get_parser()
    commandline.add_logging_group(parser)

    args = parser.parse_args()

    logger = commandline.setup_logging("structured-example", args,
                                       {"raw": sys.stdout})

    runner = TestRunner(logger)
    try:
        runner.run()
    except:
        logger.critical("Error during test run:\n%s" % traceback.format_exc())
Esempio n. 55
0
def cli():
    global logger
    global webapi_results
    global webapi_results_embed_app

    reload(sys)
    sys.setdefaultencoding('utf-8')

    parser = argparse.ArgumentParser()
    parser.add_argument("--version",
                        help="version of FxOS under test",
                        default="2.2",
                        action="store")
    parser.add_argument("--debug",
                        help="enable debug logging",
                        action="store_true")
    parser.add_argument("--list-test-groups",
                        help="print test groups available to run",
                        action="store_true")
    parser.add_argument("--include",
                        metavar="TEST-GROUP",
                        help="include this test group",
                        action="append")
    parser.add_argument("--result-file",
                        help="absolute file path to store the resulting json." \
                             "Defaults to results.json on your current path",
                        action="store")
    parser.add_argument("--html-result-file",
                        help="absolute file path to store the resulting html.",
                        action="store")
    parser.add_argument("--generate-reference",
                        help="Generate expected result files",
                        action="store_true")
    parser.add_argument('-p', "--device-profile", action="store",  type=os.path.abspath,
                        help="specify the device profile file path which could include skipped test case information")
    commandline.add_logging_group(parser)

    args = parser.parse_args()

    if not args.debug:
        logging.disable(logging.ERROR)

    logger = commandline.setup_logging("certsuite", vars(args), {})

    try:
        _run(args, logger)
    except:
        logger.critical(traceback.format_exc())
        raise