コード例 #1
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <product> <test> <output dir>"

    parser = eideticker.OptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true", dest = "enable_profiling",
                      help = "Create SPS profile to go along with capture")
    parser.add_option("--device-id", action="store", dest="device_id",
                      help="id of device (used in output json)")
    parser.add_option("--apk", action="store", dest="apk",
                      help = "Product apk to get metadata from " \
                          "(Android-specific)")
    parser.add_option("--num-runs", action="store",
                      type = "int", dest = "num_runs",
                      help = "number of runs (default: 1)")

    options, args = parser.parse_args()
    if len(args) != 3:
        parser.error("incorrect number of arguments")

    (productname, testkey, outputdir) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    manifest = manifestparser.TestManifest(manifests=[os.path.join(
                os.path.dirname(__file__), '../src/tests/manifest.ini')])

    # sanity check... does the test match a known test key?
    testkeys = [test["key"] for test in manifest.active_tests()]
    if testkey not in testkeys:
        print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
        sys.exit(1)

    testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]

    device_id = options.device_id
    if not device_id:
        device_id = os.environ.get('DEVICE_ID')
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
        sys.exit(1)

    product = eideticker.get_product(productname)

    current_date = time.strftime("%Y-%m-%d")
    datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    devicePrefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**devicePrefs)

    # update the device list for the dashboard
    devices = {}
    devicefile = os.path.join(outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
    if os.path.isfile(testfile):
        tests = json.loads(open(testfile).read())['tests']
    else:
        tests = {}
    tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
                       'defaultMeasure': testinfo['defaultMeasure'] }
    devices[device_id] = { 'name': device.model,
                           'version': device.getprop('ro.build.version.release') }
    with open(devicefile, 'w') as f:
        f.write(json.dumps({ 'devices': devices }))
    testfiledir = os.path.dirname(testfile)
    if not os.path.exists(testfiledir):
        os.mkdir(testfiledir)
    with open(testfile, 'w') as f:
        f.write(json.dumps({ 'tests': tests }))

    if options.apk:
        appinfo = eideticker.get_fennec_appinfo(options.apk)
        appname = appinfo['appname']
        print "Using application name '%s' from apk '%s'" % (appname, options.apk)
        capture_name = "%s %s" % (product['name'], appinfo['date'])
    else:
        # no apk, assume it's something static on the device
        appinfo = { 'date': 'today' }
        appname = product['appname']
        capture_name = "%s (taken on %s)" % (product['name'], current_date)

    # Run the test the specified number of times
    for i in range(num_runs):
        # Now run the test
        runtest(device, product, appname, appinfo, testinfo,
                capture_name + " #%s" % i, outputdir, datafile, data,
                enable_profiling=options.enable_profiling, **devicePrefs)

        # Kill app after test complete
        device.killProcess(appname)
コード例 #2
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <product> <test> <output dir>"

    parser = eideticker.TestOptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true",
                      dest="enable_profiling",
                      help="Create SPS profile to go along with capture")
    parser.add_option("--device-id",
                      action="store",
                      dest="device_id",
                      help="id of device (used in output json)",
                      default=os.environ.get('DEVICE_ID'))
    parser.add_option("--device-name",
                      action="store",
                      dest="device_name",
                      help="name of device to display in dashboard (if not "
                      "specified, display model name)",
                      default=os.environ.get('DEVICE_NAME'))
    parser.add_option("--apk",
                      action="store",
                      dest="apk",
                      help="Product apk to get metadata from "
                      "(Android-specific)")
    parser.add_option("--baseline",
                      action="store_true",
                      dest="baseline",
                      help="Create baseline results for dashboard")
    parser.add_option("--num-runs",
                      action="store",
                      type="int",
                      dest="num_runs",
                      help="number of runs (default: 1)")
    parser.add_option("--app-version",
                      action="store",
                      dest="app_version",
                      help="Specify app version (if not automatically "
                      "available; Android-specific)")
    parser.add_option("--sources-xml",
                      action="store",
                      dest="sources_xml",
                      help="Path to sources XML file for getting revision "
                      "information (B2G-specific)")

    options, args = parser.parse_args()

    if len(args) != 3:
        parser.print_usage()
        sys.exit(1)

    (productname, testkey, outputdir) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    testinfo = eideticker.get_testinfo(testkey)

    device_id = options.device_id
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with "
        "DEVICE_ID environment variable)"
        sys.exit(1)

    # we'll log http requests for webstartup tests only
    log_http_requests = False
    if testinfo['type'] == 'webstartup':
        log_http_requests = True

    # likewise, log actions only for web tests and b2g tests
    log_actions = False
    if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
        log_actions = True

    product = eideticker.get_product(productname)
    current_date = time.strftime("%Y-%m-%d")
    capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
                                              current_date)
    datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    device_prefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**device_prefs)

    devices = {}
    devicefile = os.path.join(outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
    if os.path.isfile(testfile):
        tests = json.loads(open(testfile).read())['tests']
    else:
        tests = {}
    tests[testkey] = {
        'shortDesc': testinfo['shortDesc'],
        'defaultMeasure': testinfo['defaultMeasure']
    }

    device_name = options.device_name
    if not device_name:
        device_name = device.model

    if options.devicetype == "android":
        devices[device_id] = {
            'name': device_name,
            'version': device.getprop('ro.build.version.release')
        }
        if options.apk:
            if options.app_version:
                raise Exception("Should specify either --app-version or "
                                "--apk, not both!")
            appinfo = eideticker.get_fennec_appinfo(options.apk)
            appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (appname,
                                                                 options.apk)
            capture_name = "%s %s" % (product['name'], appinfo['appdate'])
        else:
            if not options.app_version:
                raise Exception("Should specify --app-version if not --apk!")

            # no apk, assume it's something static on the device
            appinfo = {
                'appdate': time.strftime("%Y-%m-%d"),
                'version': options.app_version
            }
            appname = product['appname']

    elif options.devicetype == "b2g":
        if not options.sources_xml:
            raise Exception("Must specify --sources-xml on b2g!")

        devices[device_id] = {'name': device_name}
        appinicontents = device.pullFile('/system/b2g/application.ini')
        sfh = StringIO.StringIO(appinicontents)
        appinfo = eideticker.get_appinfo(sfh)
        appinfo.update(get_revision_data(options.sources_xml))
        appname = None
    else:
        print "Unknown device type '%s'!" % options.devicetype

    # update the device / test list for the dashboard
    with open(devicefile, 'w') as f:
        f.write(json.dumps({'devices': devices}))
    testfiledir = os.path.dirname(testfile)
    if not os.path.exists(testfiledir):
        os.mkdir(testfiledir)
    with open(testfile, 'w') as f:
        f.write(json.dumps({'tests': tests}))

    if options.prepare_test:
        eideticker.prepare_test(testkey, device_prefs)

    # Run the test the specified number of times
    for i in range(num_runs):
        runtest(device,
                device_prefs,
                options.capture_device,
                options.capture_area,
                product,
                appname,
                appinfo,
                testinfo,
                capture_name + " #%s" % i,
                outputdir,
                datafile,
                data,
                enable_profiling=options.enable_profiling,
                log_http_requests=log_http_requests,
                log_actions=log_actions,
                baseline=options.baseline,
                wifi_settings_file=options.wifi_settings_file,
                sync_time=options.sync_time)
        if options.devicetype == "android":
            # Kill app after test complete
            device.killProcess(appname)
コード例 #3
0
def runtest(device_prefs, testname, options, apk=None, appname=None,
            appdate=None):
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo['appname']
        print "Installing %s (version: %s, revision %s)" % (
            appinfo['appname'], appinfo['version'], appinfo['revision'])
        device = eideticker.getDevice(**device_prefs)
        device.updateApp(apk)
    else:
        appinfo = None

    options.appname = appname

    testinfo = eideticker.get_testinfo(testname)
    stableframecapture = (testinfo['type'] in ('startup', 'webstartup') or
                          testinfo['defaultMeasure'] == 'timetostableframe')

    capture_results = []

    if options.prepare_test:
        eideticker.prepare_test(testname, options)

    for i in range(options.num_runs):
        # Now run the test
        curtime = int(time.time())
        capture_file = os.path.join(CAPTURE_DIR,
                                    "metric-test-%s-%s.zip" % (appname,
                                                               curtime))
        if options.enable_profiling and options.outputdir:
            profile_relpath = os.path.join(
                'profiles', 'sps-profile-%s.zip' % time.time())
            profile_filename = os.path.join(options.outputdir, profile_relpath)
        else:
            profile_filename = None

        current_date = time.strftime("%Y-%m-%d")
        capture_name = "%s - %s (taken on %s)" % (
            testname, appname, current_date)

        testlog = eideticker.run_test(testname, options,
                                      capture_filename=capture_file,
                                      profile_filename=profile_filename,
                                      capture_name=capture_name)

        capture_uuid = uuid.uuid1().hex
        datapoint = { 'uuid': capture_uuid }
        metadata = {}
        metrics = {}

        if options.capture:
            capture = videocapture.Capture(capture_file)

            datapoint['captureFile'] = metadata['captureFile'] = capture_file
            metadata['captureFPS'] = capture.fps
            metadata['generatedVideoFPS'] = capture.generated_video_fps

            if stableframecapture:
                metrics['timetostableframe'] = \
                    eideticker.get_stable_frame_time(capture)
            else:
                metrics.update(
                    eideticker.get_standard_metrics(capture, testlog.actions))
            metadata['metrics'] = metrics

            metadata.update(eideticker.get_standard_metric_metadata(capture))

            if options.outputdir:
                # video
                video_relpath = os.path.join(
                    'videos', 'video-%s.webm' % time.time())
                video_path = os.path.join(options.outputdir, video_relpath)
                open(video_path, 'w').write(capture.get_video().read())
                metadata['video'] = video_relpath

        if options.log_checkerboard_stats:
            metrics['internalcheckerboard'] = \
                testlog.checkerboard_percent_totals

        # Want metrics data in data, so we can graph everything at once
        datapoint.update(metrics)

        if options.enable_profiling:
            metadata['profile'] = profile_filename

        # dump metadata
        if options.outputdir:
            # metadata
            metadata_path = os.path.join(options.outputdir, 'metadata',
                                         '%s.json' % capture_uuid)
            open(metadata_path, 'w').write(json.dumps(metadata))

        capture_results.append(datapoint)

    if options.devicetype == "b2g":
        # FIXME: get information from sources.xml and application.ini on
        # device, as we do in update-dashboard.py
        display_key = appkey = "FirefoxOS"
    else:
        appkey = appname
        if appdate:
            appkey = appdate.isoformat()
        else:
            appkey = appname

        if appinfo and appinfo.get('revision'):
            display_key = "%s (%s)" % (appkey, appinfo['revision'])
        else:
            display_key = appkey

    print "=== Results on %s for %s ===" % (testname, display_key)

    if options.capture:
        measures = [ ('timetostableframe',
                      'Times to first stable frame (seconds)'),
                     ('uniqueframes', 'Number of unique frames'),
                     ('fps', 'Average number of unique frames per second'),
                     ('overallentropy',
                      'Overall entropy over length of capture'),
                     ('checkerboard',
                      'Checkerboard area/duration (sum of percents NOT '
                      'percentage)'),
                     ('timetoresponse',
                      'Time to first input response (seconds)') ]
        for measure in measures:
            if capture_results[0].get(measure[0]):
                print "  %s:" % measure[1]
                print "  %s" % map(lambda c: c[measure[0]], capture_results)
                print

        print "  Capture files:"
        print "  Capture files: %s" % map(lambda c: c['captureFile'], capture_results)
        print

    if options.log_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not "
        "percentage):"
        print "  %s" % map(
            lambda c: c['internalcheckerboard'], capture_results)
        print

    if options.outputdir:
        outputfile = os.path.join(options.outputdir, "metric.json")
        resultdict = {'title': testname, 'data': {}}
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict['data'].get(appkey):
            resultdict['data'][appkey] = []
        resultdict['data'][appkey].extend(capture_results)

        with open(outputfile, 'w') as f:
            f.write(json.dumps(resultdict))
コード例 #4
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <product> <test> <output dir>"

    parser = eideticker.CaptureOptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true", dest = "enable_profiling",
                      help = "Create SPS profile to go along with capture")
    parser.add_option("--device-id", action="store", dest="device_id",
                      help="id of device (used in output json)")
    parser.add_option("--apk", action="store", dest="apk",
                      help = "Product apk to get metadata from " \
                          "(Android-specific)")
    parser.add_option("--baseline", action="store_true", dest="baseline",
                      help = "Create baseline results for dashboard")
    parser.add_option("--num-runs", action="store",
                      type = "int", dest = "num_runs",
                      help = "number of runs (default: 1)")
    parser.add_option("--app-version", action="store", dest="app_version",
                      help="Specify app version (if not automatically " \
                          "available; Android-specific)")
    parser.add_option("--sources-xml", action="store", dest="sources_xml",
                      help="Path to sources XML file for getting revision " \
                          "information (B2G-specific)")

    options, args = parser.parse_args()
    parser.validate_options(options)

    (productname, testkey, outputdir) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    manifest = eideticker.get_test_manifest()

    # sanity check... does the test match a known test key?
    testkeys = [test["key"] for test in manifest.active_tests()]
    if testkey not in testkeys:
        print "ERROR: No tests matching '%s' (options: %s)" % (testkey, ", ".join(testkeys))
        sys.exit(1)

    testinfo = [test for test in manifest.active_tests() if test['key'] == testkey][0]

    device_id = options.device_id
    if not device_id:
        device_id = os.environ.get('DEVICE_ID')
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
        sys.exit(1)

    # we'll log http requests for webstartup tests only
    log_http_requests = False
    if testinfo['type'] == 'webstartup':
        log_http_requests = True

    product = eideticker.get_product(productname)
    current_date = time.strftime("%Y-%m-%d")
    capture_name = "%s (taken on %s)" % (product['name'], current_date)
    datafile = os.path.join(outputdir, device_id, '%s.json' % testkey)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    device_prefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**device_prefs)

    devices = {}
    devicefile = os.path.join(outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    testfile = os.path.join(outputdir, '%s' % device_id, 'tests.json')
    if os.path.isfile(testfile):
        tests = json.loads(open(testfile).read())['tests']
    else:
        tests = {}
    tests[testkey] = { 'shortDesc': testinfo['shortDesc'],
                       'defaultMeasure': testinfo['defaultMeasure'] }

    if options.devicetype == "android":
        devices[device_id] = { 'name': device.model,
                               'version': device.getprop('ro.build.version.release') }
        if options.apk:
            if options.app_version:
                raise Exception("Should specify either --app-version or --apk, not both!")
            appinfo = eideticker.get_fennec_appinfo(options.apk)
            appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (appname, options.apk)
            capture_name = "%s %s" % (product['name'], appinfo['appdate'])
        else:
            if not options.app_version:
                raise Exception("Should specify --app-version if not --apk!")

            # no apk, assume it's something static on the device
            appinfo = { 'appdate': time.strftime("%Y-%m-%d"), 'version': options.app_version }
            appname = product['appname']

    elif options.devicetype == "b2g":
        if not options.sources_xml:
            raise Exception("Must specify --sources-xml on b2g!")

        devices[device_id] = { 'name': device.model }
        appinicontents = device.pullFile('/system/b2g/application.ini')
        sfh = StringIO.StringIO(appinicontents)
        appinfo = eideticker.get_appinfo(sfh)
        appinfo.update(get_revision_data(options.sources_xml))
        appname = None
    else:
        print "Unknown device type '%s'!" % options.devicetype

    # update the device / test list for the dashboard
    with open(devicefile, 'w') as f:
        f.write(json.dumps({ 'devices': devices }))
    testfiledir = os.path.dirname(testfile)
    if not os.path.exists(testfiledir):
        os.mkdir(testfiledir)
    with open(testfile, 'w') as f:
        f.write(json.dumps({ 'tests': tests }))

    capture_area = None
    if options.capture_area:
        # we validated this previously...
        capture_area = json.loads(options.capture_area)

    # Run the test the specified number of times
    for i in range(num_runs):
        # Now run the test
        runtest(device, device_prefs, options.capture_device, capture_area,
                product, appname, appinfo, testinfo,
                capture_name + " #%s" % i, outputdir, datafile, data,
                enable_profiling=options.enable_profiling,
                log_http_requests=log_http_requests,
                baseline=options.baseline)
        if options.devicetype == "android":
            # Kill app after test complete
            device.killProcess(appname)
コード例 #5
0
def runtest(device_prefs,
            testname,
            options,
            apk=None,
            appname=None,
            appdate=None):
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo['appname']
        print "Installing %s (version: %s, revision %s)" % (
            appinfo['appname'], appinfo['version'], appinfo['revision'])
        device = eideticker.getDevice(**device_prefs)
        device.updateApp(apk)
    else:
        appinfo = None

    testinfo = eideticker.get_testinfo(testname)
    stableframecapture = (testinfo['type'] in ('startup', 'webstartup')
                          or testinfo['defaultMeasure'] == 'timetostableframe')

    capture_results = []

    if options.prepare_test:
        eideticker.prepare_test(testname, device_prefs,
                                options.wifi_settings_file)

    for i in range(options.num_runs):
        # Now run the test
        curtime = int(time.time())
        capture_file = os.path.join(
            CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime))
        if options.enable_profiling and options.outputdir:
            profile_relpath = os.path.join('profiles',
                                           'sps-profile-%s.zip' % time.time())
            profile_file = os.path.join(options.outputdir, profile_relpath)
        else:
            profile_file = None

        current_date = time.strftime("%Y-%m-%d")
        capture_name = "%s - %s (taken on %s)" % (testname, appname,
                                                  current_date)

        testlog = eideticker.run_test(
            testname,
            options.capture_device,
            appname,
            capture_name,
            device_prefs,
            extra_prefs=options.extra_prefs,
            extra_env_vars=options.extra_env_vars,
            log_checkerboard_stats=options.get_internal_checkerboard_stats,
            profile_file=profile_file,
            capture_area=options.capture_area,
            camera_settings_file=options.camera_settings_file,
            capture=options.capture,
            fps=options.fps,
            capture_file=capture_file,
            wifi_settings_file=options.wifi_settings_file,
            sync_time=options.sync_time,
            use_vpxenc=options.use_vpxenc)

        capture_uuid = uuid.uuid1().hex
        datapoint = {'uuid': capture_uuid}
        metadata = {}
        metrics = {}

        if options.capture:
            capture = videocapture.Capture(capture_file)

            datapoint['captureFile'] = metadata['captureFile'] = capture_file
            metadata['captureFPS'] = capture.fps
            metadata['generatedVideoFPS'] = capture.generated_video_fps

            if stableframecapture:
                metrics['timetostableframe'] = \
                    eideticker.get_stable_frame_time(capture)
            else:
                metrics.update(
                    eideticker.get_standard_metrics(capture, testlog.actions))
            metadata['metrics'] = metrics

            metadata.update(eideticker.get_standard_metric_metadata(capture))

            if options.outputdir:
                # video
                video_relpath = os.path.join('videos',
                                             'video-%s.webm' % time.time())
                video_path = os.path.join(options.outputdir, video_relpath)
                open(video_path, 'w').write(capture.get_video().read())
                metadata['video'] = video_relpath

        if options.get_internal_checkerboard_stats:
            metrics['internalcheckerboard'] = \
                testlog.checkerboard_percent_totals

        # Want metrics data in data, so we can graph everything at once
        datapoint.update(metrics)

        if options.enable_profiling:
            metadata['profile'] = profile_file

        # dump metadata
        if options.outputdir:
            # metadata
            metadata_path = os.path.join(options.outputdir, 'metadata',
                                         '%s.json' % capture_uuid)
            open(metadata_path, 'w').write(json.dumps(metadata))

        capture_results.append(datapoint)

    if options.devicetype == "b2g":
        # FIXME: get information from sources.xml and application.ini on
        # device, as we do in update-dashboard.py
        display_key = appkey = "FirefoxOS"
    else:
        appkey = appname
        if appdate:
            appkey = appdate.isoformat()
        else:
            appkey = appname

        if appinfo and appinfo.get('revision'):
            display_key = "%s (%s)" % (appkey, appinfo['revision'])
        else:
            display_key = appkey

    print "=== Results on %s for %s ===" % (testname, display_key)

    if options.capture:
        measures = [
            ('timetostableframe', 'Times to first stable frame (seconds)'),
            ('uniqueframes', 'Number of unique frames'),
            ('fps', 'Average number of unique frames per second'),
            ('overallentropy', 'Overall entropy over length of capture'),
            ('checkerboard', 'Checkerboard area/duration (sum of percents NOT '
             'percentage)'),
            ('timetoresponse', 'Time to first input response (seconds)')
        ]
        for measure in measures:
            if capture_results[0].get(measure[0]):
                print "  %s:" % measure[1]
                print "  %s" % map(lambda c: c[measure[0]], capture_results)
                print

        print "  Capture files:"
        print "  Capture files: %s" % map(lambda c: c['captureFile'],
                                          capture_results)
        print

    if options.get_internal_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not "
        "percentage):"
        print "  %s" % map(lambda c: c['internalcheckerboard'],
                           capture_results)
        print

    if options.outputdir:
        outputfile = os.path.join(options.outputdir, "metric.json")
        resultdict = {'title': testname, 'data': {}}
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict['data'].get(appkey):
            resultdict['data'][appkey] = []
        resultdict['data'][appkey].extend(capture_results)

        with open(outputfile, 'w') as f:
            f.write(json.dumps(resultdict))
コード例 #6
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] TEST..."

    parser = eideticker.TestOptionParser(usage=usage)
    eideticker.add_dashboard_options(parser)
    parser.add_option("--enable-profiling",
                      action="store_true", dest="enable_profiling",
                      help="Create SPS profile to go along with capture")
    parser.add_option("--device-id", action="store", dest="device_id",
                      help="id of device (used in output json)",
                      default=os.environ.get('DEVICE_ID'))
    parser.add_option("--branch", action="store", dest="branch_id",
                      help="branch under test (used in output json)",
                      default=os.environ.get('BRANCH'))
    parser.add_option("--device-name", action="store", dest="device_name",
                      help="name of device to display in dashboard (if not "
                      "specified, display model name)",
                      default=os.environ.get('DEVICE_NAME'))
    parser.add_option("--apk", action="store", dest="apk",
                      help="Product apk to get metadata from "
                      "(Android-specific)")
    parser.add_option("--baseline", action="store_true", dest="baseline",
                      help="Create baseline results for dashboard")
    parser.add_option("--num-runs", action="store",
                      type="int", dest="num_runs",
                      help="number of runs (default: %default)", default=1)
    parser.add_option("--app-version", action="store", dest="app_version",
                      help="Specify app version (if not automatically "
                      "available; Android-specific)")
    parser.add_option("--sources-xml", action="store", dest="sources_xml",
                      help="Path to sources XML file for getting revision "
                      "information (B2G-specific)")
    parser.add_option("--product", action="store",
                      type="string", dest="product_name",
                      default="nightly",
                      help="product name (android-specific, default: "
                      "%default)")
    options, args = parser.parse_args()

    if not args: # need to specify at least one test to run!
        parser.print_usage()
        sys.exit(1)

    if not options.device_id:
        print "ERROR: Must specify device id (either with --device-id or with " \
            "DEVICE_ID environment variable)"
        sys.exit(1)
    if not options.branch_id:
        print "ERROR: Must specify branch (either with --branch or with " \
            "BRANCH environment variable)"
        sys.exit(1)

    # get device info
    device_prefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**device_prefs)
    device_name = options.device_name
    if not device_name:
        device_name = device.model

    # copy dashboard files to output directory (if applicable)
    eideticker.copy_dashboard_files(options.dashboard_dir)

    if options.devicetype == 'android':
        product = eideticker.get_product(options.product_name)
        device_info = { 'name': device_name,
                        'version': device.getprop('ro.build.version.release')}
    elif options.devicetype == 'b2g':
        product = eideticker.get_product('b2g-nightly')
        device_info = { 'name': device_name }
    else:
        print "ERROR: Unknown device type '%s'" % options.devicetype

    # update device index
    eideticker.update_dashboard_device_list(options.dashboard_dir, options.device_id,
                                            options.branch_id, device_info)

    # get application/build info
    if options.devicetype == "android":
        if options.apk:
            if options.app_version:
                raise Exception("Should specify either --app-version or "
                                "--apk, not both!")
            appinfo = eideticker.get_fennec_appinfo(options.apk)
            options.appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (
                options.appname, options.apk)
            options.capture_name = "%s %s" % (product['name'], appinfo['appdate'])
        else:
            if not options.app_version:
                raise Exception("Should specify --app-version if not --apk!")

            # no apk, assume it's something static on the device
            appinfo = {
                'appdate': time.strftime("%Y-%m-%d"),
                'version': options.app_version}

    elif options.devicetype == "b2g":
        if not options.sources_xml:
            raise Exception("Must specify --sources-xml on b2g!")

        appinicontents = device.pullFile('/system/b2g/application.ini')
        sfh = StringIO.StringIO(appinicontents)
        appinfo = eideticker.get_appinfo(sfh)
        appinfo.update(get_revision_data(options.sources_xml))
        options.appname = None
    else:
        print "Unknown device type '%s'!" % options.devicetype

    # run through the tests...
    failed_tests = []
    for testkey in args:
        testinfo = eideticker.get_testinfo(testkey)

        eideticker.update_dashboard_test_list(options.dashboard_dir, options.device_id,
                                              options.branch_id,
                                              testinfo)

        current_date = time.strftime("%Y-%m-%d")
        options.capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
                                                  current_date)

        if options.prepare_test:
            eideticker.prepare_test(testkey, options)

        # Run the test the specified number of times
        for i in range(options.num_runs):
            try:
                runtest(device, device_prefs, options,
                        product, appinfo, testinfo,
                        options.capture_name + " #%s" % i)
            except eideticker.TestException:
                print "Unable to run test '%s'. Skipping and continuing." % testkey
                failed_tests.append(testkey)
                break

        # synchronize with dashboard (if we have a server to upload to)
        if options.dashboard_server:
            eideticker.upload_dashboard(options)
        else:
            print "No dashboard server specified. Skipping upload."

    if failed_tests:
        print "The following tests failed: %s" % ", ".join(failed_tests)
        sys.exit(1)
コード例 #7
0
def run_test(device, outputdir, outputfile, test, url_params, num_runs,
             startup_test, no_capture, get_internal_checkerboard_stats,
             apk=None, appname = None, appdate = None, enable_profiling=False,
             extra_prefs="{}", dmtype="adb", devicetype="android", host=None,
             port=None):
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo['appname']
        print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
                                                            appinfo['version'],
                                                            appinfo['revision'])
        device.updateApp(apk)
    else:
        appinfo = None

    captures = []

    for i in range(num_runs):
        # Kill any existing instances of the processes
        device.killProcess(appname)

        # Now run the test
        curtime = int(time.time())
        capture_file = os.path.join(CAPTURE_DIR,
                                    "metric-test-%s-%s.zip" % (appname,
                                                               curtime))
        if enable_profiling:
            profile_file = os.path.join(PROFILE_DIR,
                                        "profile-%s-%s.zip" % (appname, curtime))

        args = ["runtest.py", "--url-params", url_params,
                "--extra-prefs", extra_prefs, appname, test]
        if get_internal_checkerboard_stats:
            checkerboard_logfile = tempfile.NamedTemporaryFile()
            args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
        if startup_test:
            args.extend(["--startup-test"])
        if no_capture:
            args.extend(["--no-capture"])
        else:
            args.extend(["--capture-file", capture_file])
        if enable_profiling:
            args.extend(["--profile-file", profile_file])
        if dmtype:
            args.extend(["-m", dmtype])
        if devicetype:
            args.extend(["-d", devicetype])
        if host:
            args.extend(["--host", host])
        if port:
            args.extend(["--port", port])
        print args
        retval = subprocess.call(args)
        if retval != 0:
            raise Exception("Failed to run test %s for %s" % (test, appname))

        capture_result = {}
        if not no_capture:
            capture_result['file'] = capture_file

            capture = videocapture.Capture(capture_file)

            if startup_test:
                capture_result['stableframe'] = videocapture.get_stable_frame(capture)
            else:
                capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
                capture_result['fps'] = videocapture.get_fps(capture)
                capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
            if outputdir:
                video_path = os.path.join('videos', 'video-%s.webm' % time.time())
                video_file = os.path.join(outputdir, video_path)
                open(video_file, 'w').write(capture.get_video().read())
                capture_result['video'] = video_path

        if enable_profiling:
            capture_result['profile'] = profile_file

        if get_internal_checkerboard_stats:
            internal_checkerboard_totals = parse_checkerboard_log(checkerboard_logfile.name)
            capture_result['internalcheckerboard'] = internal_checkerboard_totals

        captures.append(capture_result)

    appkey = appname
    if appdate:
        appkey = appdate.isoformat()
    else:
        appkey = appname

    if appinfo and appinfo.get('revision'):
        display_key = "%s (%s)" % (appkey, appinfo['revision'])
    else:
        display_key = appkey
    print "=== Results for %s ===" % display_key

    if not no_capture:
        if startup_test:
            print "  First stable frames:"
            print "  %s" % map(lambda c: c['stableframe'], captures)
            print
        else:
            print "  Number of unique frames:"
            print "  %s" % map(lambda c: c['uniqueframes'], captures)
            print

            print "  Average number of unique frames per second:"
            print "  %s" % map(lambda c: c['fps'], captures)
            print

            print "  Checkerboard area/duration (sum of percents NOT percentage):"
            print "  %s" % map(lambda c: c['checkerboard'], captures)
            print

        print "  Capture files:"
        print "  Capture files: %s" % map(lambda c: c['file'], captures)
        print

    if enable_profiling:
        print "  Profile files:"
        print "  Profile files: %s" % map(lambda c: c['profile'], captures)
        print

    if get_internal_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not percentage):"
        print "  %s" % map(lambda c: c['internalcheckerboard'], captures)
        print

    if outputfile:
        resultdict = { 'title': test, 'data': {} }
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict['data'].get(appkey):
            resultdict['data'][appkey] = []
        resultdict['data'][appkey].extend(captures)

        with open(outputfile, 'w') as f:
            f.write(json.dumps(resultdict))
コード例 #8
0
def run_test(device,
             outputdir,
             outputfile,
             test,
             url_params,
             num_runs,
             startup_test,
             no_capture,
             get_internal_checkerboard_stats,
             apk=None,
             appname=None,
             appdate=None,
             profile_file=None,
             dmtype="adb",
             host=None,
             port=None):
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo['appname']
        print "Installing %s (version: %s, revision %s)" % (
            appinfo['appname'], appinfo['version'], appinfo['revision'])
        device.updateApp(apk)
    else:
        appinfo = None

    captures = []

    for i in range(num_runs):
        # Kill any existing instances of the processes
        device.killProcess(appname)

        # Now run the test
        capture_file = os.path.join(
            CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, int(time.time())))
        args = ["runtest.py", "--url-params", url_params, appname, test]
        if get_internal_checkerboard_stats:
            checkerboard_logfile = tempfile.NamedTemporaryFile()
            args.extend(["--checkerboard-log-file", checkerboard_logfile.name])
        if startup_test:
            args.extend(["--startup-test"])
        if no_capture:
            args.extend(["--no-capture"])
        else:
            args.extend(["--capture-file", capture_file])
        if profile_file:
            args.extend(["--profile-file", profile_file])
        if dmtype:
            args.extend(["-m", dmtype])
        if host:
            args.extend(["--host", host])
        if port:
            args.extend(["--port", port])
        print args
        retval = subprocess.call(args)
        if retval != 0:
            raise Exception("Failed to run test %s for %s" % (test, appname))

        capture_result = {}
        if not no_capture:
            capture_result['file'] = capture_file

            capture = videocapture.Capture(capture_file)

            framediff_sums = videocapture.get_framediff_sums(capture)
            if startup_test:
                capture_result['stableframe'] = videocapture.get_stable_frame(
                    capture)
            else:
                capture_result[
                    'uniqueframes'] = videocapture.get_num_unique_frames(
                        capture)
                capture_result['fps'] = videocapture.get_fps(capture)
                capture_result[
                    'checkerboard'] = videocapture.get_checkerboarding_area_duration(
                        capture)
            if outputdir:
                video_path = os.path.join('videos',
                                          'video-%s.webm' % time.time())
                video_file = os.path.join(outputdir, video_path)
                open(video_file, 'w').write(capture.get_video().read())
                capture_result['video'] = video_path

        if get_internal_checkerboard_stats:
            internal_checkerboard_totals = parse_checkerboard_log(
                checkerboard_logfile.name)
            capture_result[
                'internalcheckerboard'] = internal_checkerboard_totals

        captures.append(capture_result)

    appkey = appname
    if appdate:
        appkey = appdate.isoformat()
    else:
        appkey = appname

    if appinfo and appinfo.get('revision'):
        display_key = "%s (%s)" % (appkey, appinfo['revision'])
    else:
        display_key = appkey
    print "=== Results for %s ===" % display_key

    if not no_capture:
        if startup_test:
            print "  First stable frames:"
            print "  %s" % map(lambda c: c['stableframe'], captures)
            print
        else:
            print "  Number of unique frames:"
            print "  %s" % map(lambda c: c['uniqueframes'], captures)
            print

            print "  Average number of unique frames per second:"
            print "  %s" % map(lambda c: c['fps'], captures)
            print

            print "  Checkerboard area/duration (sum of percents NOT percentage):"
            print "  %s" % map(lambda c: c['checkerboard'], captures)
            print

        print "  Capture files (for further reference):"
        print "  Capture files: %s" % map(lambda c: c['file'], captures)
        print

    if get_internal_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not percentage):"
        print "  %s" % map(lambda c: c['internalcheckerboard'], captures)
        print

    if outputfile:
        resultdict = {'title': test, 'data': {}}
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict['data'].get(appkey):
            resultdict['data'][appkey] = []
        resultdict['data'][appkey].extend(captures)

        with open(outputfile, 'w') as f:
            f.write(json.dumps(resultdict))
コード例 #9
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <test> <output dir>"

    parser = eideticker.OptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true",
                      dest="enable_profiling",
                      help="Create SPS profile to go along with capture")
    parser.add_option("--device-id",
                      action="store",
                      dest="device_id",
                      help="id of device (used in output json)")
    parser.add_option(
        "--product",
        action="store",
        dest="product",
        help="Restrict testing to product (options: %s)" %
        ", ".join([product["name"] for product in default_products]))
    parser.add_option("--num-runs",
                      action="store",
                      type="int",
                      dest="num_runs",
                      help="number of runs (default: 1)")

    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error("incorrect number of arguments")

    (testname, outputdir) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    testnames = [test["name"] for test in default_tests]
    if testname not in testnames:
        print "ERROR: No tests matching '%s' (options: %s)" % (
            testname, ", ".join(testnames))
        sys.exit(1)
    else:
        test = [test for test in default_tests if test['name'] == testname][0]

    device_id = options.device_id
    if not device_id:
        device_id = os.environ.get('DEVICE_ID')
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
        sys.exit(1)

    products = default_products
    if options.product:
        products = [
            product for product in default_products
            if product['name'] == options.product
        ]
        if not products:
            print "ERROR: No products matching '%s'" % options.product
            sys.exit(1)

    current_date = time.strftime("%Y-%m-%d")
    datafile = os.path.join(outputdir, 'data-%s.json' % device_id)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    devicePrefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**devicePrefs)

    # update the device list for the dashboard
    devices = {}
    devicefile = os.path.join(outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    devices[device_id] = {
        'name': device.model,
        'version': device.getprop('ro.build.version.release')
    }
    with open(devicefile, 'w') as f:
        f.write(json.dumps({'devices': devices}))

    for product in products:
        if product.get('url'):
            product_fname = os.path.join(DOWNLOAD_DIR,
                                         "%s.apk" % product['name'])
            appinfo = eideticker.get_fennec_appinfo(product_fname)
            appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (appname,
                                                                 product_fname)
            capture_name = "%s %s" % (product['name'], appinfo['date'])
        else:
            appinfo = {}
            appname = product['appname']
            capture_name = "%s (taken on %s)" % (product['name'], current_date)

        if appinfo.get('appname'):
            appname = appinfo['appname']
        else:
            appname = product['appname']

        # Run the test the specified number of times
        for i in range(num_runs):
            # Now run the test
            runtest(device,
                    product,
                    current_date,
                    appname,
                    appinfo,
                    test,
                    capture_name + " #%s" % i,
                    outputdir,
                    datafile,
                    data,
                    enable_profiling=options.enable_profiling,
                    **devicePrefs)

            # Kill app after test complete
            device.killProcess(appname)
コード例 #10
0
def runtest(device_prefs, testname, options, apk=None, appname=None, appdate=None):
    device = None
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo["appname"]
        print "Installing %s (version: %s, revision %s)" % (appinfo["appname"], appinfo["version"], appinfo["revision"])
        device = eideticker.getDevice(**device_prefs)
        device.updateApp(apk)
    else:
        appinfo = None

    testinfo = eideticker.get_testinfo(testname)
    stableframecapture = (
        testinfo["type"] in ("startup", "webstartup") or testinfo["defaultMeasure"] == "timetostableframe"
    )

    capture_results = []

    for i in range(options.num_runs):
        # Kill any existing instances of the processes (for Android)
        if device:
            device.killProcess(appname)

        # Now run the test
        curtime = int(time.time())
        capture_file = os.path.join(CAPTURE_DIR, "metric-test-%s-%s.zip" % (appname, curtime))
        if options.enable_profiling:
            profile_file = os.path.join(PROFILE_DIR, "profile-%s-%s.zip" % (appname, curtime))
        else:
            profile_file = None

        current_date = time.strftime("%Y-%m-%d")
        capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date)

        if options.prepare_test:
            eideticker.prepare_test(testname, device_prefs)

        testlog = eideticker.run_test(
            testname,
            options.capture_device,
            appname,
            capture_name,
            device_prefs,
            extra_prefs=options.extra_prefs,
            extra_env_vars=options.extra_env_vars,
            log_checkerboard_stats=options.get_internal_checkerboard_stats,
            profile_file=profile_file,
            capture_area=options.capture_area,
            no_capture=options.no_capture,
            fps=options.fps,
            capture_file=capture_file,
            wifi_settings_file=options.wifi_settings_file,
            sync_time=options.sync_time,
        )

        capture_result = {}
        if not options.no_capture:
            capture_result["file"] = capture_file

            capture = videocapture.Capture(capture_file)
            capture_result["captureFPS"] = capture.fps

            if stableframecapture:
                capture_result["timetostableframe"] = eideticker.get_stable_frame_time(capture)
            else:
                capture_result.update(eideticker.get_standard_metrics(capture, testlog.actions))
            if options.outputdir:
                # video
                video_relpath = os.path.join("videos", "video-%s.webm" % time.time())
                video_path = os.path.join(options.outputdir, video_relpath)
                open(video_path, "w").write(capture.get_video().read())
                capture_result["video"] = video_relpath

                # framediff
                framediff_relpath = os.path.join("framediffs", "framediff-%s.json" % time.time())
                framediff_path = os.path.join(options.outputdir, framediff_relpath)
                with open(framediff_path, "w") as f:
                    framediff = videocapture.get_framediff_sums(capture)
                    f.write(json.dumps({"diffsums": framediff}))
                capture_result["frameDiff"] = framediff_relpath

        if options.enable_profiling:
            capture_result["profile"] = profile_file

        if options.get_internal_checkerboard_stats:
            capture_result["internalcheckerboard"] = testlog.checkerboard_percent_totals

        capture_results.append(capture_result)

    if options.devicetype == "b2g":
        # FIXME: get information from sources.xml and application.ini on
        # device, as we do in update-dashboard.py
        display_key = appkey = "FirefoxOS"
    else:
        appkey = appname
        if appdate:
            appkey = appdate.isoformat()
        else:
            appkey = appname

        if appinfo and appinfo.get("revision"):
            display_key = "%s (%s)" % (appkey, appinfo["revision"])
        else:
            display_key = appkey

    print "=== Results on %s for %s ===" % (testname, display_key)

    if not options.no_capture:
        if stableframecapture:
            print "  Times to first stable frame (seconds):"
            print "  %s" % map(lambda c: c["timetostableframe"], capture_results)
            print
        else:
            print "  Number of unique frames:"
            print "  %s" % map(lambda c: c["uniqueframes"], capture_results)
            print

            print "  Average number of unique frames per second:"
            print "  %s" % map(lambda c: c["fps"], capture_results)
            print

            print "  Checkerboard area/duration (sum of percents NOT percentage):"
            print "  %s" % map(lambda c: c["checkerboard"], capture_results)
            print

            print "  Time to first input response: "
            print "  %s" % map(lambda c: c.get("timetoresponse"), capture_results)
            print

        print "  Capture files:"
        print "  Capture files: %s" % map(lambda c: c["file"], capture_results)
        print

    if options.enable_profiling:
        print "  Profile files:"
        print "  Profile files: %s" % map(lambda c: c["profile"], capture_results)
        print

    if options.get_internal_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not percentage):"
        print "  %s" % map(lambda c: c["internalcheckerboard"], capture_results)
        print

    if options.outputdir:
        outputfile = os.path.join(options.outputdir, "metric.json")
        resultdict = {"title": testname, "data": {}}
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict["data"].get(appkey):
            resultdict["data"][appkey] = []
        resultdict["data"][appkey].extend(capture_results)

        with open(outputfile, "w") as f:
            f.write(json.dumps(resultdict))
コード例 #11
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <test> <output dir>"

    parser = eideticker.OptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true", dest = "enable_profiling",
                      help = "Create SPS profile to go along with capture")
    parser.add_option("--device-id", action="store", dest="device_id",
                      help="id of device (used in output json)")
    parser.add_option("--product",
                      action="store", dest="product",
                      help = "Restrict testing to product (options: %s)" %
                      ", ".join([product["name"] for product in default_products]))
    parser.add_option("--num-runs", action="store",
                      type = "int", dest = "num_runs",
                      help = "number of runs (default: 1)")

    options, args = parser.parse_args()
    if len(args) != 2:
        parser.error("incorrect number of arguments")

    (testname, outputdir) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    testnames = [test["name"] for test in default_tests]
    if testname not in testnames:
        print "ERROR: No tests matching '%s' (options: %s)" % (testname, ", ".join(testnames))
        sys.exit(1)
    else:
        test = [test for test in default_tests if test['name'] == testname][0]

    device_id = options.device_id
    if not device_id:
        device_id = os.environ.get('DEVICE_ID')
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with DEVICE_ID environment variable)"
        sys.exit(1)

    products = default_products
    if options.product:
        products = [product for product in default_products if product['name'] == options.product]
        if not products:
            print "ERROR: No products matching '%s'" % options.product
            sys.exit(1)

    current_date = time.strftime("%Y-%m-%d")
    datafile = os.path.join(outputdir, 'data-%s.json' % device_id)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    devicePrefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**devicePrefs)

    # update the device list for the dashboard
    devices = {}
    devicefile = os.path.join(outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    devices[device_id] = { 'name': device.model,
                           'version': device.getprop('ro.build.version.release') }
    with open(devicefile, 'w') as f:
        f.write(json.dumps({ 'devices': devices }))

    for product in products:
        if product.get('url'):
            product_fname = os.path.join(DOWNLOAD_DIR, "%s.apk" % product['name'])
            appinfo = eideticker.get_fennec_appinfo(product_fname)
            appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (appname, product_fname)
            capture_name = "%s %s" % (product['name'], appinfo['date'])
        else:
            appinfo = { }
            appname = product['appname']
            capture_name = "%s (taken on %s)" % (product['name'], current_date)

        if appinfo.get('appname'):
            appname = appinfo['appname']
        else:
            appname = product['appname']

        # Run the test the specified number of times
        for i in range(num_runs):
            # Now run the test
            runtest(device, product, current_date, appname, appinfo, test,
                    capture_name + " #%s" % i, outputdir, datafile, data,
                    enable_profiling=options.enable_profiling, **devicePrefs)

            # Kill app after test complete
            device.killProcess(appname)
コード例 #12
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] TEST..."

    parser = eideticker.TestOptionParser(usage=usage)
    eideticker.add_dashboard_options(parser)
    parser.add_option("--enable-profiling",
                      action="store_true",
                      dest="enable_profiling",
                      help="Create SPS profile to go along with capture")
    parser.add_option("--dashboard-id",
                      action="store",
                      dest="dashboard_id",
                      help="id of dashboard (used in output json)",
                      default=os.environ.get('DASHBOARD_ID'))
    parser.add_option("--dashboard-name",
                      action="store",
                      dest="dashboard_name",
                      help="name of dashboard to display",
                      default=os.environ.get('DASHBOARD_NAME'))
    parser.add_option("--device-id",
                      action="store",
                      dest="device_id",
                      help="id of device (used in output json)",
                      default=os.environ.get('DEVICE_ID'))
    parser.add_option("--branch",
                      action="store",
                      dest="branch_id",
                      help="branch under test (used in output json)",
                      default=os.environ.get('BRANCH'))
    parser.add_option("--device-name",
                      action="store",
                      dest="device_name",
                      help="name of device to display in dashboard (if not "
                      "specified, display model name)",
                      default=os.environ.get('DEVICE_NAME'))
    parser.add_option("--apk",
                      action="store",
                      dest="apk",
                      help="Product apk to get metadata from "
                      "(Android-specific)")
    parser.add_option("--baseline",
                      action="store_true",
                      dest="baseline",
                      help="Create baseline results for dashboard")
    parser.add_option("--num-runs",
                      action="store",
                      type="int",
                      dest="num_runs",
                      help="number of runs (default: %default)",
                      default=1)
    parser.add_option("--app-version",
                      action="store",
                      dest="app_version",
                      help="Specify app version (if not automatically "
                      "available; Android-specific)")
    parser.add_option("--sources-xml",
                      action="store",
                      dest="sources_xml",
                      help="Path to sources XML file for getting revision "
                      "information (B2G-specific)")
    parser.add_option("--product",
                      action="store",
                      type="string",
                      dest="product_name",
                      default="nightly",
                      help="product name (android-specific, default: "
                      "%default)")
    options, args = parser.parse_args()

    if not args:  # need to specify at least one test to run!
        parser.print_usage()
        sys.exit(1)

    if not options.dashboard_id:
        parser.error("Must specify dashboard id (either with --dashboard-id "
                     "or with DASHBOARD_ID environment variable)")
    if not options.dashboard_name:
        parser.error("Must specify dashboard name (either with "
                     "--dashboard-name or with DASHBOARD_NAME environment "
                     "varaiable)")
    if not options.device_id:
        parser.error("Must specify device id (either with --device-id or with "
                     "DEVICE_ID environment variable)")
    if not options.branch_id:
        parser.error("Must specify branch (either with --branch or with "
                     "BRANCH environment variable)")

    # get device info
    device_prefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**device_prefs)
    device_name = options.device_name
    if not device_name:
        device_name = device.model

    # copy dashboard files to output directory (if applicable)
    eideticker.copy_dashboard_files(options.dashboard_dir)

    if options.devicetype == 'android':
        product = eideticker.get_product(options.product_name)
        device_info = {
            'name': device_name,
            'version': device.getprop('ro.build.version.release')
        }
    elif options.devicetype == 'b2g':
        product = eideticker.get_product('b2g-nightly')
        device_info = {'name': device_name}
    else:
        print "ERROR: Unknown device type '%s'" % options.devicetype

    # update dashboard / device index
    eideticker.update_dashboard_list(options.dashboard_dir,
                                     options.dashboard_id,
                                     options.dashboard_name)
    eideticker.update_dashboard_device_list(options.dashboard_dir,
                                            options.dashboard_id,
                                            options.device_id,
                                            options.branch_id, device_info)

    # get application/build info
    if options.devicetype == "android":
        if options.apk:
            if options.app_version:
                raise Exception("Should specify either --app-version or "
                                "--apk, not both!")
            appinfo = eideticker.get_fennec_appinfo(options.apk)
            options.appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (
                options.appname, options.apk)
            options.capture_name = "%s %s" % (product['name'],
                                              appinfo['appdate'])
        else:
            if not options.app_version:
                raise Exception("Should specify --app-version if not --apk!")

            # no apk, assume it's something static on the device
            appinfo = {
                'appdate': time.strftime("%Y-%m-%d"),
                'version': options.app_version
            }

    elif options.devicetype == "b2g":
        if not options.sources_xml:
            raise Exception("Must specify --sources-xml on b2g!")

        appinicontents = device.pullFile('/system/b2g/application.ini')
        sfh = StringIO.StringIO(appinicontents)
        appinfo = eideticker.get_appinfo(sfh)
        appinfo.update(get_revision_data(options.sources_xml))
        options.appname = None
    else:
        print "Unknown device type '%s'!" % options.devicetype

    # run through the tests...
    failed_tests = []
    for testkey in args:
        testinfo = eideticker.get_testinfo(testkey)

        eideticker.update_dashboard_test_list(options.dashboard_dir,
                                              options.dashboard_id,
                                              options.device_id,
                                              options.branch_id, testinfo)

        current_date = time.strftime("%Y-%m-%d")
        options.capture_name = "%s - %s (taken on %s)" % (
            testkey, product['name'], current_date)

        if options.prepare_test:
            eideticker.prepare_test(testkey, options)

        # Run the test the specified number of times
        for i in range(options.num_runs):
            try:
                runtest(device, device_prefs, options, product, appinfo,
                        testinfo, options.capture_name + " #%s" % i)
            except eideticker.TestException:
                print "Unable to run test '%s'. Skipping and continuing." % testkey
                failed_tests.append(testkey)
                break

        # synchronize with dashboard (if we have a server to upload to)
        if options.dashboard_server:
            eideticker.upload_dashboard(options)
        else:
            print "No dashboard server specified. Skipping upload."

    if failed_tests:
        print "The following tests failed: %s" % ", ".join(failed_tests)
        sys.exit(1)
コード例 #13
0
def main(args=sys.argv[1:]):
    usage = "usage: %prog [options] <product> <test>"

    parser = eideticker.TestOptionParser(usage=usage)
    parser.add_option("--enable-profiling",
                      action="store_true", dest="enable_profiling",
                      help="Create SPS profile to go along with capture")
    parser.add_option("--device-id", action="store", dest="device_id",
                      help="id of device (used in output json)",
                      default=os.environ.get('DEVICE_ID'))
    parser.add_option("--device-name", action="store", dest="device_name",
                      help="name of device to display in dashboard (if not "
                      "specified, display model name)",
                      default=os.environ.get('DEVICE_NAME'))
    parser.add_option("--apk", action="store", dest="apk",
                      help="Product apk to get metadata from "
                      "(Android-specific)")
    parser.add_option("--baseline", action="store_true", dest="baseline",
                      help="Create baseline results for dashboard")
    parser.add_option("--num-runs", action="store",
                      type="int", dest="num_runs",
                      help="number of runs (default: 1)")
    parser.add_option("--app-version", action="store", dest="app_version",
                      help="Specify app version (if not automatically "
                      "available; Android-specific)")
    parser.add_option("--sources-xml", action="store", dest="sources_xml",
                      help="Path to sources XML file for getting revision "
                      "information (B2G-specific)")
    parser.add_option("--output-dir", action="store",
                      type="string", dest="outputdir", default=eideticker.DASHBOARD_DIR,
                      help="output results to directory instead of src/dashboard")

    options, args = parser.parse_args()

    if len(args) != 2:
        parser.print_usage()
        sys.exit(1)

    (productname, testkey) = args
    num_runs = 1
    if options.num_runs:
        num_runs = options.num_runs

    testinfo = eideticker.get_testinfo(testkey)

    device_id = options.device_id
    if not device_id:
        print "ERROR: Must specify device id (either with --device-id or with "
        "DEVICE_ID environment variable)"
        sys.exit(1)

    # we'll log http requests for webstartup tests only
    log_http_requests = False
    if testinfo['type'] == 'webstartup':
        log_http_requests = True

    # likewise, log actions only for web tests and b2g tests
    log_actions = False
    if testinfo['type'] == 'web' or testinfo['type'] == 'b2g':
        log_actions = True

    product = eideticker.get_product(productname)
    current_date = time.strftime("%Y-%m-%d")
    capture_name = "%s - %s (taken on %s)" % (testkey, product['name'],
                                              current_date)
    datafile = os.path.join(options.outputdir, device_id, '%s.json' % testkey)

    data = NestedDict()
    if os.path.isfile(datafile):
        data.update(json.loads(open(datafile).read()))

    device_prefs = eideticker.getDevicePrefs(options)
    device = eideticker.getDevice(**device_prefs)

    devices = {}
    devicefile = os.path.join(options.outputdir, 'devices.json')
    if os.path.isfile(devicefile):
        devices = json.loads(open(devicefile).read())['devices']
    testfile = os.path.join(options.outputdir, '%s' % device_id, 'tests.json')
    if os.path.isfile(testfile):
        tests = json.loads(open(testfile).read())['tests']
    else:
        tests = {}
    tests[testkey] = {'shortDesc': testinfo['shortDesc'],
                      'defaultMeasureId': testinfo['defaultMeasure']}

    device_name = options.device_name
    if not device_name:
        device_name = device.model

    if options.devicetype == "android":
        devices[device_id] = {
            'name': device_name,
            'version': device.getprop('ro.build.version.release')}
        if options.apk:
            if options.app_version:
                raise Exception("Should specify either --app-version or "
                                "--apk, not both!")
            appinfo = eideticker.get_fennec_appinfo(options.apk)
            appname = appinfo['appname']
            print "Using application name '%s' from apk '%s'" % (
                appname, options.apk)
            capture_name = "%s %s" % (product['name'], appinfo['appdate'])
        else:
            if not options.app_version:
                raise Exception("Should specify --app-version if not --apk!")

            # no apk, assume it's something static on the device
            appinfo = {
                'appdate': time.strftime("%Y-%m-%d"),
                'version': options.app_version}
            appname = product['appname']

    elif options.devicetype == "b2g":
        if not options.sources_xml:
            raise Exception("Must specify --sources-xml on b2g!")

        devices[device_id] = {'name': device_name}
        appinicontents = device.pullFile('/system/b2g/application.ini')
        sfh = StringIO.StringIO(appinicontents)
        appinfo = eideticker.get_appinfo(sfh)
        appinfo.update(get_revision_data(options.sources_xml))
        appname = None
    else:
        print "Unknown device type '%s'!" % options.devicetype

    # copy dashboard files to output directory (if applicable)
    eideticker.copy_dashboard_files(options.outputdir)

    # update the device / test list for the dashboard
    with open(devicefile, 'w') as f:
        f.write(json.dumps({'devices': devices}))
    testfiledir = os.path.dirname(testfile)
    if not os.path.exists(testfiledir):
        os.mkdir(testfiledir)
    with open(testfile, 'w') as f:
        f.write(json.dumps({'tests': tests}))

    if options.prepare_test:
        eideticker.prepare_test(
            testkey, device_prefs, options.wifi_settings_file)

    # Run the test the specified number of times
    for i in range(num_runs):
        runtest(device, device_prefs, options,
                product, appname, appinfo, testinfo,
                capture_name + " #%s" % i, datafile, data,
                log_http_requests=log_http_requests,
                log_actions=log_actions)
コード例 #14
0
def runtest(device_prefs, capture_device, outputdir, outputfile, testname, url_params, num_runs,
             startup_test, no_capture, get_internal_checkerboard_stats,
             apk=None, appname = None, appdate = None, enable_profiling=False,
             extra_prefs={}, extra_env_vars={}):
    device = None
    if apk:
        appinfo = eideticker.get_fennec_appinfo(apk)
        appname = appinfo['appname']
        print "Installing %s (version: %s, revision %s)" % (appinfo['appname'],
                                                            appinfo['version'],
                                                            appinfo['revision'])
        device = eideticker.getDevice(**device_prefs)
        device.updateApp(apk)
    else:
        appinfo = None

    captures = []

    for i in range(num_runs):
        # Kill any existing instances of the processes (for Android)
        if device:
            device.killProcess(appname)

        # Now run the test
        curtime = int(time.time())
        capture_file = os.path.join(CAPTURE_DIR,
                                    "metric-test-%s-%s.zip" % (appname,
                                                               curtime))
        if enable_profiling:
            profile_file = os.path.join(PROFILE_DIR,
                                        "profile-%s-%s.zip" % (appname, curtime))
        else:
            profile_file = None

        if get_internal_checkerboard_stats:
            checkerboard_log_file = tempfile.NamedTemporaryFile()
        else:
            checkerboard_log_file = None

        current_date = time.strftime("%Y-%m-%d")
        capture_name = "%s - %s (taken on %s)" % (testname, appname, current_date)

        eideticker.run_test(testname, capture_device,
                            appname, capture_name, device_prefs,
                            extra_prefs=extra_prefs,
                            extra_env_vars=extra_env_vars,
                            checkerboard_log_file=checkerboard_log_file,
                            profile_file=profile_file,
                            no_capture=no_capture,
                            capture_file=capture_file)

        capture_result = {}
        if not no_capture:
            capture_result['file'] = capture_file

            capture = videocapture.Capture(capture_file)

            if startup_test:
                capture_result['stableframe'] = videocapture.get_stable_frame(capture)
            else:
                capture_result['uniqueframes'] = videocapture.get_num_unique_frames(capture)
                capture_result['fps'] = videocapture.get_fps(capture)
                capture_result['checkerboard'] = videocapture.get_checkerboarding_area_duration(capture)
            if outputdir:
                video_path = os.path.join('videos', 'video-%s.webm' % time.time())
                video_file = os.path.join(outputdir, video_path)
                open(video_file, 'w').write(capture.get_video().read())
                capture_result['video'] = video_path

        if enable_profiling:
            capture_result['profile'] = profile_file

        if get_internal_checkerboard_stats:
            internal_checkerboard_totals = parse_checkerboard_log(checkerboard_log_file.name)
            capture_result['internalcheckerboard'] = internal_checkerboard_totals

        captures.append(capture_result)

    appkey = appname
    if appdate:
        appkey = appdate.isoformat()
    else:
        appkey = appname

    if appinfo and appinfo.get('revision'):
        display_key = "%s (%s)" % (appkey, appinfo['revision'])
    else:
        display_key = appkey
    print "=== Results for %s ===" % display_key

    if not no_capture:
        if startup_test:
            print "  First stable frames:"
            print "  %s" % map(lambda c: c['stableframe'], captures)
            print
        else:
            print "  Number of unique frames:"
            print "  %s" % map(lambda c: c['uniqueframes'], captures)
            print

            print "  Average number of unique frames per second:"
            print "  %s" % map(lambda c: c['fps'], captures)
            print

            print "  Checkerboard area/duration (sum of percents NOT percentage):"
            print "  %s" % map(lambda c: c['checkerboard'], captures)
            print

        print "  Capture files:"
        print "  Capture files: %s" % map(lambda c: c['file'], captures)
        print

    if enable_profiling:
        print "  Profile files:"
        print "  Profile files: %s" % map(lambda c: c['profile'], captures)
        print

    if get_internal_checkerboard_stats:
        print "  Internal Checkerboard Stats (sum of percents, not percentage):"
        print "  %s" % map(lambda c: c['internalcheckerboard'], captures)
        print

    if outputfile:
        resultdict = { 'title': testname, 'data': {} }
        if os.path.isfile(outputfile):
            resultdict.update(json.loads(open(outputfile).read()))

        if not resultdict['data'].get(appkey):
            resultdict['data'][appkey] = []
        resultdict['data'][appkey].extend(captures)

        with open(outputfile, 'w') as f:
            f.write(json.dumps(resultdict))