def setUp(self):
self.pids = []
self.threads = []
self.profile = mozprofile.FirefoxProfile()
self.runner = mozrunner.FirefoxRunner(os.environ['BROWSER_PATH'],
profile=self.profile)
def start_jsbridge(options):
import mozrunner
import jsbridge
resolve_options(options)
if not options.get('port'):
options.port = '24242'
options.port = int(options.port)
options.binary = options.get('binary')
plugins = [jsbridge.extension_path, options.path_to_ext_root]
profile = mozrunner.FirefoxProfile(plugins=plugins,
preferences={
'browser.startup.homepage':
'about:blank',
'startup.homepage_welcome_url':
'about:blank'
})
runner = mozrunner.FirefoxRunner(profile=profile,
binary=options.binary,
cmdargs=["-jsbridge",
str(options.port)])
runner.start()
back_channel, bridge = jsbridge.wait_and_create_network(
"127.0.0.1", options.port)
return Bunch(back_channel=back_channel, bridge=bridge, runner=runner)
def find_binary(self):
# This gets called by the superclass constructor. It will
# always get called, even if a binary was passed into the
# constructor, because we want to have full control over
# what the exact setting of self.binary is.
if not self.__real_binary:
self.__real_binary = self.__find_xulrunner_binary()
if not self.__real_binary:
dummy_profile = {}
runner = mozrunner.FirefoxRunner(profile=dummy_profile)
self.__real_binary = runner.find_binary()
self.names = runner.names
return self.__real_binary
# Make url-classifier updates so rare that they won't affect tests.
'urlclassifier.updateinterval':
172800,
# Point the url-classifier to a nonexistent local URL for fast failures.
'browser.safebrowsing.provider.0.gethashURL':
'http://localhost/safebrowsing-dummy/gethash',
'browser.safebrowsing.provider.0.keyURL':
'http://localhost/safebrowsing-dummy/newkey',
'browser.safebrowsing.provider.0.updateURL':
'http://localhost/safebrowsing-dummy/update',
}
env = {}
env.update(os.environ)
env['MOZ_NO_REMOTE'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'stack'
env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
cmdargs = []
if sys.platform == 'darwin':
cmdargs.append('-foreground')
dir = os.path.dirname(__file__)
addon = os.path.join(dir, 'test_addon')
profile = mozrunner.FirefoxProfile(addons=[addon], preferences=DEFAULT_PREFS)
runner = mozrunner.FirefoxRunner(profile=profile, env=env, cmdargs=cmdargs)
runner.start()
def setUp(self):
self.pids = []
self.threads = [ ]
self.profile = mozprofile.FirefoxProfile()
self.runner = mozrunner.FirefoxRunner(self.profile)
def start():
DEFAULT_PREFS = {
# allow debug output via dump to be printed to the system console
# (setting it here just in case, even though PlainTextConsole also
# sets this preference)
'browser.dom.window.dump.enabled': True,
# warn about possibly incorrect code
'javascript.options.strict': True,
'javascript.options.showInConsole': True,
# Allow remote connections to the debugger
'devtools.debugger.remote-enabled' : True,
'extensions.sdk.console.logLevel': 'info',
'extensions.checkCompatibility.nightly' : False,
# Disable extension updates and notifications.
'extensions.update.enabled' : False,
'extensions.update.notifyUser' : False,
# From:
# http://hg.mozilla.org/mozilla-central/file/1dd81c324ac7/build/automation.py.in#l372
# Only load extensions from the application and user profile.
# AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_APPLICATION
'extensions.enabledScopes' : 5,
# Disable metadata caching for installed add-ons by default
'extensions.getAddons.cache.enabled' : False,
# Disable intalling any distribution add-ons
'extensions.installDistroAddons' : False,
# Allow installing extensions dropped into the profile folder
'extensions.autoDisableScopes' : 10,
# Disable app update
'app.update.enabled' : False,
# Point update checks to a nonexistent local URL for fast failures.
'extensions.update.url' : 'http://localhost/extensions-dummy/updateURL',
'extensions.blocklist.url' : 'http://localhost/extensions-dummy/blocklistURL',
# Make sure opening about:addons won't hit the network.
'extensions.webservice.discoverURL' : 'http://localhost/extensions-dummy/discoveryURL',
'browser.startup.homepage' : 'about:blank',
'startup.homepage_welcome_url' : 'about:blank',
'devtools.errorconsole.enabled' : True,
'devtools.chrome.enabled' : True,
'devtools.debugger.remote-enabled' : True,
'devtools.debugger.remote-port': 6789,
'devtools.debugger.prompt-connection': False,
# From:
# http://hg.mozilla.org/mozilla-central/file/1dd81c324ac7/build/automation.py.in#l388
# Make url-classifier updates so rare that they won't affect tests.
'urlclassifier.updateinterval' : 172800,
# Point the url-classifier to a nonexistent local URL for fast failures.
'browser.safebrowsing.provider.0.gethashURL' : 'http://localhost/safebrowsing-dummy/gethash',
'browser.safebrowsing.provider.0.keyURL' : 'http://localhost/safebrowsing-dummy/newkey',
'browser.safebrowsing.provider.0.updateURL' : 'http://localhost/safebrowsing-dummy/update',
}
env = {}
env.update(os.environ)
env['MOZ_NO_REMOTE'] = '1'
env['XPCOM_DEBUG_BREAK'] = 'stack'
env['NS_TRACE_MALLOC_DISABLE_STACKS'] = '1'
cmdargs = []
if sys.platform == 'darwin':
cmdargs.append('-foreground')
profile = mozrunner.FirefoxProfile(addons=[], preferences=DEFAULT_PREFS)
runner = mozrunner.FirefoxRunner(
profile=profile,
env=env,
cmdargs=cmdargs,
kp_kwargs={
"stdout":subprocess.PIPE,
"stdin":subprocess.PIPE,
"stderr":subprocess.PIPE
})
runner.start()
def cli(args):
global results
tests_ran = False
parser = argparse.ArgumentParser()
parser.add_argument('--firefox-path',
help='path to firefox binary',
default=None)
parser.add_argument('--use-b2g',
action='store_true',
help='Use marionette to run tests on firefox os')
parser.add_argument('--run-android-browser',
action='store_true',
help='Run benchmarks on stock Android browser')
parser.add_argument('--run-dolphin',
action='store_true',
help='Run benchmarks on Dolphin browser')
parser.add_argument('--chrome-path',
help='path to chrome executable',
default=None)
parser.add_argument('--post-results',
action='store_true',
help='if specified, post results to datazilla')
parser.add_argument('--device-serial',
help='serial number of the android or b2g device',
default=None)
parser.add_argument('--run-benchmarks',
help='specify which benchmarks to run')
parser.add_argument('--smoketest',
action='store_true',
help='only run smoketest')
parser.add_argument('--json-result',
help='store pure json result to file',
default=None)
parser.add_argument('--test-host',
help='network interface on which to listen and serve',
default=moznetwork.get_ip())
parser.add_argument('--test-port',
help='port to host http server',
default=None)
commandline.add_logging_group(parser)
args = parser.parse_args(args)
logging.basicConfig()
logger = commandline.setup_logging('mozbench', vars(args), {})
if not args.use_b2g and not args.firefox_path:
logger.error('you must specify one of --use-b2g or ' +
'--firefox-path')
return 1
if args.firefox_path:
use_android = args.firefox_path.endswith('.apk')
else:
use_android = False
if use_android:
logger.info('prepare for installing fennec')
fennec_pkg_name = get_fennec_pkg_name(args.firefox_path)
success = install_fennec(logger, args.firefox_path, fennec_pkg_name,
args.device_serial)
if not success:
logger.error('fennec installation fail')
return 1
logger.info('fennec installation succeed')
else:
if args.run_android_browser:
logger.warning('stock Android browser only supported on Android')
if args.run_dolphin:
logger.warning('dolphin browser only supported on Android')
logger.info('starting webserver on %s' % args.test_host)
routes = [('POST', '/results', results_handler),
('GET', '/*', wptserve.handlers.file_handler)]
static_path = os.path.abspath(
os.path.join(os.path.dirname(__file__), 'static'))
# start http server and request handler
httpd = None
if args.test_port:
try:
port = int(args.test_port)
httpd = wptserve.server.WebTestHttpd(host=args.test_host,
port=port,
routes=routes,
doc_root=static_path)
except Exception as e:
logger.error(e.message)
return 1
else:
while httpd is None:
try:
port = 10000 + random.randrange(0, 50000)
httpd = wptserve.server.WebTestHttpd(host=args.test_host,
port=port,
routes=routes,
doc_root=static_path)
# pass if port number has been used, then try another one
except socket.error as e:
pass
except Exception as e:
logger.error(e.message)
return 1
httpd.start()
httpd_logger = logging.getLogger("wptserve")
httpd_logger.setLevel(logging.ERROR)
logger.info('starting webserver on %s:%s' % (httpd.host, str(httpd.port)))
url_prefix = 'http://' + httpd.host + ':' + str(httpd.port) + '/'
result_recorder = ResultRecorder()
with open(os.path.join(os.path.dirname(__file__), 'benchmarks.json')) as f:
benchmarks = json.load(f)
# Determine platform
platform = mozinfo.os
os_version = mozinfo.version
processor = mozinfo.processor
if args.use_b2g:
platform = 'b2g'
elif use_android:
platform = 'android'
device = mozdevice.ADBAndroid(args.device_serial)
os_version = device.get_prop('ro.build.version.release')
processor = device.get_prop('ro.product.cpu.abi')
result_recorder.platform = platform
result_recorder.os_version = os_version
result_recorder.processor = processor
for benchmark in benchmarks:
suite = benchmark['suite']
url = url_prefix + benchmark['url']
num_runs = benchmark['number_of_runs']
timeout = benchmark['timeout']
name = benchmark['name']
value = benchmark['value']
if args.smoketest and suite != 'smoketest':
continue
# Check if benchmark is enabled for platform
if args.run_benchmarks:
if not suite in args.run_benchmarks.strip().split(','):
continue
elif not ('all' in benchmark['enabled']
or platform in benchmark['enabled']):
logger.info('skipping disabled benchmark: %s for platform %s' %
(suite, platform))
continue
logger.info('starting benchmark: %s' % suite)
result_recorder.set_browser('firefox.nightly')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
# Run firefox
for i in xrange(0, num_runs):
logger.info('firefox run %d' % i)
if args.use_b2g:
runner = B2GRunner(cmdargs=[url],
device_serial=args.device_serial)
elif use_android:
runner = AndroidRunner(app_name=fennec_pkg_name,
activity_name='.App',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
else:
runner = mozrunner.FirefoxRunner(binary=args.firefox_path,
cmdargs=[url])
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('firefox results: %s' % json.dumps(results))
# Run chrome (if desired)
if args.chrome_path is not None:
result_recorder.set_browser('chrome.canary')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('chrome run %d' % i)
if use_android:
runner = AndroidRunner(
app_name=args.chrome_path,
activity_name='com.google.android.apps.chrome.Main',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
else:
runner = ChromeRunner(binary=args.chrome_path,
cmdargs=[url])
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('chrome results: %s' % json.dumps(results))
# Run stock AOSP browser (if desired)
if use_android and args.run_android_browser:
result_recorder.set_browser('android-browser')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('android browser run %d' % i)
runner = AndroidRunner(app_name='com.android.browser',
activity_name='.BrowserActivity',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('android browser results: %s' %
json.dumps(results))
# Run Dolphin browser (if desired)
if use_android and args.run_dolphin:
result_recorder.set_browser('dolphin')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('dolphin run %d' % i)
runner = AndroidRunner(app_name='mobi.mgeek.TunnyBrowser',
activity_name='.BrowserActivity',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('dolphin results: %s' % json.dumps(results))
if suite == 'smoketest' and not tests_ran:
logger.error('smoketest failed to produce results - skipping '
'remaining tests')
break
if args.post_results:
logger.info('posting results...')
postresults(logger, result_recorder.get_influxdb_results())
if args.json_result:
with open(args.json_result, 'w') as outputFile:
outputFile.write(json.dumps(result_recorder.get_results()) + '\n')
# Only flag the job as failed if no tests ran at all
return 0 if tests_ran else 1
