def get_ip(self):
import moznetwork
if os.name != "nt":
return moznetwork.get_ip()
else:
self.error(
"ERROR: you must specify a --remote-webserver=<ip address>\n")
def verifyRemoteOptions(self, options):
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
self.error(
"You must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.geckoPath and not options.emulator:
self.error(
"You must specify --emulator if you specify --gecko-path")
if options.logcat_dir and not options.emulator:
self.error(
"You must specify --emulator if you specify --logcat-dir")
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" % options.xrePath)
xpcshell = os.path.join(options.xrePath, 'xpcshell')
if not os.access(xpcshell, os.F_OK):
self.error('xpcshell not found at %s' % xpcshell)
if self.elf_arm(xpcshell):
self.error('--xre-path points to an ARM version of xpcshell; it '
'should instead point to a version that can run on '
'your desktop')
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
return options
def updateApp(self, appBundlePath, processName=None, destPath=None,
ipAddr=None, port=30000, wait=False):
# port ^^^ is here for backwards compatibility only, we now
# determine a port automatically and safely
wait = (wait or ipAddr)
cmd = 'updt '
if processName is None:
# Then we pass '' for processName
cmd += "'' " + appBundlePath
else:
cmd += processName + ' ' + appBundlePath
if destPath:
cmd += " " + destPath
if wait:
if not ipAddr:
ipAddr = moznetwork.get_ip()
serverSocket = self._getRebootServerSocket(ipAddr)
cmd += " %s %s" % serverSocket.getsockname()
self._logger.debug("updateApp using command: " % cmd)
self._runCmds([{'cmd': cmd}])
if wait:
self._waitForRebootPing(serverSocket)
def reboot(self, ipAddr=None, port=30000, wait=False):
# port ^^^ is here for backwards compatibility only, we now
# determine a port automatically and safely
wait = (wait or ipAddr)
cmd = 'rebt'
self._logger.info("Rebooting device")
# if we're waiting, create a listening server and pass information on
# it to the device before rebooting (we do this instead of just polling
# to make sure the device actually rebooted -- yes, there are probably
# simpler ways of doing this like polling uptime, but this is what we're
# doing for now)
if wait:
if not ipAddr:
ipAddr = moznetwork.get_ip()
serverSocket = self._getRebootServerSocket(ipAddr)
# The update.info command tells the SUTAgent to send a TCP message
# after restarting.
destname = '/data/data/com.mozilla.SUTAgentAndroid/files/update.info'
data = "%s,%s\rrebooting\r" % serverSocket.getsockname()
self._runCmds([{'cmd': 'push %s %s' % (destname, len(data)),
'data': data}])
cmd += " %s %s" % serverSocket.getsockname()
# actually reboot device
self._runCmds([{'cmd': cmd}])
# if we're waiting, wait for a callback ping from the agent before
# continuing (and throw an exception if we don't receive said ping)
if wait:
self._waitForRebootPing(serverSocket)
def verifyRemoteOptions(self, options):
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
self.error("You must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.geckoPath and not options.emulator:
self.error("You must specify --emulator if you specify --gecko-path")
if options.logcat_dir and not options.emulator:
self.error("You must specify --emulator if you specify --logcat-dir")
if not os.path.isdir(options.xrePath):
self.error("--xre-path '%s' is not a directory" % options.xrePath)
xpcshell = os.path.join(options.xrePath, 'xpcshell')
if not os.access(xpcshell, os.F_OK):
self.error('xpcshell not found at %s' % xpcshell)
if self.elf_arm(xpcshell):
self.error('--xre-path points to an ARM version of xpcshell; it '
'should instead point to a version that can run on '
'your desktop')
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
return options
def server_init(self):
"""
Additional initialization required to satisfy MochitestDesktop.startServers
"""
self._locations = None
self.server = None
self.wsserver = None
self.websocketProcessBridge = None
self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90
if self.options.remoteWebServer is None:
if os.name != "nt":
self.options.remoteWebServer = moznetwork.get_ip()
else:
raise Exception("--remote-webserver must be specified")
self.options.webServer = self.options.remoteWebServer
self.options.webSocketPort = '9988'
self.options.httpdPath = None
self.options.keep_open = False
self.options.pidFile = ""
self.options.subsuite = None
self.options.xrePath = None
if build_obj and 'MOZ_HOST_BIN' in os.environ:
self.options.xrePath = os.environ['MOZ_HOST_BIN']
if not self.options.utilityPath:
self.options.utilityPath = self.options.xrePath
if not self.options.xrePath:
self.options.xrePath = self.options.utilityPath
if build_obj:
self.options.certPath = os.path.join(build_obj.topsrcdir, 'build',
'pgo', 'certs')
def setup_hosts(self):
hostnames = ["web-platform.test",
"www.web-platform.test",
"www1.web-platform.test",
"www2.web-platform.test",
"xn--n8j6ds53lwwkrqhv28a.web-platform.test",
"xn--lve-6lad.web-platform.test"]
host_ip = moznetwork.get_ip()
temp_dir = tempfile.mkdtemp()
hosts_path = os.path.join(temp_dir, "hosts")
remote_path = "/system/etc/hosts"
try:
self.device.getFile("/system/etc/hosts", hosts_path)
with open(hosts_path) as f:
hosts_file = HostsFile.from_file(f)
for canonical_hostname in hostnames:
hosts_file.set_host(HostsLine(host_ip, canonical_hostname))
with open(hosts_path, "w") as f:
hosts_file.to_file(f)
self.logger.info("Installing hosts file")
self.device.remount()
self.device.removeFile(remote_path)
self.device.pushFile(hosts_path, remote_path)
finally:
os.unlink(hosts_path)
os.rmdir(temp_dir)
def DroidConnectByHWID(hwid, timeout=30, **kwargs):
"""Try to connect to the given device by waiting for it to show up using mDNS with the given timeout."""
zc = Zeroconf(moznetwork.get_ip())
evt = threading.Event()
listener = ZeroconfListener(hwid, evt)
sb = ServiceBrowser(zc, "_sutagent._tcp.local.", listener)
foundIP = None
if evt.wait(timeout):
# we found the hwid
foundIP = listener.ip
sb.cancel()
zc.close()
if foundIP is not None:
return DroidSUT(foundIP, **kwargs)
print "Connected via SUT to %s [at %s]" % (hwid, foundIP)
# try connecting via adb
try:
sut = DroidADB(deviceSerial=hwid, **kwargs)
except:
return None
print "Connected via ADB to %s" % (hwid)
return sut
def synchronizeTime(self):
self._logger.info("Synchronizing time...")
ntpdate_wait_time = 5
ntpdate_retries = 5
num_retries = 0
synced_time = False
while not synced_time:
try:
self.shellCheckOutput([self.DEFAULT_NTPDATE_LOCATION, "-c", "1", "-d",
"-h", moznetwork.get_ip(), "-s"], root=True,
timeout=ntpdate_wait_time)
synced_time = True
except mozdevice.DMError:
# HACK: we need to force a reconnection here on SUT (until bug
# 1009862 is fixed and lands in a released version of
# mozdevice)
self._sock = None
if num_retries == ntpdate_retries:
raise Exception("Exceeded maximum number of retries "
"synchronizing time!")
# FIXME: as of this writing mozdevice doesn't distinguishing
# between timeouts and other errors (bug 1010328), so we're
# just gonna assume timeout
num_retries+=1
self._logger.info("Attempt to synchronize time failed, "
"retrying (try %s/%s)..." % (num_retries,
ntpdate_retries))
self._logger.info("Time synchronized!")
def DroidConnectByHWID(hwid, timeout=30, **kwargs):
"""Try to connect to the given device by waiting for it to show up using mDNS with the given timeout."""
zc = Zeroconf(moznetwork.get_ip())
evt = threading.Event()
listener = ZeroconfListener(hwid, evt)
sb = ServiceBrowser(zc, "_sutagent._tcp.local.", listener)
foundIP = None
if evt.wait(timeout):
# we found the hwid
foundIP = listener.ip
sb.cancel()
zc.close()
if foundIP is not None:
return DroidSUT(foundIP, **kwargs)
print "Connected via SUT to %s [at %s]" % (hwid, foundIP)
# try connecting via adb
try:
sut = DroidADB(deviceSerial=hwid, **kwargs)
except:
return None
print "Connected via ADB to %s" % (hwid)
return sut
def get_ip(self):
import moznetwork
if os.name != "nt":
return moznetwork.get_ip()
else:
self.error(
"ERROR: you must specify a --remote-webserver=<ip address>\n")
def test_get_ip_using_get_interface(ip_addresses):
"""Test that the control flow path for get_ip() using
_get_interface_list() is works"""
with mock.patch("socket.gethostbyname") as byname:
# Force socket.gethostbyname to return None
byname.return_value = None
assert moznetwork.get_ip() in ip_addresses
def reboot(self, ipAddr=None, port=30000, wait=False):
# port ^^^ is here for backwards compatibility only, we now
# determine a port automatically and safely
wait = (wait or ipAddr)
cmd = 'rebt'
self._logger.info("Rebooting device")
# if we're waiting, create a listening server and pass information on
# it to the device before rebooting (we do this instead of just polling
# to make sure the device actually rebooted -- yes, there are probably
# simpler ways of doing this like polling uptime, but this is what we're
# doing for now)
if wait:
if not ipAddr:
ipAddr = moznetwork.get_ip()
serverSocket = self._getRebootServerSocket(ipAddr)
# The update.info command tells the SUTAgent to send a TCP message
# after restarting.
destname = '/data/data/com.mozilla.SUTAgentAndroid/files/update.info'
data = "%s,%s\rrebooting\r" % serverSocket.getsockname()
self._runCmds([{'cmd': 'push %s %s' % (destname, len(data)),
'data': data}])
cmd += " %s %s" % serverSocket.getsockname()
# actually reboot device
self._runCmds([{'cmd': cmd}])
# if we're waiting, wait for a callback ping from the agent before
# continuing (and throw an exception if we don't receive said ping)
if wait:
self._waitForRebootPing(serverSocket)
def updateApp(self, appBundlePath, processName=None, destPath=None,
ipAddr=None, port=30000, wait=False):
# port ^^^ is here for backwards compatibility only, we now
# determine a port automatically and safely
wait = (wait or ipAddr)
cmd = 'updt '
if processName is None:
# Then we pass '' for processName
cmd += "'' " + appBundlePath
else:
cmd += processName + ' ' + appBundlePath
if destPath:
cmd += " " + destPath
if wait:
if not ipAddr:
ipAddr = moznetwork.get_ip()
serverSocket = self._getRebootServerSocket(ipAddr)
cmd += " %s %s" % serverSocket.getsockname()
self._logger.debug("updateApp using command: " % cmd)
self._runCmds([{'cmd': cmd}])
if wait:
self._waitForRebootPing(serverSocket)
def check_server(device):
logger.info("Checking access to host machine")
if _adbflag:
return True
routes = [("GET", "/", test_handler)]
host_ip = moznetwork.get_ip()
for port in [8000, 8001]:
try:
server = wptserve.WebTestHttpd(host=host_ip, port=port, routes=routes)
server.start()
except:
logger.critical("Error starting local server on port %s:\n%s" %
(port, traceback.format_exc()))
return False
try:
device.shell_output("curl http://%s:%i" % (host_ip, port))
except mozdevice.ADBError as e:
if 'curl: not found' in e.message:
logger.warning("Could not check access to host machine: curl not present.")
logger.warning("If timeouts occur, check your network configuration.")
break
logger.critical("Failed to connect to server running on host machine ip %s port %i. Check network configuration." % (host_ip, port))
return False
finally:
logger.debug("Stopping server")
server.stop()
return True
def validate(self, parser, options, context):
"""Validate b2g options."""
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"You must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if not options.b2gPath and hasattr(context, 'b2g_home'):
options.b2gPath = context.b2g_home
if hasattr(context, 'device_name') and not options.emulator:
if context.device_name.startswith('emulator'):
options.emulator = 'x86' if 'x86' in context.device_name else 'arm'
if options.geckoPath and not options.emulator:
parser.error(
"You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
parser.error("You must specify --emulator if you specify --logdir")
elif not options.logdir and options.emulator and build_obj:
options.logdir = os.path.join(build_obj.topobjdir, '_tests',
'testing', 'mochitest')
if hasattr(context, 'xre_path'):
options.xrePath = context.xre_path
if not os.path.isdir(options.xrePath):
parser.error("--xre-path '%s' is not a directory" %
options.xrePath)
xpcshell = os.path.join(options.xrePath, 'xpcshell')
if not os.access(xpcshell, os.F_OK):
parser.error('xpcshell not found at %s' % xpcshell)
if self.elf_arm(xpcshell):
parser.error('--xre-path points to an ARM version of xpcshell; it '
'should instead point to a version that can run on '
'your desktop')
if not options.httpdPath and build_obj:
options.httpdPath = os.path.join(build_obj.topobjdir, '_tests',
'testing', 'mochitest')
# Bug 1071866 - B2G Mochitests do not always produce a leak log.
options.ignoreMissingLeaks.append("default")
# Bug 1070068 - Leak logging does not work for tab processes on B2G.
options.ignoreMissingLeaks.append("tab")
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
return options
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
objdir_xpi_stage = os.path.join(build_obj.distdir, 'xpi-stage')
if os.path.isdir(objdir_xpi_stage):
options.extensionsToInstall = [
os.path.join(objdir_xpi_stage, 'mochijar'),
os.path.join(objdir_xpi_stage, 'specialpowers'),
]
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.app is None:
options.app = "org.mozilla.geckoview.test"
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if build_obj:
options.topsrcdir = build_obj.topsrcdir
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
if options.coverage_output_dir and not options.enable_coverage:
parser.error("--coverage-output-dir must be used with --enable-coverage")
if options.enable_coverage:
if not options.autorun:
parser.error(
"--enable-coverage cannot be used with --no-autorun")
if not options.coverage_output_dir:
parser.error(
"--coverage-output-dir must be specified when using --enable-coverage")
parent_dir = os.path.dirname(options.coverage_output_dir)
if not os.path.isdir(options.coverage_output_dir):
parser.error(
"The directory for the coverage output does not exist: %s" %
parent_dir)
# allow us to keep original application around for cleanup while
# running tests
options.remoteappname = options.app
return options
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def test_get_ip(self):
""" Attempt to test the IP address returned by
moznetwork.get_ip() is valid """
ip = moznetwork.get_ip()
# Check the IP returned by moznetwork is in the list
self.assertTrue(verify_ip_in_list(ip))
def create_httpd(self, need_external_ip):
host = "127.0.0.1"
if need_external_ip:
host = moznetwork.get_ip()
root = self.server_root or os.path.join(os.path.dirname(here), "www")
rv = httpd.FixtureServer(root, host=host)
rv.start()
return rv
def env_options():
# The server host is set to public localhost IP so that resources can be accessed
# from Android emulator
return {
"server_host": moznetwork.get_ip(),
"bind_address": False,
"supports_debugger": True
}
def synchronizeTime(self):
self._logger.info("Synchronizing time...")
self.shellCheckOutput([
self.DEFAULT_NTPDATE_LOCATION, "-c", "1", "-d", "-h",
moznetwork.get_ip(), "-s"
],
root=True)
self._logger.info("Time synchronized!")
def test_get_ip(self):
""" Attempt to test the IP address returned by
moznetwork.get_ip() is valid """
ip = moznetwork.get_ip()
# Check the IP returned by moznetwork is in the list
self.assertTrue(verify_ip_in_list(ip))
def create_httpd(self, need_external_ip):
host = "127.0.0.1"
if need_external_ip:
host = moznetwork.get_ip()
root = self.server_root or os.path.join(os.path.dirname(here), "www")
rv = httpd.FixtureServer(root, host=host)
rv.start()
return rv
def start_httpd(self):
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
self.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.baseurl)
def validate(self, parser, options, context):
"""Validate b2g options."""
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"You must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if not options.b2gPath and hasattr(context, 'b2g_home'):
options.b2gPath = context.b2g_home
if hasattr(context, 'device_name') and not options.emulator:
if context.device_name.startswith('emulator'):
options.emulator = 'x86' if 'x86' in context.device_name else 'arm'
if options.geckoPath and not options.emulator:
parser.error(
"You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
parser.error("You must specify --emulator if you specify --logdir")
elif not options.logdir and options.emulator and build_obj:
options.logdir = os.path.join(
build_obj.topobjdir, '_tests', 'testing', 'mochitest')
if hasattr(context, 'xre_path'):
options.xrePath = context.xre_path
if not os.path.isdir(options.xrePath):
parser.error("--xre-path '%s' is not a directory" % options.xrePath)
xpcshell = os.path.join(options.xrePath, 'xpcshell')
if not os.access(xpcshell, os.F_OK):
parser.error('xpcshell not found at %s' % xpcshell)
if self.elf_arm(xpcshell):
parser.error('--xre-path points to an ARM version of xpcshell; it '
'should instead point to a version that can run on '
'your desktop')
if not options.httpdPath and build_obj:
options.httpdPath = os.path.join(
build_obj.topobjdir, '_tests', 'testing', 'mochitest')
# Bug 1071866 - B2G Mochitests do not always produce a leak log.
options.ignoreMissingLeaks.append("default")
# Bug 1070068 - Leak logging does not work for tab processes on B2G.
options.ignoreMissingLeaks.append("tab")
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
return options
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.app is None:
if build_obj:
options.app = build_obj.substs['ANDROID_PACKAGE_NAME']
else:
parser.error("You must specify either appPath or app")
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if build_obj:
options.topsrcdir = build_obj.topsrcdir
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
if not options.robocopApk and build_obj:
apk = build_obj.substs.get('GRADLE_ANDROID_APP_ANDROIDTEST_APK')
if apk and os.path.exists(apk):
options.robocopApk = apk
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
parser.error("Unable to find robocop APK '%s'" %
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
# Disable e10s by default on Android because we don't run Android
# e10s jobs anywhere yet.
options.e10s = False
mozinfo.update({'e10s': options.e10s})
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app
return options
def run_capture(options, capture_file):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_file,
options.capture_device,
options.mode,
capture=options.capture)
host = moznetwork.get_ip()
docroot = eideticker.runtest.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0,
host=host,
docroot=docroot,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
capture_server.end_capture
}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (httpd.docroot, httpd.host,
httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
mode = options.mode
if not mode:
mode = device.hdmiResolution
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
httpd.stop()
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = "http://%s:%d/" % (host, port)
self.logger.info("running webserver on %s" % self.baseurl)
self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www"))
self.httpd.start()
def start_httpd(self, need_external_ip):
host = "127.0.0.1"
if need_external_ip:
host = moznetwork.get_ip()
self.httpd = MozHttpd(host=host,
port=0,
docroot=os.path.join(os.path.dirname(os.path.dirname(__file__)), 'www'))
self.httpd.start()
self.marionette.baseurl = 'http://%s:%d/' % (host, self.httpd.httpd.server_port)
self.logger.info('running webserver on %s' % self.marionette.baseurl)
def test_get_ip_using_get_interface(ip_addresses):
""" Test that the control flow path for get_ip() using
_get_interface_list() is works """
if mozinfo.isLinux or mozinfo.isMac:
with mock.patch('socket.gethostbyname') as byname:
# Force socket.gethostbyname to return None
byname.return_value = None
assert moznetwork.get_ip() in ip_addresses
def __init__(self, testinfo, actions={}, docroot=None, **kwargs):
super(WebTest, self).__init__(testinfo,
track_start_frame=True,
track_end_frame=True,
**kwargs)
self.actions = actions
self.capture_server = CaptureServer(self)
self.host = moznetwork.get_ip()
self.http = mozhttpd.MozHttpd(docroot=docroot,
host=self.host,
port=0,
log_requests=True,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
self.capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
self.capture_server.end_capture
}, {
'method':
'POST',
'path':
'/api/captures/input/?',
'function':
self.capture_server.input
}])
self.http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries += 1
import socket
s = socket.socket()
try:
s.connect((self.host, self.http.httpd.server_port))
connected = True
except Exception:
self.log("Can't connect to %s:%s, retrying..." %
(self.host, self.http.httpd.server_port))
self.log("Test URL is: %s" % self.url)
if not connected:
raise "Could not open webserver. Error!"
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def run_capture(options, capture_file):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_file, options.capture_device,
options.mode,
no_capture=options.no_capture)
host = moznetwork.get_ip()
docroot = eideticker.runtest.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[
{'method': 'GET',
'path': '/api/captures/start/?',
'function': capture_server.start_capture},
{'method': 'GET',
'path': '/api/captures/end/?',
'function': capture_server.end_capture}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (
httpd.docroot, httpd.host, httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
mode = options.mode
if not mode:
mode = device.hdmiResolution
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
session = device.marionette.session
if 'b2g' not in session:
raise Exception("bad session value %s returned by start_session" %
session)
device.unlock()
# wait for device to become ready (yes, this is terrible, can we
# detect this condition in marionette somehow?)
time.sleep(5)
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
device.killProcess(options.appname)
httpd.stop()
def write_hosts_file(config, device):
new_hosts = make_hosts_file(config, moznetwork.get_ip())
current_hosts = device.get_file("/etc/hosts")
if new_hosts == current_hosts:
return
hosts_fd, hosts_path = tempfile.mkstemp()
try:
with os.fdopen(hosts_fd, "w") as f:
f.write(new_hosts)
device.remount()
device.push(hosts_path, "/etc/hosts")
finally:
os.remove(hosts_path)
def write_hosts_file(config, device):
new_hosts = make_hosts_file(config, moznetwork.get_ip())
current_hosts = device.get_file("/etc/hosts")
if new_hosts == current_hosts:
return
hosts_fd, hosts_path = tempfile.mkstemp()
try:
with os.fdopen(hosts_fd, "w") as f:
f.write(new_hosts)
device.remount()
device.push(hosts_path, "/etc/hosts")
finally:
os.remove(hosts_path)
def test_get_ip_using_get_interface(self):
""" Test that the control flow path for get_ip() using
_get_interface_list() is works """
if mozinfo.isLinux:
with mock.patch("socket.gethostbyname") as byname:
# Force socket.gethostbyname to return None
byname.return_value = None
ip = moznetwork.get_ip()
# Check the IP returned by moznetwork is in the list
self.assertTrue(verify_ip_in_list(ip))
def test_get_ip_using_get_interface(self):
""" Test that the control flow path for get_ip() using
_get_interface_list() is works """
if mozinfo.isLinux or mozinfo.isMac:
with mock.patch('socket.gethostbyname') as byname:
# Force socket.gethostbyname to return None
byname.return_value = None
ip = moznetwork.get_ip()
# Check the IP returned by moznetwork is in the list
self.assertTrue(verify_ip_in_list(ip))
def __init__(self, testinfo, options, device, capture_controller, use_actions=True):
Test.__init__(self, testinfo, options, device, capture_controller,
track_start_frame=True, track_end_frame=True)
# get actions for web tests
actions_path = os.path.join(testinfo['here'], "actions.json")
if use_actions:
if os.path.exists(actions_path):
self.actions = json.loads(open(actions_path).read())
else:
raise TestException("Test needs actions but no actions file '%s'" %
actions_path)
else:
self.actions = None
self.capture_server = CaptureServer(self)
self.host = moznetwork.get_ip()
self.http = mozhttpd.MozHttpd(
docroot=TEST_DIR, host=self.host, port=0, log_requests=True,
urlhandlers=[
{'method': 'GET',
'path': '/api/captures/start/?',
'function': self.capture_server.start_capture},
{'method': 'GET',
'path': '/api/captures/end/?',
'function': self.capture_server.end_capture},
{'method': 'POST',
'path': '/api/captures/input/?',
'function': self.capture_server.input}])
self.http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries += 1
import socket
s = socket.socket()
try:
s.connect((self.host, self.http.httpd.server_port))
connected = True
except Exception:
self.logger.info("Can't connect to %s:%s, retrying..." %
(self.host, self.http.httpd.server_port))
self.logger.info("Test URL is: %s" % self.url)
if not connected:
raise "Could not open webserver. Error!"
def setup_hosts(self):
hosts = ["web-platform.test",
"www.web-platform.test",
"www1.web-platform.test",
"www2.web-platform.test",
"xn--n8j6ds53lwwkrqhv28a.web-platform.test",
"xn--lve-6lad.web-platform.test"]
host_ip = moznetwork.get_ip()
temp_dir = tempfile.mkdtemp()
hosts_path = os.path.join(temp_dir, "hosts")
remote_path = "/system/etc/hosts"
try:
self.device.getFile("/system/etc/hosts", hosts_path)
with open(hosts_path, "a+") as f:
hosts_present = set()
for line in f:
line = line.strip()
if not line:
continue
ip, host = line.split()
hosts_present.add(host)
if host in hosts and ip != host_ip:
raise Exception("Existing hosts file has an ip for %s" % host)
f.seek(f.tell() - 1)
if f.read() != "\n":
f.write("\n")
for host in hosts:
f.write("%s%s%s\n" % (host_ip, " " * (28 - len(host_ip)), host))
self.logger.info("Installing hosts file")
self.device.remount()
self.device.removeFile(remote_path)
self.device.pushFile(hosts_path, remote_path)
finally:
os.unlink(hosts_path)
os.rmdir(temp_dir)
def __init__(self, testinfo, actions={}, docroot=None, request_log_file=None,
**kwargs):
super(WebTest, self).__init__(testinfo, track_start_frame = True,
track_end_frame = True, **kwargs)
self.actions = actions
self.request_log_file = request_log_file
self.capture_server = CaptureServer(self)
self.host = moznetwork.get_ip()
self.http = mozhttpd.MozHttpd(docroot=docroot,
host=self.host, port=0,
log_requests=bool(request_log_file),
urlhandlers = [
{ 'method': 'GET',
'path': '/api/captures/start/?',
'function': self.capture_server.start_capture },
{ 'method': 'GET',
'path': '/api/captures/end/?',
'function': self.capture_server.end_capture },
{ 'method': 'POST',
'path': '/api/captures/input/?',
'function': self.capture_server.input } ])
self.http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries+=1
import socket
s = socket.socket()
try:
s.connect((self.host, self.http.httpd.server_port))
connected = True
except Exception:
self.log("Can't connect to %s:%s, retrying..." % (
self.host, self.http.httpd.server_port))
self.log("Test URL is: %s" % self.url)
if not connected:
raise "Could not open webserver. Error!"
def check_server(device):
logger.info("Checking access to host machine")
if _adbflag:
return True
routes = [("GET", "/", test_handler)]
host_ip = moznetwork.get_ip()
for port in [8000, 8001]:
try:
server = wptserve.WebTestHttpd(host=host_ip,
port=port,
routes=routes)
server.start()
except:
logger.critical("Error starting local server on port %s:\n%s" %
(port, traceback.format_exc()))
return False
try:
device.shell_output("curl http://%s:%i" % (host_ip, port))
except mozdevice.ADBError as e:
if 'curl: not found' in e.message:
logger.warning(
"Could not check access to host machine: curl not present."
)
logger.warning(
"If timeouts occur, check your network configuration.")
break
logger.critical(
"Failed to connect to server running on host machine ip %s port %i. Check network configuration."
% (host_ip, port))
return False
finally:
logger.debug("Stopping server")
server.stop()
return True
def run_capture(options, capture_filename):
device_prefs = eideticker.getDevicePrefs(options)
capture_server = CaptureServer(capture_filename, options)
host = moznetwork.get_ip()
docroot = eideticker.test.TEST_DIR
httpd = mozhttpd.MozHttpd(port=0, host=host, docroot=docroot, urlhandlers=[
{'method': 'GET',
'path': '/api/captures/start/?',
'function': capture_server.start_capture},
{'method': 'GET',
'path': '/api/captures/end/?',
'function': capture_server.end_capture}])
httpd.start(block=False)
print "Serving '%s' at %s:%s" % (
httpd.docroot, httpd.host, httpd.httpd.server_port)
device = eideticker.getDevice(**device_prefs)
url = "http://%s:%s/getdimensions.html" % (host, httpd.httpd.server_port)
device.executeCommand("tap", [100, 100])
if device_prefs['devicetype'] == 'android':
device.launchFennec(options.appname, url=url)
else:
if not options.wifi_settings_file:
print "WIFI settings file (see --help) required for B2G!"
sys.exit(1)
device.restartB2G()
device.connectWIFI(json.loads(open(options.wifi_settings_file).read()))
device.marionette.execute_script("window.location.href='%s';" % url)
while not capture_server.finished:
time.sleep(0.25)
capture_server.convert_capture()
httpd.stop()
def verifyRemoteOptions(self, options, auto):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
if not options.remoteTestRoot:
options.remoteTestRoot = auto._devicemanager.deviceRoot + "/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + auto._product
if options.utilityPath is None:
options.utilityPath = productRoot + "/bin"
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
print "ERROR: you must specify a --remote-webserver=<ip address>\n"
return None
options.webServer = options.remoteWebServer
if not options.httpPort:
options.httpPort = auto.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = auto.DEFAULT_SSL_PORT
if options.geckoPath and not options.emulator:
self.error(
"You must specify --emulator if you specify --gecko-path")
if options.logdir and not options.emulator:
self.error("You must specify --emulator if you specify --logdir")
#if not options.emulator and not options.deviceIP:
# print "ERROR: you must provide a device IP"
# return None
if options.remoteLogFile == None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath == None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
objdir_xpi_stage = os.path.join(build_obj.distdir, 'xpi-stage')
if os.path.isdir(objdir_xpi_stage):
options.extensionsToInstall = [
os.path.join(objdir_xpi_stage, 'mochijar'),
os.path.join(objdir_xpi_stage, 'specialpowers'),
]
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.app is None:
if build_obj:
options.app = build_obj.substs['ANDROID_PACKAGE_NAME']
else:
parser.error("You must specify either appPath or app")
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if build_obj:
options.topsrcdir = build_obj.topsrcdir
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
if not options.robocopApk and build_obj:
apk = build_obj.substs.get('GRADLE_ANDROID_APP_ANDROIDTEST_APK')
if apk and os.path.exists(apk):
options.robocopApk = apk
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
parser.error(
"Unable to find robocop APK '%s'" %
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
if options.coverage_output_dir and not options.enable_coverage:
parser.error("--coverage-output-dir must be used with --enable-coverage")
if options.enable_coverage:
if not options.autorun:
parser.error(
"--enable-coverage cannot be used with --no-autorun")
if not options.coverage_output_dir:
parser.error(
"--coverage-output-dir must be specified when using --enable-coverage")
parent_dir = os.path.dirname(options.coverage_output_dir)
if not os.path.isdir(options.coverage_output_dir):
parser.error(
"The directory for the coverage output does not exist: %s" %
parent_dir)
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app
return options
def verifyRemoteOptions(self, options, automation):
options_logger = logging.getLogger('MochitestRemote')
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
options_logger.error("you must specify a --remote-webserver=<ip address>")
return None
options.webServer = options.remoteWebServer
if (options.deviceIP == None):
options_logger.error("you must provide a device IP")
return None
if (options.remoteLogFile == None):
options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
if (options.remoteLogFile.count('/') < 1):
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# remoteAppPath or app must be specified to find the product to launch
if (options.remoteAppPath and options.app):
options_logger.error("You cannot specify both the remoteAppPath and the app setting")
return None
elif (options.remoteAppPath):
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif (options.app == None):
# Neither remoteAppPath nor app are set -- error
options_logger.error("You must specify either appPath or app")
return None
# Only reset the xrePath if it wasn't provided
if (options.xrePath == None):
options.xrePath = options.utilityPath
if (options.pidFile != ""):
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# Robocop specific deprecated options.
if options.robocop:
if options.robocopIni:
options_logger.error("can not use deprecated --robocop and replacement --robocop-ini together")
return None
options.robocopIni = options.robocop
del options.robocop
if options.robocopPath:
if options.robocopApk:
options_logger.error("can not use deprecated --robocop-path and replacement --robocop-apk together")
return None
options.robocopApk = os.path.join(options.robocopPath, 'robocop.apk')
del options.robocopPath
# Robocop specific options
if options.robocopIni != "":
if not os.path.exists(options.robocopIni):
options_logger.error("Unable to find specified robocop .ini manifest '%s'" % options.robocopIni)
return None
options.robocopIni = os.path.abspath(options.robocopIni)
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
options_logger.error("Unable to find robocop APK '%s'" % options.robocopApk)
return None
options.robocopApk = os.path.abspath(options.robocopApk)
if options.robocopIds != "":
if not os.path.exists(options.robocopIds):
options_logger.error("Unable to find specified robocop IDs file '%s'" % options.robocopIds)
return None
options.robocopIds = os.path.abspath(options.robocopIds)
# allow us to keep original application around for cleanup while running robocop via 'am'
options.remoteappname = options.app
return options
def env_options():
# The server host is set to public localhost IP so that resources can be accessed
# from Android emulator
return {"server_host": moznetwork.get_ip(),
"bind_address": False,
"supports_debugger": True}
def synchronizeTime(self):
self.shellCheckOutput([
self.DEFAULT_NTPDATE_LOCATION, "-c", "1", "-d", "-h",
moznetwork.get_ip(), "-s"
],
root=True)
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
device_args = {'deviceRoot': options.remoteTestRoot}
if options.dm_trans == "adb":
device_args['adbPath'] = options.adbPath
if options.deviceIP:
device_args['host'] = options.deviceIP
device_args['port'] = options.devicePort
elif options.deviceSerial:
device_args['deviceSerial'] = options.deviceSerial
options.dm = DroidADB(**device_args)
elif options.dm_trans == 'sut':
if options.deviceIP is None:
parser.error(
"If --dm_trans = sut, you must provide a device IP")
device_args['host'] = options.deviceIP
device_args['port'] = options.devicePort
options.dm = DroidSUT(**device_args)
if not options.remoteTestRoot:
options.remoteTestRoot = options.dm.deviceRoot
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.remoteLogFile is None:
options.remoteLogFile = options.remoteTestRoot + \
'/logs/mochitest.log'
if options.remoteLogFile.count('/') < 1:
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
if options.remoteAppPath and options.app:
parser.error(
"You cannot specify both the remoteAppPath and the app setting")
elif options.remoteAppPath:
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif options.app is None:
if build_obj:
options.app = build_obj.substs['ANDROID_PACKAGE_NAME']
else:
# Neither remoteAppPath nor app are set -- error
parser.error("You must specify either appPath or app")
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# Robocop specific options
if options.robocopIni != "":
if not os.path.exists(options.robocopIni):
parser.error(
"Unable to find specified robocop .ini manifest '%s'" %
options.robocopIni)
options.robocopIni = os.path.abspath(options.robocopIni)
if not options.robocopApk and build_obj:
options.robocopApk = os.path.join(build_obj.topobjdir, 'mobile', 'android',
'tests', 'browser',
'robocop', 'robocop-debug.apk')
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
parser.error(
"Unable to find robocop APK '%s'" %
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
# Disable e10s by default on Android because we don't run Android
# e10s jobs anywhere yet.
options.e10s = False
mozinfo.update({'e10s': options.e10s})
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app
return options
def cli(args):
global results
tests_ran = False
parser = argparse.ArgumentParser()
parser.add_argument('--firefox-path', help='path to firefox binary',
default=None)
parser.add_argument('--use-b2g', action='store_true',
help='Use marionette to run tests on firefox os')
parser.add_argument('--run-android-browser', action='store_true',
help='Run benchmarks on stock Android browser')
parser.add_argument('--run-dolphin', action='store_true',
help='Run benchmarks on Dolphin browser')
parser.add_argument('--chrome-path', help='path to chrome executable',
default=None)
parser.add_argument('--post-results', action='store_true',
help='if specified, post results to datazilla')
parser.add_argument('--device-serial',
help='serial number of the android or b2g device',
default=None)
parser.add_argument('--run-benchmarks',
help='specify which benchmarks to run')
parser.add_argument('--smoketest', action='store_true',
help='only run smoketest')
parser.add_argument('--json-result', help='store pure json result to file',
default=None)
parser.add_argument('--test-host',
help='network interface on which to listen and serve',
default=moznetwork.get_ip())
parser.add_argument('--test-port',
help='port to host http server',
default=None)
commandline.add_logging_group(parser)
args = parser.parse_args(args)
logging.basicConfig()
logger = commandline.setup_logging('mozbench', vars(args), {})
if not args.use_b2g and not args.firefox_path:
logger.error('you must specify one of --use-b2g or ' +
'--firefox-path')
return 1
if args.firefox_path:
use_android = args.firefox_path.endswith('.apk')
else:
use_android = False
if use_android:
logger.info('prepare for installing fennec')
fennec_pkg_name = get_fennec_pkg_name(args.firefox_path)
success = install_fennec(logger, args.firefox_path, fennec_pkg_name,
args.device_serial)
if not success:
logger.error('fennec installation fail')
return 1
logger.info('fennec installation succeed')
else:
if args.run_android_browser:
logger.warning('stock Android browser only supported on Android')
if args.run_dolphin:
logger.warning('dolphin browser only supported on Android')
logger.info('starting webserver on %s' % args.test_host)
routes = [('POST', '/results', results_handler),
('GET', '/*', wptserve.handlers.file_handler)]
static_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
'static'))
# start http server and request handler
httpd = None
if args.test_port:
try:
port = int(args.test_port)
httpd = wptserve.server.WebTestHttpd(host=args.test_host, port=port,
routes=routes, doc_root=static_path)
except Exception as e:
logger.error(e.message)
return 1
else:
while httpd is None:
try:
port = 10000 + random.randrange(0, 50000)
httpd = wptserve.server.WebTestHttpd(host=args.test_host, port=port,
routes=routes, doc_root=static_path)
# pass if port number has been used, then try another one
except socket.error as e:
pass
except Exception as e:
logger.error(e.message)
return 1
httpd.start()
httpd_logger = logging.getLogger("wptserve")
httpd_logger.setLevel(logging.ERROR)
logger.info('starting webserver on %s:%s' %(httpd.host, str(httpd.port)))
url_prefix = 'http://' + httpd.host + ':' + str(httpd.port) + '/'
result_recorder = ResultRecorder()
with open(os.path.join(os.path.dirname(__file__), 'benchmarks.json')) as f:
benchmarks = json.load(f)
# Determine platform
platform = mozinfo.os
os_version = mozinfo.version
processor = mozinfo.processor
if args.use_b2g:
platform = 'b2g'
elif use_android:
platform = 'android'
device = mozdevice.ADBAndroid(args.device_serial)
os_version = device.get_prop('ro.build.version.release')
processor = device.get_prop('ro.product.cpu.abi')
result_recorder.platform= platform
result_recorder.os_version = os_version
result_recorder.processor = processor
for benchmark in benchmarks:
suite = benchmark['suite']
url = url_prefix + benchmark['url']
num_runs = benchmark['number_of_runs']
timeout = benchmark['timeout']
name = benchmark['name']
value = benchmark['value']
if args.smoketest and suite != 'smoketest':
continue
# Check if benchmark is enabled for platform
if args.run_benchmarks:
if not suite in args.run_benchmarks.strip().split(','):
continue
elif not ('all' in benchmark['enabled'] or
platform in benchmark['enabled']):
logger.info('skipping disabled benchmark: %s for platform %s' %
(suite, platform))
continue
logger.info('starting benchmark: %s' % suite)
result_recorder.set_browser('firefox.nightly')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
# Run firefox
for i in xrange(0, num_runs):
logger.info('firefox run %d' % i)
if args.use_b2g:
runner = B2GRunner(cmdargs=[url], device_serial=args.device_serial)
elif use_android:
runner = AndroidRunner(app_name=fennec_pkg_name,
activity_name='.App',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
else:
runner = mozrunner.FirefoxRunner(binary=args.firefox_path,
cmdargs=[url])
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('firefox results: %s' % json.dumps(results))
# Run chrome (if desired)
if args.chrome_path is not None:
result_recorder.set_browser('chrome.canary')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('chrome run %d' % i)
if use_android:
runner = AndroidRunner(app_name=args.chrome_path,
activity_name='com.google.android.apps.chrome.Main',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
else:
runner = ChromeRunner(binary=args.chrome_path, cmdargs=[url])
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('chrome results: %s' % json.dumps(results))
# Run stock AOSP browser (if desired)
if use_android and args.run_android_browser:
result_recorder.set_browser('android-browser')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('android browser run %d' % i)
runner = AndroidRunner(app_name='com.android.browser',
activity_name='.BrowserActivity',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('android browser results: %s' %
json.dumps(results))
# Run Dolphin browser (if desired)
if use_android and args.run_dolphin:
result_recorder.set_browser('dolphin')
result_recorder.set_benchmark(suite)
result_recorder.set_result_name(name)
result_recorder.set_result_value_name(value)
for i in xrange(0, num_runs):
logger.info('dolphin run %d' % i)
runner = AndroidRunner(app_name='mobi.mgeek.TunnyBrowser',
activity_name='.BrowserActivity',
intent='android.intent.action.VIEW',
url=url,
device_serial=args.device_serial)
version, results = runtest(logger, runner, timeout)
result_recorder.set_browser_version(version)
if results is None:
logger.error('no results found')
else:
tests_ran = True
result_recorder.add_results(results)
logger.info('dolphin results: %s' % json.dumps(results))
if suite == 'smoketest' and not tests_ran:
logger.error('smoketest failed to produce results - skipping '
'remaining tests')
break
if args.post_results:
logger.info('posting results...')
postresults(logger, result_recorder.get_influxdb_results())
if args.json_result:
with open(args.json_result, 'w') as outputFile:
outputFile.write(json.dumps(result_recorder.get_results()) + '\n')
# Only flag the job as failed if no tests ran at all
return 0 if tests_ran else 1
def __init__(self,
testinfo,
options,
device,
capture_controller,
use_actions=True):
Test.__init__(self,
testinfo,
options,
device,
capture_controller,
track_start_frame=True,
track_end_frame=True)
# get actions for web tests
actions_path = os.path.join(testinfo['here'], "actions.json")
if use_actions:
if os.path.exists(actions_path):
self.actions = json.loads(open(actions_path).read())
else:
raise TestException(
"Test needs actions but no actions file '%s'" %
actions_path)
else:
self.actions = None
self.capture_server = CaptureServer(self)
self.host = moznetwork.get_ip()
self.http = mozhttpd.MozHttpd(docroot=TEST_DIR,
host=self.host,
port=0,
log_requests=True,
urlhandlers=[{
'method':
'GET',
'path':
'/api/captures/start/?',
'function':
self.capture_server.start_capture
}, {
'method':
'GET',
'path':
'/api/captures/end/?',
'function':
self.capture_server.end_capture
}, {
'method':
'POST',
'path':
'/api/captures/input/?',
'function':
self.capture_server.input
}])
self.http.start(block=False)
connected = False
tries = 0
while not connected and tries < 20:
tries += 1
import socket
s = socket.socket()
try:
s.connect((self.host, self.http.httpd.server_port))
connected = True
except Exception:
self.logger.info("Can't connect to %s:%s, retrying..." %
(self.host, self.http.httpd.server_port))
self.logger.info("Test URL is: %s" % self.url)
if not connected:
raise "Could not open webserver. Error!"
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
device_args = {'deviceRoot': options.remoteTestRoot}
device_args['adbPath'] = options.adbPath
if options.deviceIP:
device_args['host'] = options.deviceIP
device_args['port'] = options.devicePort
elif options.deviceSerial:
device_args['deviceSerial'] = options.deviceSerial
if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
device_args['logLevel'] = logging.DEBUG
options.dm = DroidADB(**device_args)
if not options.remoteTestRoot:
options.remoteTestRoot = options.dm.deviceRoot
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.remoteLogFile is None:
options.remoteLogFile = options.remoteTestRoot + \
'/logs/mochitest.log'
if options.remoteLogFile.count('/') < 1:
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
if options.remoteAppPath and options.app:
parser.error(
"You cannot specify both the remoteAppPath and the app setting")
elif options.remoteAppPath:
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif options.app is None:
if build_obj:
options.app = build_obj.substs['ANDROID_PACKAGE_NAME']
else:
# Neither remoteAppPath nor app are set -- error
parser.error("You must specify either appPath or app")
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if build_obj:
options.topsrcdir = build_obj.topsrcdir
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# Robocop specific options
if options.robocopIni != "":
if not os.path.exists(options.robocopIni):
parser.error(
"Unable to find specified robocop .ini manifest '%s'" %
options.robocopIni)
options.robocopIni = os.path.abspath(options.robocopIni)
if not options.robocopApk and build_obj:
if build_obj.substs.get('MOZ_BUILD_MOBILE_ANDROID_WITH_GRADLE'):
options.robocopApk = os.path.join(build_obj.topobjdir, 'gradle', 'build',
'mobile', 'android', 'app', 'outputs', 'apk',
'app-official-australis-debug-androidTest-'
'unaligned.apk')
else:
options.robocopApk = os.path.join(build_obj.topobjdir, 'mobile', 'android',
'tests', 'browser',
'robocop', 'robocop-debug.apk')
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
parser.error(
"Unable to find robocop APK '%s'" %
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
# Disable e10s by default on Android because we don't run Android
# e10s jobs anywhere yet.
options.e10s = False
mozinfo.update({'e10s': options.e10s})
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app
return options
def start_fixture_servers(self):
root = self.server_root or os.path.join(os.path.dirname(here), "www")
if self.appName == "fennec":
return serve.start(root, host=moznetwork.get_ip())
else:
return serve.start(root)
def validate(self, parser, options, context):
"""Validate android options."""
if build_obj:
options.log_mach = '-'
if options.dm_trans == "adb":
if options.deviceIP:
options.dm = DroidADB(
options.deviceIP,
options.devicePort,
deviceRoot=options.remoteTestRoot)
elif options.deviceSerial:
options.dm = DroidADB(
None,
None,
deviceSerial=options.deviceSerial,
deviceRoot=options.remoteTestRoot)
else:
options.dm = DroidADB(deviceRoot=options.remoteTestRoot)
elif options.dm_trans == 'sut':
if options.deviceIP is None:
parser.error(
"If --dm_trans = sut, you must provide a device IP")
options.dm = DroidSUT(
options.deviceIP,
options.devicePort,
deviceRoot=options.remoteTestRoot)
if not options.remoteTestRoot:
options.remoteTestRoot = options.dm.deviceRoot
if options.remoteWebServer is None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
parser.error(
"you must specify a --remote-webserver=<ip address>")
options.webServer = options.remoteWebServer
if options.remoteLogFile is None:
options.remoteLogFile = options.remoteTestRoot + \
'/logs/mochitest.log'
if options.remoteLogFile.count('/') < 1:
options.remoteLogFile = options.remoteTestRoot + \
'/' + options.remoteLogFile
if options.remoteAppPath and options.app:
parser.error(
"You cannot specify both the remoteAppPath and the app setting")
elif options.remoteAppPath:
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif options.app is None:
if build_obj:
options.app = build_obj.substs['ANDROID_PACKAGE_NAME']
else:
# Neither remoteAppPath nor app are set -- error
parser.error("You must specify either appPath or app")
if build_obj and 'MOZ_HOST_BIN' in os.environ:
options.xrePath = os.environ['MOZ_HOST_BIN']
# Only reset the xrePath if it wasn't provided
if options.xrePath is None:
options.xrePath = options.utilityPath
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# Robocop specific options
if options.robocopIni != "":
if not os.path.exists(options.robocopIni):
parser.error(
"Unable to find specified robocop .ini manifest '%s'" %
options.robocopIni)
options.robocopIni = os.path.abspath(options.robocopIni)
if not options.robocopApk and build_obj:
options.robocopApk = os.path.join(build_obj.topobjdir, 'build', 'mobile',
'robocop', 'robocop-debug.apk')
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
parser.error(
"Unable to find robocop APK '%s'" %
options.robocopApk)
options.robocopApk = os.path.abspath(options.robocopApk)
if options.robocopIds != "":
if not os.path.exists(options.robocopIds):
parser.error(
"Unable to find specified robocop IDs file '%s'" %
options.robocopIds)
options.robocopIds = os.path.abspath(options.robocopIds)
# allow us to keep original application around for cleanup while
# running robocop via 'am'
options.remoteappname = options.app
return options
def verifyRemoteOptions(self, options, automation):
options_logger = logging.getLogger('MochitestRemote')
if not options.remoteTestRoot:
options.remoteTestRoot = automation._devicemanager.deviceRoot
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
options_logger.error("you must specify a --remote-webserver=<ip address>")
return None
options.webServer = options.remoteWebServer
if (options.deviceIP == None):
options_logger.error("you must provide a device IP")
return None
if (options.remoteLogFile == None):
options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
if (options.remoteLogFile.count('/') < 1):
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# remoteAppPath or app must be specified to find the product to launch
if (options.remoteAppPath and options.app):
options_logger.error("You cannot specify both the remoteAppPath and the app setting")
return None
elif (options.remoteAppPath):
options.app = options.remoteTestRoot + "/" + options.remoteAppPath
elif (options.app == None):
# Neither remoteAppPath nor app are set -- error
options_logger.error("You must specify either appPath or app")
return None
# Only reset the xrePath if it wasn't provided
if (options.xrePath == None):
options.xrePath = options.utilityPath
if (options.pidFile != ""):
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# Robocop specific deprecated options.
if options.robocop:
if options.robocopIni:
options_logger.error("can not use deprecated --robocop and replacement --robocop-ini together")
return None
options.robocopIni = options.robocop
del options.robocop
if options.robocopPath:
if options.robocopApk:
options_logger.error("can not use deprecated --robocop-path and replacement --robocop-apk together")
return None
options.robocopApk = os.path.join(options.robocopPath, 'robocop.apk')
del options.robocopPath
# Robocop specific options
if options.robocopIni != "":
if not os.path.exists(options.robocopIni):
options_logger.error("Unable to find specified robocop .ini manifest '%s'" % options.robocopIni)
return None
options.robocopIni = os.path.abspath(options.robocopIni)
if options.robocopApk != "":
if not os.path.exists(options.robocopApk):
options_logger.error("Unable to find robocop APK '%s'" % options.robocopApk)
return None
options.robocopApk = os.path.abspath(options.robocopApk)
if options.robocopIds != "":
if not os.path.exists(options.robocopIds):
options_logger.error("Unable to find specified robocop IDs file '%s'" % options.robocopIds)
return None
options.robocopIds = os.path.abspath(options.robocopIds)
# allow us to keep original application around for cleanup while running robocop via 'am'
options.remoteappname = options.app
return options
def __init__(self, automation):
ReftestOptions.__init__(self)
self.automation = automation
defaults = {}
defaults["logFile"] = "reftest.log"
# app, xrePath and utilityPath variables are set in main function
defaults["app"] = ""
defaults["xrePath"] = ""
defaults["utilityPath"] = ""
defaults["runTestsInParallel"] = False
self.add_option(
"--remote-app-path",
action="store",
type="string",
dest="remoteAppPath",
help=
"Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both."
)
defaults["remoteAppPath"] = None
self.add_option("--deviceIP",
action="store",
type="string",
dest="deviceIP",
help="ip address of remote device to test")
defaults["deviceIP"] = None
self.add_option("--deviceSerial",
action="store",
type="string",
dest="deviceSerial",
help="adb serial number of remote device to test")
defaults["deviceSerial"] = None
self.add_option("--devicePort",
action="store",
type="string",
dest="devicePort",
help="port of remote device to test")
defaults["devicePort"] = 20701
self.add_option(
"--remote-product-name",
action="store",
type="string",
dest="remoteProductName",
help=
"Name of product to test - either fennec or firefox, defaults to fennec"
)
defaults["remoteProductName"] = "fennec"
self.add_option(
"--remote-webserver",
action="store",
type="string",
dest="remoteWebServer",
help="IP Address of the webserver hosting the reftest content")
defaults["remoteWebServer"] = moznetwork.get_ip()
self.add_option("--http-port",
action="store",
type="string",
dest="httpPort",
help="port of the web server for http traffic")
defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
self.add_option("--ssl-port",
action="store",
type="string",
dest="sslPort",
help="Port for https traffic to the web server")
defaults["sslPort"] = automation.DEFAULT_SSL_PORT
self.add_option(
"--remote-logfile",
action="store",
type="string",
dest="remoteLogFile",
help=
"Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME."
)
defaults["remoteLogFile"] = None
self.add_option("--pidfile",
action="store",
type="string",
dest="pidFile",
help="name of the pidfile to generate")
defaults["pidFile"] = ""
self.add_option(
"--bootstrap",
action="store_true",
dest="bootstrap",
help="test with a bootstrap addon required for native Fennec")
defaults["bootstrap"] = False
self.add_option(
"--dm_trans",
action="store",
type="string",
dest="dm_trans",
help=
"the transport to use to communicate with device: [adb|sut]; default=sut"
)
defaults["dm_trans"] = "sut"
self.add_option(
"--remoteTestRoot",
action="store",
type="string",
dest="remoteTestRoot",
help=
"remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)"
)
defaults["remoteTestRoot"] = None
self.add_option("--httpd-path",
action="store",
type="string",
dest="httpdPath",
help="path to the httpd.js file")
defaults["httpdPath"] = None
defaults["localLogName"] = None
self.set_defaults(**defaults)
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer',
'permissions',
'webapi',
'user-agent',
'crash-reporter',
'search-id',
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
test_groups = set(args.include if args.include else test_groups)
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True)
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
dm.forward("tcp:2828", "tcp:2828")
retries = 0
while retries < 5:
try:
m = marionette.Marionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
if args.version not in supported_versions:
print "%s is not a valid version. Please enter one of %s" % \
(args.version, supported_versions)
raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (result_file_path, e.errno)
raise
# get build properties
buildpropoutput = dm.shellCheckOutput(["cat", "/system/build.prop"])
for buildprop in [line for line in buildpropoutput.splitlines() if '=' \
in line]:
eq = buildprop.find('=')
prop = buildprop[:eq]
val = buildprop[eq + 1:]
report['buildprops'][prop] = val
# get process list
report['processes_running'] = map(lambda p: { 'name': p[1], 'user': p[2] },
dm.getProcessList())
# kernel version
report['kernel_version'] = dm.shellCheckOutput(["cat", "/proc/version"])
# application.ini information
appinicontents = dm.pullFile('/system/b2g/application.ini')
sf = StringIO.StringIO(appinicontents)
config = ConfigParser.ConfigParser()
config.readfp(sf)
report['application_ini'] = {}
for section in config.sections():
report['application_ini'][section] = dict(config.items(section))
logger.suite_start(tests=[])
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
omni_ref_path = pkg_resources.resource_filename(
__name__, os.path.join('expected_omni_results', 'omni.ja.%s' % args.version))
omni_analyzer = OmniAnalyzer(omni_ref_path, logger=logger)
diff = omni_analyzer.run()
report["omni_result"] = diff
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(
host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
errors = False
logger.test_start('webapi')
logger.debug('Running webapi verifier tests')
for apptype in ['web', 'privileged', 'certified']:
global webapi_results
webapi_results = None
appname = '%s WebAPI Verifier' % apptype.capitalize()
apppath = os.path.join(static_path, 'webapi-test-app')
install_app(logger, appname, args.version, apptype, apppath, True,
{'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";' % (addr * 2)},
True)
try:
wait.Wait(timeout=120).until(lambda: webapi_results is not None)
except wait.TimeoutException:
logger.error('Timed out waiting for results for test: %s' % last_test_started)
errors = True
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
if webapi_results is None:
continue
if "headers" not in report:
report["headers"] = headers
results_filename = '%s.%s.json' % (args.version, apptype)
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(webapi_results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(
__name__, os.path.sep.join(['expected_webapi_results', results_filename]))
parse_webapi_results(file_path, webapi_results, '%s-' % apptype, logger, report)
logger.debug('Done.')
if errors:
logger.test_end('webapi', 'ERROR')
else:
logger.test_end('webapi', 'OK')
if 'permissions' in test_groups:
errors = False
logger.test_start('permissions')
logger.debug('Running permissions tests')
permissions = get_permissions()
# test default permissions
for apptype in ['web', 'privileged', 'certified']:
logger.debug('Testing default permissions: %s' % apptype)
results = {}
expected_webapi_results = None
appname = 'Default Permissions Test App'
fxos_appgen.uninstall_app(appname)
installed_appname = appname.lower().replace(" ", "-")
fxos_appgen.generate_app(appname, install=True, app_type=apptype,
all_perm=True)
for permission in permissions:
result = get_permission(permission, installed_appname)
results[permission] = result
results_filename = '%s.%s.json' % (args.version, apptype)
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join(['expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, '%s-' % apptype,
logger, report)
fxos_appgen.uninstall_app(appname)
# test individual permissions
logger.debug('Testing individual permissions')
results = {}
# first install test app for embed-apps permission test
embed_appname = 'Embed Apps Test App'
apppath = os.path.join(static_path, 'embed-apps-test-app')
install_app(logger, embed_appname, args.version, 'certified', apppath, True,
{'results_uri.js': 'RESULTS_URI="http://%s:%s/webapi_results_embed_apps";' % addr},
False)
appname = 'Permissions Test App'
installed_appname = appname.lower().replace(" ", "-")
apppath = os.path.join(static_path, 'permissions-test-app')
install_app(logger, appname, args.version, 'web', apppath, False,
{'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";' % (addr * 2)})
for permission in [None] + permissions:
webapi_results = None
webapi_results_embed_app = None
# if we try to launch after killing too quickly, the app seems
# to not fully launch
time.sleep(5)
if permission is not None:
logger.debug('testing permission: %s' % permission)
set_permission(permission, u'allow', installed_appname)
fxos_appgen.launch_app(appname)
try:
wait.Wait(timeout=60).until(lambda: webapi_results is not None)
# embed-apps results are posted to a separate URL
if webapi_results_embed_app:
webapi_results['embed-apps'] = webapi_results_embed_app['embed-apps']
else:
webapi_results['embed-apps'] = False
if permission is None:
expected_webapi_results = webapi_results
else:
results[permission] = diff_results(expected_webapi_results, webapi_results)
except wait.TimeoutException:
logger.error('Timed out waiting for results')
errors = True
if permission is not None:
results[permission] = 'timed out'
else:
# If we timeout on our baseline results there is
# no point in proceeding.
logger.error('Could not get baseline results for permissions. Skipping tests.')
break
kill('app://' + installed_appname)
if permission is not None:
set_permission(permission, u'deny', installed_appname)
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
# we test open-remote-window separately as opening a remote
# window might stop the test app
results['open-remote-window'] = test_open_remote_window(logger,
args.version, addr)
results_filename = '%s.permissions.json' % args.version
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join(['expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, 'individual-',
logger, report)
logger.debug('Done.')
if errors:
logger.test_end('permissions', 'ERROR')
else:
logger.test_end('permissions', 'OK')
# clean up embed-apps test app
logger.debug('uninstalling: %s' % embed_appname)
fxos_appgen.uninstall_app(embed_appname)
if 'user-agent' in test_groups:
logger.test_start('user-agent')
logger.debug('Running user agent tests')
user_agent_string = run_marionette_script("return navigator.userAgent;")
logger.debug('UserAgent: %s' % user_agent_string)
valid = test_user_agent(user_agent_string, logger)
if valid:
logger.test_end('user-agent', 'OK')
else:
logger.test_end('user-agent', 'ERROR')
if 'crash-reporter' in test_groups:
logger.test_start('crash-reporter')
logger.debug('start checking test reporter')
crash_report_toggle = (report.get('application_ini', {})
.get('Crash Reporter', {})
.get('enabled'))
if crash_report_toggle == '1':
logger.test_end('crash-reporter', 'OK')
else:
logger.test_end('crash-reporter', 'ERROR')
if 'search-id' in test_groups:
logger.test_start('search-id')
fxos_appgen.launch_app('browser')
script = """
result = window.wrappedJSObject.UrlHelper.getUrlFromInput('hello world');
return result;
"""
m = marionette.Marionette('localhost', 2828)
m.start_session()
browser = m.find_element('css selector', 'iframe[src="app://search.gaiamobile.org/newtab.html"]')
m.switch_to_frame(browser)
url = m.execute_script(script)
m.delete_session()
report['search-oemid'] = url
oemid_rexp = re.compile('client=mobile-firefoxos&channel=fm:org.mozilla:([A-Z0-9.]+):official&')
match = oemid_rexp.match(url)
if match:
logger.test_status('search-id', 'oemid', 'PASS', message='oemid: %s' % match.groups()[0])
else:
logger.test_status('search-id', 'oemid', 'FAIL', message='no oemid found in url: %s' % url)
logger.test_end('search-id', 'OK')
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
def __init__(self, automation):
ReftestOptions.__init__(self)
self.automation = automation
defaults = {}
defaults["logFile"] = "reftest.log"
# app, xrePath and utilityPath variables are set in main function
defaults["app"] = ""
defaults["xrePath"] = ""
defaults["utilityPath"] = ""
defaults["runTestsInParallel"] = False
self.add_option(
"--remote-app-path",
action="store",
type="string",
dest="remoteAppPath",
help="Path to remote executable relative to device root using only forward slashes. Either this or app must be specified, but not both.",
)
defaults["remoteAppPath"] = None
self.add_option(
"--deviceIP", action="store", type="string", dest="deviceIP", help="ip address of remote device to test"
)
defaults["deviceIP"] = None
self.add_option(
"--deviceSerial",
action="store",
type="string",
dest="deviceSerial",
help="adb serial number of remote device to test",
)
defaults["deviceSerial"] = None
self.add_option(
"--devicePort", action="store", type="string", dest="devicePort", help="port of remote device to test"
)
defaults["devicePort"] = 20701
self.add_option(
"--remote-product-name",
action="store",
type="string",
dest="remoteProductName",
help="Name of product to test - either fennec or firefox, defaults to fennec",
)
defaults["remoteProductName"] = "fennec"
self.add_option(
"--remote-webserver",
action="store",
type="string",
dest="remoteWebServer",
help="IP Address of the webserver hosting the reftest content",
)
defaults["remoteWebServer"] = moznetwork.get_ip()
self.add_option(
"--http-port",
action="store",
type="string",
dest="httpPort",
help="port of the web server for http traffic",
)
defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
self.add_option(
"--ssl-port", action="store", type="string", dest="sslPort", help="Port for https traffic to the web server"
)
defaults["sslPort"] = automation.DEFAULT_SSL_PORT
self.add_option(
"--remote-logfile",
action="store",
type="string",
dest="remoteLogFile",
help="Name of log file on the device relative to device root. PLEASE USE ONLY A FILENAME.",
)
defaults["remoteLogFile"] = None
self.add_option(
"--pidfile", action="store", type="string", dest="pidFile", help="name of the pidfile to generate"
)
defaults["pidFile"] = ""
self.add_option(
"--bootstrap",
action="store_true",
dest="bootstrap",
help="test with a bootstrap addon required for native Fennec",
)
defaults["bootstrap"] = False
self.add_option(
"--dm_trans",
action="store",
type="string",
dest="dm_trans",
help="the transport to use to communicate with device: [adb|sut]; default=sut",
)
defaults["dm_trans"] = "sut"
self.add_option(
"--remoteTestRoot",
action="store",
type="string",
dest="remoteTestRoot",
help="remote directory to use as test root (eg. /mnt/sdcard/tests or /data/local/tests)",
)
defaults["remoteTestRoot"] = None
self.add_option(
"--httpd-path", action="store", type="string", dest="httpdPath", help="path to the httpd.js file"
)
defaults["httpdPath"] = None
defaults["localLogName"] = None
self.set_defaults(**defaults)
def verifyRemoteOptions(self, options, auto):
if options.runTestsInParallel:
self.error("Cannot run parallel tests here")
if not options.remoteTestRoot:
options.remoteTestRoot = auto._devicemanager.getDeviceRoot() + "/reftest"
options.remoteProfile = options.remoteTestRoot + "/profile"
productRoot = options.remoteTestRoot + "/" + auto._product
if options.utilityPath == auto.DIST_BIN:
options.utilityPath = productRoot + "/bin"
if options.remoteWebServer == None:
if os.name != "nt":
options.remoteWebServer = moznetwork.get_ip()
else:
print "ERROR: you must specify a --remote-webserver=<ip address>\n"
return None
options.webServer = options.remoteWebServer
if not options.httpPort:
options.httpPort = auto.DEFAULT_HTTP_PORT
if not options.sslPort:
options.sslPort = auto.DEFAULT_SSL_PORT
if options.geckoPath and not options.emulator:
self.error("You must specify --emulator if you specify --gecko-path")
if options.logcat_dir and not options.emulator:
self.error("You must specify --emulator if you specify --logcat-dir")
#if not options.emulator and not options.deviceIP:
# print "ERROR: you must provide a device IP"
# return None
if options.remoteLogFile == None:
options.remoteLogFile = "reftest.log"
options.localLogName = options.remoteLogFile
options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
# Ensure that the options.logfile (which the base class uses) is set to
# the remote setting when running remote. Also, if the user set the
# log file name there, use that instead of reusing the remotelogfile as above.
if (options.logFile):
# If the user specified a local logfile name use that
options.localLogName = options.logFile
options.logFile = options.remoteLogFile
# Only reset the xrePath if it wasn't provided
if options.xrePath == None:
options.xrePath = options.utilityPath
options.xrePath = os.path.abspath(options.xrePath)
if options.pidFile != "":
f = open(options.pidFile, 'w')
f.write("%s" % os.getpid())
f.close()
# httpd-path is specified by standard makefile targets and may be specified
# on the command line to select a particular version of httpd.js. If not
# specified, try to select the one from from the xre bundle, as required in bug 882932.
if not options.httpdPath:
options.httpdPath = os.path.join(options.xrePath, "components")
return options
def setUp(self):
super(TcpSocketTestCase, self).setUp()
self.server = TCPTestServer((moznetwork.get_ip(), get_free_port()))
self.server.start()