class MarionetteTestRunner(object):
textrunnerclass = MarionetteTextTestRunner
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
app=None, bin=None, profile=None, autolog=False, revision=None,
logger=None, testgroup="marionette", noWindow=False,
logcat_dir=None, xml_output=None, repeat=0, gecko_path=None,
testvars=None, tree=None, type=None, device_serial=None,
symbols_path=None, timeout=None, es_servers=None, **kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device_serial = device_serial
self.symbols_path = symbols_path
self.timeout = timeout
self._device = None
self._capabilities = None
self._appName = None
self.es_servers = es_servers
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
@property
def capabilities(self):
if self._capabilities:
return self._capabilities
self.marionette.start_session()
self._capabilities = self.marionette.session_capabilities
self.marionette.delete_session()
return self._capabilities
@property
def device(self):
if self._device:
return self._device
self._device = self.capabilities.get('device')
return self._device
@property
def appName(self):
if self._appName:
return self._appName
self._appName = self.capabilities.get('browserName')
return self._appName
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl,
timeout=self.timeout)
elif self.address:
host, port = self.address.split(':')
try:
#establish a telnet connection so we can vertify the data come back
tlconnection = Telnet(host, port)
except:
raise Exception("could not connect to given marionette host/port")
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl,
timeout=self.timeout)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
timeout=self.timeout)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
for es_server in self.es_servers:
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup=self.testgroup,
os='android',
platform='emulator',
harness='marionette',
server=es_server,
restserver=None,
machine=socket.gethostname(),
logfile=logfile)
testgroup.set_primary_product(
tree=self.tree,
buildtype='opt',
revision=self.revision)
testgroup.add_test_suite(
testsuite='b2g emulator testsuite',
elapsedtime=elapsedtime.seconds,
cmdline='',
passed=self.passed,
failed=self.failed,
todo=self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >=0:
for test in tests:
self.run_test(test)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
if self.failed > 0:
self.logger.info('\nFAILED TESTS\n-------')
for failed_test in self.failures:
self.logger.info('%s' % failed_test[0])
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_') or filename.startswith('browser_')) and
(filename.endswith('.py') or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name,file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-', ' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({ atype[1:]: 'true' })
elif atype.startswith('-'):
testargs.update({ atype[1:]: 'false' })
else:
testargs.update({ atype: 'true' })
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(disabled=False)
manifest_tests = manifest.active_tests(disabled=False,
device=self.device,
app=self.appName)
skip_tests = list(set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped),
self.device,
self.appName))
self.todo += 1
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name,
filepath,
suite,
testloader,
self.marionette,
self.testvars,
**self.test_kwargs)
break
if suite.countTestCases():
runner = self.textrunnerclass(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1], 'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccesses'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure), 'TEST-UNEXPECTED-PASS'))
if hasattr(results, 'expectedFailures'):
self.passed += len(results.expectedFailures)
def register_handlers(self):
self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
def generate_xml(self, results_list):
def _extract_xml(test, text='', result='passed'):
cls_name = test.__class__.__name__
testcase = doc.createElement('testcase')
testcase.setAttribute('classname', cls_name)
testcase.setAttribute('name', unicode(test).split()[0])
testcase.setAttribute('time', str(test.duration))
testsuite.appendChild(testcase)
if result in ['failure', 'error', 'skipped']:
f = doc.createElement(result)
f.setAttribute('message', 'test %s' % result)
f.appendChild(doc.createTextNode(text))
testcase.appendChild(f)
doc = dom.Document()
testsuite = doc.createElement('testsuite')
testsuite.setAttribute('name', 'Marionette')
testsuite.setAttribute('time', str(self.elapsedtime.total_seconds()))
testsuite.setAttribute('tests', str(sum([results.testsRun for
results in results_list])))
def failed_count(results):
count = len(results.failures)
if hasattr(results, 'unexpectedSuccesses'):
count += len(results.unexpectedSuccesses)
return count
testsuite.setAttribute('failures', str(sum([failed_count(results)
for results in results_list])))
testsuite.setAttribute('errors', str(sum([len(results.errors)
for results in results_list])))
if hasattr(results, 'skipped'):
testsuite.setAttribute('skips', str(sum([len(results.skipped) +
len(results.expectedFailures)
for results in results_list])))
for results in results_list:
for tup in results.errors:
_extract_xml(tup[0], text=tup[1], result='error')
for tup in results.failures:
_extract_xml(tup[0], text=tup[1], result='failure')
if hasattr(results, 'unexpectedSuccesses'):
for test in results.unexpectedSuccesses:
# unexpectedSuccesses is a list of Testcases only, no tuples
_extract_xml(test, text='TEST-UNEXPECTED-PASS', result='failure')
if hasattr(results, 'skipped'):
for tup in results.skipped:
_extract_xml(tup[0], text=tup[1], result='skipped')
if hasattr(results, 'expectedFailures'):
for tup in results.expectedFailures:
_extract_xml(tup[0], text=tup[1], result='skipped')
for test in results.tests_passed:
_extract_xml(test)
doc.appendChild(testsuite)
return doc.toprettyxml(encoding='utf-8')
class MarionetteTestRunner(object):
def __init__(self,
address=None,
emulator=None,
emulatorBinary=None,
emulatorImg=None,
emulator_res='480x800',
homedir=None,
app=None,
bin=None,
profile=None,
autolog=False,
revision=None,
es_server=None,
rest_server=None,
logger=None,
testgroup="marionette",
noWindow=False,
logcat_dir=None,
xml_output=None,
repeat=0,
gecko_path=None,
testvars=None,
tree=None,
type=None,
device=None,
symbols_path=None,
**kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.xml_output = xml_output
self.repeat = repeat
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device = device
self.symbols_path = symbols_path
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__),
'www'))
self.httpd.start()
def start_marionette(self):
assert (self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl)
elif self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(
os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(testgroup=self.testgroup,
os='android',
platform='emulator',
harness='marionette',
server=self.es_server,
restserver=self.rest_server,
machine=socket.gethostname(),
logfile=logfile)
testgroup.set_primary_product(tree=self.tree,
buildtype='opt',
revision=self.revision)
testgroup.add_test_suite(testsuite='b2g emulator testsuite',
elapsedtime=elapsedtime.seconds,
cmdline='',
passed=self.passed,
failed=self.failed,
todo=self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >= 0:
for test in tests:
self.run_test(test)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
def run_test(self, test):
if not self.httpd:
print "starting httpd"
self.start_httpd()
if not self.marionette:
self.start_marionette()
filepath = os.path.abspath(test)
if os.path.isdir(filepath):
for root, dirs, files in os.walk(filepath):
for filename in files:
if ((filename.startswith('test_')
or filename.startswith('browser_'))
and (filename.endswith('.py')
or filename.endswith('.js'))):
filepath = os.path.join(root, filename)
self.run_test(filepath)
if self.marionette.check_for_crash():
return
return
mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])
testloader = unittest.TestLoader()
suite = unittest.TestSuite()
if file_ext == '.ini':
testargs = {}
if self.type is not None:
testtypes = self.type.replace('+', ' +').replace('-',
' -').split()
for atype in testtypes:
if atype.startswith('+'):
testargs.update({atype[1:]: 'true'})
elif atype.startswith('-'):
testargs.update({atype[1:]: 'false'})
else:
testargs.update({atype: 'true'})
manifest = TestManifest()
manifest.read(filepath)
manifest_tests = manifest.active_tests(disabled=False)
for i in manifest.get(tests=manifest_tests, **testargs):
self.run_test(i["path"])
if self.marionette.check_for_crash():
return
return
self.logger.info('TEST-START %s' % os.path.basename(test))
for handler in self.test_handlers:
if handler.match(os.path.basename(test)):
handler.add_tests_to_suite(mod_name, filepath, suite,
testloader, self.marionette,
self.testvars, **self.test_kwargs)
break
if suite.countTestCases():
runner = MarionetteTextTestRunner(verbosity=3,
marionette=self.marionette)
results = runner.run(suite)
self.results.append(results)
self.failed += len(results.failures) + len(results.errors)
if hasattr(results, 'skipped'):
self.todo += len(results.skipped) + len(
results.expectedFailures)
self.passed += results.passed
for failure in results.failures + results.errors:
self.failures.append((results.getInfo(failure[0]), failure[1],
'TEST-UNEXPECTED-FAIL'))
if hasattr(results, 'unexpectedSuccess'):
self.failed += len(results.unexpectedSuccesses)
for failure in results.unexpectedSuccesses:
self.failures.append((results.getInfo(failure[0]),
failure[1], 'TEST-UNEXPECTED-PASS'))
def register_handlers(self):
self.test_handlers.extend([MarionetteTestCase, MarionetteJSTestCase])
def cleanup(self):
if self.httpd:
self.httpd.stop()
__del__ = cleanup
def generate_xml(self, results_list):
def _extract_xml(test, text='', result='passed'):
cls_name = test.__class__.__name__
testcase = doc.createElement('testcase')
testcase.setAttribute('classname', cls_name)
testcase.setAttribute('name', unicode(test).split()[0])
testcase.setAttribute('time', str(test.duration))
testsuite.appendChild(testcase)
if result in ['failure', 'error', 'skipped']:
f = doc.createElement(result)
f.setAttribute('message', 'test %s' % result)
f.appendChild(doc.createTextNode(text))
testcase.appendChild(f)
doc = dom.Document()
testsuite = doc.createElement('testsuite')
testsuite.setAttribute('name', 'Marionette')
testsuite.setAttribute('time', str(self.elapsedtime.total_seconds()))
testsuite.setAttribute(
'tests', str(sum([results.testsRun for results in results_list])))
def failed_count(results):
count = len(results.failures)
if hasattr(results, 'unexpectedSuccesses'):
count += len(results.unexpectedSuccesses)
return count
testsuite.setAttribute(
'failures',
str(sum([failed_count(results) for results in results_list])))
testsuite.setAttribute(
'errors',
str(sum([len(results.errors) for results in results_list])))
if hasattr(results, 'skipped'):
testsuite.setAttribute(
'skips',
str(
sum([
len(results.skipped) + len(results.expectedFailures)
for results in results_list
])))
for results in results_list:
for tup in results.errors:
_extract_xml(*tup, result='error')
for tup in results.failures:
_extract_xml(*tup, result='failure')
if hasattr(results, 'unexpectedSuccesses'):
for test in results.unexpectedSuccesses:
# unexpectedSuccesses is a list of Testcases only, no tuples
_extract_xml(test,
text='TEST-UNEXPECTED-PASS',
result='failure')
if hasattr(results, 'skipped'):
for tup in results.skipped:
_extract_xml(*tup, result='skipped')
if hasattr(results, 'expectedFailures'):
for tup in results.expectedFailures:
_extract_xml(*tup, result='skipped')
for test in results.tests_passed:
_extract_xml(test)
doc.appendChild(testsuite)
return doc.toprettyxml(encoding='utf-8')
class MarionetteTestRunner(object):
def __init__(
self,
address=None,
emulator=None,
emulatorBinary=None,
emulatorImg=None,
emulator_res="480x800",
homedir=None,
app=None,
bin=None,
profile=None,
autolog=False,
revision=None,
es_server=None,
rest_server=None,
logger=None,
testgroup="marionette",
noWindow=False,
logcat_dir=None,
xml_output=None,
repeat=0,
perf=False,
perfserv=None,
gecko_path=None,
testvars=None,
tree=None,
device=None,
symbols_path=None,
):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.perfrequest = None
self.xml_output = xml_output
self.repeat = repeat
self.perf = perf
self.perfserv = perfserv
self.gecko_path = gecko_path
self.testvars = {}
self.tree = tree
self.device = device
self.symbols_path = symbols_path
if testvars:
if not os.path.exists(testvars):
raise Exception("--testvars file does not exist")
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger("Marionette")
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars["xml_output"] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
self.perfrequest = None
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("", 0))
port = s.getsockname()[1]
s.close()
self.baseurl = "http://%s:%d/" % (host, port)
self.logger.info("running webserver on %s" % self.baseurl)
self.httpd = MozHttpd(host=host, port=port, docroot=os.path.join(os.path.dirname(__file__), "www"))
self.httpd.start()
def start_marionette(self):
assert self.baseurl is not None
if self.bin:
if self.address:
host, port = self.address.split(":")
else:
host = "localhost"
port = 2828
self.marionette = Marionette(
host=host, port=int(port), app=self.app, bin=self.bin, profile=self.profile, baseurl=self.baseurl
)
elif self.address:
host, port = self.address.split(":")
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host,
port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
)
else:
self.marionette = Marionette(host=host, port=int(port), baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path,
)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info("posting results to autolog")
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir), "emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup=self.testgroup,
os="android",
platform="emulator",
harness="marionette",
server=self.es_server,
restserver=self.rest_server,
machine=socket.gethostname(),
logfile=logfile,
)
testgroup.set_primary_product(tree=self.tree, buildtype="opt", revision=self.revision)
testgroup.add_test_suite(
testsuite="b2g emulator testsuite",
elapsedtime=elapsedtime.seconds,
cmdline="",
passed=self.passed,
failed=self.failed,
todo=self.todo,
)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests, testtype=None):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >= 0:
for test in tests:
self.run_test(test, testtype)
self.repeat -= 1
self.logger.info("\nSUMMARY\n-------")
self.logger.info("passed: %d" % self.passed)
self.logger.info("failed: %d" % self.failed)
self.logger.info("todo: %d" % self.todo)
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.perfrequest and options.perf:
try:
self.perfrequest.submit()
except Exception, e:
print "Could not submit to datazilla"
print e
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, "w") as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
class MarionetteTestRunner(object):
def __init__(self, address=None, emulator=None, emulatorBinary=None,
emulatorImg=None, emulator_res='480x800', homedir=None,
app=None, bin=None, profile=None, autolog=False, revision=None,
es_server=None, rest_server=None, logger=None,
testgroup="marionette", noWindow=False, logcat_dir=None,
xml_output=None, repeat=0, perf=False, perfserv=None,
gecko_path=None, testvars=None, tree=None, type=None,
device=None, symbols_path=None, **kwargs):
self.address = address
self.emulator = emulator
self.emulatorBinary = emulatorBinary
self.emulatorImg = emulatorImg
self.emulator_res = emulator_res
self.homedir = homedir
self.app = app
self.bin = bin
self.profile = profile
self.autolog = autolog
self.testgroup = testgroup
self.revision = revision
self.es_server = es_server
self.rest_server = rest_server
self.logger = logger
self.noWindow = noWindow
self.httpd = None
self.baseurl = None
self.marionette = None
self.logcat_dir = logcat_dir
self.perfrequest = None
self.xml_output = xml_output
self.repeat = repeat
self.perf = perf
self.perfserv = perfserv
self.gecko_path = gecko_path
self.testvars = {}
self.test_kwargs = kwargs
self.tree = tree
self.type = type
self.device = device
self.symbols_path = symbols_path
if testvars:
if not os.path.exists(testvars):
raise Exception('--testvars file does not exist')
import json
with open(testvars) as f:
self.testvars = json.loads(f.read())
# set up test handlers
self.test_handlers = []
self.register_handlers()
self.reset_test_stats()
if self.logger is None:
self.logger = logging.getLogger('Marionette')
self.logger.setLevel(logging.INFO)
self.logger.addHandler(logging.StreamHandler())
if self.logcat_dir:
if not os.access(self.logcat_dir, os.F_OK):
os.mkdir(self.logcat_dir)
# for XML output
self.testvars['xml_output'] = self.xml_output
self.results = []
def reset_test_stats(self):
self.passed = 0
self.failed = 0
self.todo = 0
self.failures = []
self.perfrequest = None
def start_httpd(self):
host = moznetwork.get_ip()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(("",0))
port = s.getsockname()[1]
s.close()
self.baseurl = 'http://%s:%d/' % (host, port)
self.logger.info('running webserver on %s' % self.baseurl)
self.httpd = MozHttpd(host=host,
port=port,
docroot=os.path.join(os.path.dirname(__file__), 'www'))
self.httpd.start()
def start_marionette(self):
assert(self.baseurl is not None)
if self.bin:
if self.address:
host, port = self.address.split(':')
else:
host = 'localhost'
port = 2828
self.marionette = Marionette(host=host,
port=int(port),
app=self.app,
bin=self.bin,
profile=self.profile,
baseurl=self.baseurl)
elif self.address:
host, port = self.address.split(':')
if self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
host=host, port=int(port),
connectToRunningEmulator=True,
homedir=self.homedir,
baseurl=self.baseurl,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
self.marionette = Marionette(host=host,
port=int(port),
baseurl=self.baseurl)
elif self.emulator:
self.marionette = Marionette.getMarionetteOrExit(
emulator=self.emulator,
emulatorBinary=self.emulatorBinary,
emulatorImg=self.emulatorImg,
emulator_res=self.emulator_res,
homedir=self.homedir,
baseurl=self.baseurl,
noWindow=self.noWindow,
logcat_dir=self.logcat_dir,
gecko_path=self.gecko_path,
symbols_path=self.symbols_path)
else:
raise Exception("must specify binary, address or emulator")
def post_to_autolog(self, elapsedtime):
self.logger.info('posting results to autolog')
logfile = None
if self.emulator:
filename = os.path.join(os.path.abspath(self.logcat_dir),
"emulator-%d.log" % self.marionette.emulator.port)
if os.access(filename, os.F_OK):
logfile = filename
# This is all autolog stuff.
# See: https://wiki.mozilla.org/Auto-tools/Projects/Autolog
from mozautolog import RESTfulAutologTestGroup
testgroup = RESTfulAutologTestGroup(
testgroup = self.testgroup,
os = 'android',
platform = 'emulator',
harness = 'marionette',
server = self.es_server,
restserver = self.rest_server,
machine = socket.gethostname(),
logfile = logfile)
testgroup.set_primary_product(
tree = self.tree,
buildtype = 'opt',
revision = self.revision)
testgroup.add_test_suite(
testsuite = 'b2g emulator testsuite',
elapsedtime = elapsedtime.seconds,
cmdline = '',
passed = self.passed,
failed = self.failed,
todo = self.todo)
# Add in the test failures.
for f in self.failures:
testgroup.add_test_failure(test=f[0], text=f[1], status=f[2])
testgroup.submit()
def run_tests(self, tests):
self.reset_test_stats()
starttime = datetime.utcnow()
while self.repeat >=0:
for test in tests:
self.run_test(test)
self.repeat -= 1
self.logger.info('\nSUMMARY\n-------')
self.logger.info('passed: %d' % self.passed)
self.logger.info('failed: %d' % self.failed)
self.logger.info('todo: %d' % self.todo)
try:
self.marionette.check_for_crash()
except:
traceback.print_exc()
self.elapsedtime = datetime.utcnow() - starttime
if self.autolog:
self.post_to_autolog(self.elapsedtime)
if self.perfrequest and options.perf:
try:
self.perfrequest.submit()
except Exception, e:
print "Could not submit to datazilla"
print e
if self.xml_output:
xml_dir = os.path.dirname(os.path.abspath(self.xml_output))
if not os.path.exists(xml_dir):
os.makedirs(xml_dir)
with open(self.xml_output, 'w') as f:
f.write(self.generate_xml(self.results))
if self.marionette.instance:
self.marionette.instance.close()
self.marionette.instance = None
del self.marionette
