def install(marionette=None, version='1.3'):
"""Installs the app on the attached device. Raises if app is
already installed, but a guard can be added using ``is_installed``.
This function can optionally reuse an existing Marionette session.
"""
# fxos_appgen will create a new Marionette instance if unspecified
fxos_appgen.generate_app(name,
install=True,
version=version,
all_perm=True,
marionette=marionette)
def install(marionette=None, version='1.3'):
"""Installs the app on the attached device. Raises if app is
already installed, but a guard can be added using ``is_installed``.
This function can optionally reuse an existing Marionette session.
"""
# fxos_appgen will create a new Marionette instance if unspecified
fxos_appgen.generate_app(name,
install=True,
version=version,
all_perm=True,
marionette=marionette)
def _run(args, logger):
# This function is to simply make the cli() function easier to handle
test_groups = [
'omni-analyzer',
'permissions',
'webapi',
'user-agent',
'crash-reporter',
'search-id',
]
if args.list_test_groups:
for t in test_groups:
print t
return 0
test_groups = set(args.include if args.include else test_groups)
report = {'buildprops': {}}
logging.basicConfig()
# Step 1: Get device information
try:
dm = mozdevice.DeviceManagerADB(runAdbAsRoot=True)
except mozdevice.DMError as e:
print "Error connecting to device via adb (error: %s). Please be " \
"sure device is connected and 'remote debugging' is enabled." % \
e.msg
raise
# wait here to make sure marionette is running
logger.debug('Attempting to set up port forwarding for marionette')
dm.forward("tcp:2828", "tcp:2828")
retries = 0
while retries < 5:
try:
m = marionette.Marionette()
m.start_session()
m.delete_session()
break
except (IOError, TypeError):
time.sleep(5)
retries += 1
else:
raise Exception("Couldn't connect to marionette after %d attempts. " \
"Is the marionette extension installed?" % retries)
if args.version not in supported_versions:
print "%s is not a valid version. Please enter one of %s" % \
(args.version, supported_versions)
raise Exception("%s is not a valid version" % args.version)
result_file_path = args.result_file
if not result_file_path:
result_file_path = "results.json"
# Make sure we can write to the results file before running tests.
# This will also ensure this file exists in case we error out later on.
try:
result_file = open(result_file_path, "w")
result_file.close()
except IOError as e:
print 'Could not open result file for writing: %s errno: %d' % (result_file_path, e.errno)
raise
# get build properties
buildpropoutput = dm.shellCheckOutput(["cat", "/system/build.prop"])
for buildprop in [line for line in buildpropoutput.splitlines() if '=' \
in line]:
eq = buildprop.find('=')
prop = buildprop[:eq]
val = buildprop[eq + 1:]
report['buildprops'][prop] = val
# get process list
report['processes_running'] = map(lambda p: { 'name': p[1], 'user': p[2] },
dm.getProcessList())
# kernel version
report['kernel_version'] = dm.shellCheckOutput(["cat", "/proc/version"])
# application.ini information
appinicontents = dm.pullFile('/system/b2g/application.ini')
sf = StringIO.StringIO(appinicontents)
config = ConfigParser.ConfigParser()
config.readfp(sf)
report['application_ini'] = {}
for section in config.sections():
report['application_ini'][section] = dict(config.items(section))
logger.suite_start(tests=[])
# run the omni.ja analyzer
if 'omni-analyzer' in test_groups:
omni_ref_path = pkg_resources.resource_filename(
__name__, os.path.join('expected_omni_results', 'omni.ja.%s' % args.version))
omni_analyzer = OmniAnalyzer(omni_ref_path, logger=logger)
diff = omni_analyzer.run()
report["omni_result"] = diff
# start webserver
if 'webapi' in test_groups or 'permissions' in test_groups:
httpd = wptserve.server.WebTestHttpd(
host=moznetwork.get_ip(), port=8000, routes=routes, doc_root=static_path)
httpd.start()
addr = (httpd.host, httpd.port)
# run webapi and webidl tests
if 'webapi' in test_groups:
errors = False
logger.test_start('webapi')
logger.debug('Running webapi verifier tests')
for apptype in ['web', 'privileged', 'certified']:
global webapi_results
webapi_results = None
appname = '%s WebAPI Verifier' % apptype.capitalize()
apppath = os.path.join(static_path, 'webapi-test-app')
install_app(logger, appname, args.version, apptype, apppath, True,
{'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";' % (addr * 2)},
True)
try:
wait.Wait(timeout=120).until(lambda: webapi_results is not None)
except wait.TimeoutException:
logger.error('Timed out waiting for results for test: %s' % last_test_started)
errors = True
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
if webapi_results is None:
continue
if "headers" not in report:
report["headers"] = headers
results_filename = '%s.%s.json' % (args.version, apptype)
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(webapi_results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(
__name__, os.path.sep.join(['expected_webapi_results', results_filename]))
parse_webapi_results(file_path, webapi_results, '%s-' % apptype, logger, report)
logger.debug('Done.')
if errors:
logger.test_end('webapi', 'ERROR')
else:
logger.test_end('webapi', 'OK')
if 'permissions' in test_groups:
errors = False
logger.test_start('permissions')
logger.debug('Running permissions tests')
permissions = get_permissions()
# test default permissions
for apptype in ['web', 'privileged', 'certified']:
logger.debug('Testing default permissions: %s' % apptype)
results = {}
expected_webapi_results = None
appname = 'Default Permissions Test App'
fxos_appgen.uninstall_app(appname)
installed_appname = appname.lower().replace(" ", "-")
fxos_appgen.generate_app(appname, install=True, app_type=apptype,
all_perm=True)
for permission in permissions:
result = get_permission(permission, installed_appname)
results[permission] = result
results_filename = '%s.%s.json' % (args.version, apptype)
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join(['expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, '%s-' % apptype,
logger, report)
fxos_appgen.uninstall_app(appname)
# test individual permissions
logger.debug('Testing individual permissions')
results = {}
# first install test app for embed-apps permission test
embed_appname = 'Embed Apps Test App'
apppath = os.path.join(static_path, 'embed-apps-test-app')
install_app(logger, embed_appname, args.version, 'certified', apppath, True,
{'results_uri.js': 'RESULTS_URI="http://%s:%s/webapi_results_embed_apps";' % addr},
False)
appname = 'Permissions Test App'
installed_appname = appname.lower().replace(" ", "-")
apppath = os.path.join(static_path, 'permissions-test-app')
install_app(logger, appname, args.version, 'web', apppath, False,
{'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";' % (addr * 2)})
for permission in [None] + permissions:
webapi_results = None
webapi_results_embed_app = None
# if we try to launch after killing too quickly, the app seems
# to not fully launch
time.sleep(5)
if permission is not None:
logger.debug('testing permission: %s' % permission)
set_permission(permission, u'allow', installed_appname)
fxos_appgen.launch_app(appname)
try:
wait.Wait(timeout=60).until(lambda: webapi_results is not None)
# embed-apps results are posted to a separate URL
if webapi_results_embed_app:
webapi_results['embed-apps'] = webapi_results_embed_app['embed-apps']
else:
webapi_results['embed-apps'] = False
if permission is None:
expected_webapi_results = webapi_results
else:
results[permission] = diff_results(expected_webapi_results, webapi_results)
except wait.TimeoutException:
logger.error('Timed out waiting for results')
errors = True
if permission is not None:
results[permission] = 'timed out'
else:
# If we timeout on our baseline results there is
# no point in proceeding.
logger.error('Could not get baseline results for permissions. Skipping tests.')
break
kill('app://' + installed_appname)
if permission is not None:
set_permission(permission, u'deny', installed_appname)
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
# we test open-remote-window separately as opening a remote
# window might stop the test app
results['open-remote-window'] = test_open_remote_window(logger,
args.version, addr)
results_filename = '%s.permissions.json' % args.version
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join(['expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, 'individual-',
logger, report)
logger.debug('Done.')
if errors:
logger.test_end('permissions', 'ERROR')
else:
logger.test_end('permissions', 'OK')
# clean up embed-apps test app
logger.debug('uninstalling: %s' % embed_appname)
fxos_appgen.uninstall_app(embed_appname)
if 'user-agent' in test_groups:
logger.test_start('user-agent')
logger.debug('Running user agent tests')
user_agent_string = run_marionette_script("return navigator.userAgent;")
logger.debug('UserAgent: %s' % user_agent_string)
valid = test_user_agent(user_agent_string, logger)
if valid:
logger.test_end('user-agent', 'OK')
else:
logger.test_end('user-agent', 'ERROR')
if 'crash-reporter' in test_groups:
logger.test_start('crash-reporter')
logger.debug('start checking test reporter')
crash_report_toggle = (report.get('application_ini', {})
.get('Crash Reporter', {})
.get('enabled'))
if crash_report_toggle == '1':
logger.test_end('crash-reporter', 'OK')
else:
logger.test_end('crash-reporter', 'ERROR')
if 'search-id' in test_groups:
logger.test_start('search-id')
fxos_appgen.launch_app('browser')
script = """
result = window.wrappedJSObject.UrlHelper.getUrlFromInput('hello world');
return result;
"""
m = marionette.Marionette('localhost', 2828)
m.start_session()
browser = m.find_element('css selector', 'iframe[src="app://search.gaiamobile.org/newtab.html"]')
m.switch_to_frame(browser)
url = m.execute_script(script)
m.delete_session()
report['search-oemid'] = url
oemid_rexp = re.compile('client=mobile-firefoxos&channel=fm:org.mozilla:([A-Z0-9.]+):official&')
match = oemid_rexp.match(url)
if match:
logger.test_status('search-id', 'oemid', 'PASS', message='oemid: %s' % match.groups()[0])
else:
logger.test_status('search-id', 'oemid', 'FAIL', message='no oemid found in url: %s' % url)
logger.test_end('search-id', 'OK')
logger.suite_end()
with open(result_file_path, "w") as result_file:
result_file.write(json.dumps(report, indent=2))
logger.debug('Results have been stored in: %s' % result_file_path)
if args.html_result_file is not None:
make_html_report(args.html_result_file, report)
logger.debug('HTML Results have been stored in: %s' % args.html_result_file)
def test_permissions(logger, report, args, addr):
errors = False
#logger.test_start('permissions')
logger.debug('Running permissions tests')
permissions = get_permissions()
# test default permissions
for apptype in ['web', 'privileged', 'certified']:
logger.debug('Testing default permissions: %s' % apptype)
results = {}
expected_webapi_results = None
appname = 'Default Permissions Test App'
fxos_appgen.uninstall_app(appname)
installed_appname = appname.lower().replace(" ", "-")
fxos_appgen.generate_app(appname, install=True, app_type=apptype,
all_perm=True)
for permission in permissions:
result = get_permission(permission, installed_appname)
results[permission] = result
results_folder = 'permissions_ref/'
results_filename = '%s.json' % apptype
if args.generate_reference:
dirname = os.path.dirname(results_folder + results_filename)
if not os.path.exists(dirname) and dirname <> "":
os.makedirs(dirname)
with open(results_folder + results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join([expected_results_path, 'expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, '%s-' % apptype,
logger, report)
fxos_appgen.uninstall_app(appname)
# test individual permissions
logger.debug('Testing individual permissions')
results = {}
# first install test app for embed-apps permission test
embed_appname = 'Embed Apps Test App'
sampleapppath = os.path.join(static_path, 'sample_apps')
apppath = os.path.join(sampleapppath, 'embed-apps-test-app')
install_app(logger, embed_appname, args.version, 'certified', apppath, True,
{'results_uri.js': 'RESULTS_URI="http://%s:%s/webapi_results_embed_apps";' % addr},
False)
appname = 'Permissions Test App'
installed_appname = appname.lower().replace(" ", "-")
sampleapppath = os.path.join(static_path, 'sample_apps')
apppath = os.path.join(sampleapppath, 'permissions-test-app')
install_app(logger, appname, args.version, 'web', apppath, False,
{'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";' % (addr * 2)})
for permission in [None] + permissions:
global webapi_results
global webapi_results_embed_app
webapi_results = None
webapi_results_embed_app = None
# if we try to launch after killing too quickly, the app seems
# to not fully launch
time.sleep(5)
if permission is not None:
logger.debug('testing permission: %s' % permission)
set_permission(permission, u'allow', installed_appname)
else:
logger.debug('testing permission: None')
fxos_appgen.launch_app(appname)
try:
wait.Wait(timeout=60).until(lambda: webapi_results is not None)
# embed-apps results are posted to a separate URL
if webapi_results_embed_app:
webapi_results['embed-apps'] = webapi_results_embed_app['embed-apps']
else:
webapi_results['embed-apps'] = False
if permission is None:
expected_webapi_results = webapi_results
else:
results[permission] = diff_results(expected_webapi_results, webapi_results)
except wait.TimeoutException:
logger.error('Timed out waiting for results')
errors = True
if permission is not None:
results[permission] = 'timed out'
else:
# If we timeout on our baseline results there is
# no point in proceeding.
logger.error('Could not get baseline results for permissions. Skipping tests.')
break
kill('app://' + installed_appname)
if permission is not None:
set_permission(permission, u'deny', installed_appname)
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
# we test open-remote-window separately as opening a remote
# window might stop the test app
results['open-remote-window'] = test_open_remote_window(logger,
args.version, addr)
results_folder = 'permissions_ref/'
results_filename = 'permissions.json'
if args.generate_reference:
dirname = os.path.dirname(results_folder + results_filename)
if not os.path.exists(dirname) and dirname <> "":
os.makedirs(dirname)
with open(results_folder + results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(__name__,
os.path.sep.join([expected_results_path, 'expected_permissions_results',
results_filename]))
parse_permissions_results(file_path, results, 'individual-',
logger, report)
logger.debug('Done.')
if errors:
logger.error('Test individual with errors')
# clean up embed-apps test app
logger.debug('uninstalling: %s' % embed_appname)
fxos_appgen.uninstall_app(embed_appname)
def test_permissions(logger, report, args, addr):
errors = False
#logger.test_start('permissions')
logger.debug('Running permissions tests')
permissions = get_permissions()
# test default permissions
for apptype in ['web', 'privileged', 'certified']:
logger.debug('Testing default permissions: %s' % apptype)
results = {}
expected_webapi_results = None
appname = 'Default Permissions Test App'
fxos_appgen.uninstall_app(appname)
installed_appname = appname.lower().replace(" ", "-")
fxos_appgen.generate_app(appname,
install=True,
app_type=apptype,
all_perm=True)
for permission in permissions:
result = get_permission(permission, installed_appname)
results[permission] = result
results_filename = '%s.json' % apptype
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(
__name__,
os.path.sep.join([
expected_results_path, 'expected_permissions_results',
results_filename
]))
parse_permissions_results(file_path, results, '%s-' % apptype,
logger, report)
fxos_appgen.uninstall_app(appname)
# test individual permissions
logger.debug('Testing individual permissions')
results = {}
# first install test app for embed-apps permission test
embed_appname = 'Embed Apps Test App'
apppath = os.path.join(static_path, 'embed-apps-test-app')
install_app(
logger, embed_appname, args.version, 'certified', apppath, True, {
'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results_embed_apps";' % addr
}, False)
appname = 'Permissions Test App'
installed_appname = appname.lower().replace(" ", "-")
apppath = os.path.join(static_path, 'permissions-test-app')
install_app(
logger, appname, args.version, 'web', apppath, False, {
'results_uri.js':
'RESULTS_URI="http://%s:%s/webapi_results";LOG_URI="http://%s:%s/webapi_log";'
% (addr * 2)
})
for permission in [None] + permissions:
global webapi_results
global webapi_results_embed_app
webapi_results = None
webapi_results_embed_app = None
# if we try to launch after killing too quickly, the app seems
# to not fully launch
time.sleep(5)
if permission is not None:
logger.debug('testing permission: %s' % permission)
set_permission(permission, u'allow', installed_appname)
else:
logger.debug('testing permission: None')
fxos_appgen.launch_app(appname)
try:
wait.Wait(timeout=60).until(lambda: webapi_results is not None)
# embed-apps results are posted to a separate URL
if webapi_results_embed_app:
webapi_results['embed-apps'] = webapi_results_embed_app[
'embed-apps']
else:
webapi_results['embed-apps'] = False
if permission is None:
expected_webapi_results = webapi_results
else:
results[permission] = diff_results(expected_webapi_results,
webapi_results)
except wait.TimeoutException:
logger.error('Timed out waiting for results')
errors = True
if permission is not None:
results[permission] = 'timed out'
else:
# If we timeout on our baseline results there is
# no point in proceeding.
logger.error(
'Could not get baseline results for permissions. Skipping tests.'
)
break
kill('app://' + installed_appname)
if permission is not None:
set_permission(permission, u'deny', installed_appname)
logger.debug('uninstalling: %s' % appname)
fxos_appgen.uninstall_app(appname)
# we test open-remote-window separately as opening a remote
# window might stop the test app
results['open-remote-window'] = test_open_remote_window(
logger, args.version, addr)
results_filename = 'permissions.json'
if args.generate_reference:
with open(results_filename, 'w') as f:
f.write(json.dumps(results, sort_keys=True, indent=2))
else:
file_path = pkg_resources.resource_filename(
__name__,
os.path.sep.join([
expected_results_path, 'expected_permissions_results',
results_filename
]))
parse_permissions_results(file_path, results, 'individual-', logger,
report)
logger.debug('Done.')
if errors:
logger.error('Test individual with errors')
# clean up embed-apps test app
logger.debug('uninstalling: %s' % embed_appname)
fxos_appgen.uninstall_app(embed_appname)
