class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
profile_class = Profile
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir), self.app_name)
def _start(self, profile=None, addons=(), cmdargs=()):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def _stop(self):
self.runner.stop()
rmtree(self.tempdir)
def _get_app_info(self):
return mozversion.get_version(binary=self.binary)
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if self.verbose:
level = "TRACE" if self.verbose >= 2 else "DEBUG"
profile_args["preferences"]["marionette.logging"] = level
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
if self.workspace:
profile_args['profile'] = tempfile.mkdtemp(
suffix='.mozrunner-{:.0f}'.format(time.time()),
dir=self.workspace)
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile_name = '{}-{:.0f}'.format(
os.path.basename(self.profile_path),
time.time()
)
if self.workspace:
profile_args["path_to"] = os.path.join(self.workspace,
profile_name)
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1'})
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
class GaiaUnitTestRunner(object):
def __init__(self, binary=None, profile=None, symbols_path=None,
browser_arg=()):
self.binary = binary
self.profile = profile
self.symbols_path = symbols_path
self.browser_arg = browser_arg
def run(self):
self.profile_dir = os.path.join(tempfile.mkdtemp(suffix='.gaiaunittest'),
'profile')
shutil.copytree(self.profile, self.profile_dir)
cmdargs = ['--runapp', 'Test Agent']
if self.browser_arg:
cmdargs += list(self.browser_arg)
profile = Profile(profile=self.profile_dir)
self.runner = Runner(binary=self.binary,
profile=profile,
clean_profile=False,
cmdargs=cmdargs,
symbols_path=self.symbols_path)
self.runner.start()
def cleanup(self):
print 'checking for crashes'
crashed = self.runner.check_for_crashes()
self.runner.cleanup()
shutil.rmtree(os.path.dirname(self.profile_dir))
return crashed
__del__ = cleanup
def start(self):
profile_args = {"preferences": self.required_prefs}
if not self.profile_path:
profile_args["restore"] = False
profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile = Profile.clone(**profile_args)
if self.gecko_log is None:
self.gecko_log = 'gecko.log'
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
os.remove(self.gecko_log)
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args={
'processOutputLine': [NullOutput()],
'logfile': self.gecko_log})
self.runner.start()
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
profile_class = Profile
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name)
def _start(self, profile=None, addons=(), cmdargs=()):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def _stop(self):
self.runner.stop()
rmtree(self.tempdir)
def _get_app_info(self):
return mozversion.get_version(binary=self.binary)
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
if self.prefs:
profile_args["preferences"].update(self.prefs)
if not self.profile_path:
profile_args["restore"] = False
profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile = Profile.clone(**profile_args)
if self.gecko_log is None:
self.gecko_log = 'gecko.log'
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
if platform.system() is 'Windows':
# NOTE: windows has a weird filesystem where it happily 'closes'
# the file, but complains if you try to delete it. You get a
# 'file still in use' error. Sometimes you can wait a bit and
# a retry will succeed.
# If all retries fail, we'll just continue without removing
# the file. In this case, if we are restarting the instance,
# then the new logs just get appended to the old file.
tries = 0
while tries < 10:
try:
os.remove(self.gecko_log)
break
except WindowsError as e:
if e.errno == errno.EACCES:
tries += 1
time.sleep(0.5)
else:
raise e
else:
os.remove(self.gecko_log)
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args={
'processOutputLine': [NullOutput()],
'logfile': self.gecko_log})
self.runner.start()
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs, profile=profile)
self.runner.start()
return True
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"][
"marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel":
"jsdebugger",
"devtools.debugger.remote-enabled":
True,
"devtools.chrome.enabled":
True,
"devtools.debugger.prompt-connection":
False,
"marionette.debugging.clicktostart":
True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({
'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
})
self.runner = Runner(binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] +
self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile,
addons=addons,
preferences=preferences)
self._logger.info("Launching %s" % self.binary)
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def _start(
self,
profile=None,
addons=(),
cmdargs=(),
preferences=None,
adb_profile_dir=None,
):
profile = self._create_profile(profile=profile,
addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print()
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True,
)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print()
LOG.warning("Process exited with code %s" % exitcode)
# we don't need stdin, and GUI will not work in Windowed mode if set
# see: https://stackoverflow.com/a/40108817
# also, don't stream to stdout: https://bugzilla.mozilla.org/show_bug.cgi?id=1653349
devnull = open(os.devnull, "wb")
self.runner.process_args = {
"processOutputLine": [get_default_logger("process").info],
"stdin": devnull,
"stream": None,
"onFinish": _on_exit,
}
self.runner.start()
def _start(self, profile=None, addons=(), cmdargs=()):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if self.verbose:
level = "TRACE" if self.verbose >= 2 else "DEBUG"
profile_args["preferences"]["marionette.logging"] = level
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
if self.workspace:
profile_args['profile'] = tempfile.mkdtemp(
suffix='.mozrunner-{:.0f}'.format(time.time()),
dir=self.workspace)
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile_name = '{}-{:.0f}'.format(
os.path.basename(self.profile_path),
time.time()
)
if self.workspace:
profile_args["path_to"] = os.path.join(self.workspace,
profile_name)
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def run(self):
self.profile_dir = os.path.join(tempfile.mkdtemp(suffix='.gaiaunittest'),
'profile')
shutil.copytree(self.profile, self.profile_dir)
cmdargs = ['--runapp', 'Test Agent']
if self.browser_arg:
cmdargs += list(self.browser_arg)
profile = Profile(profile=self.profile_dir)
self.runner = Runner(binary=self.binary,
profile=profile,
clean_profile=False,
cmdargs=cmdargs,
symbols_path=self.symbols_path)
self.runner.start()
def start(self, date=datetime.date.today()):
self.install(date)
if self.profile:
profile = self.app.profileClass(profile=self.profile,
addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
self.runner = Runner(binary=self.app.binary,
cmdargs=self.cmdargs,
profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
class NightlyRunner(object):
def __init__(self,
addons=None,
appname="firefox",
repo_name=None,
profile=None,
cmdargs=[]):
if appname.lower() == 'thunderbird':
self.app = ThunderbirdNightly(repo_name=repo_name)
elif appname.lower() == 'mobile':
self.app = FennecNightly(repo_name=repo_name)
else:
self.app = FirefoxNightly(repo_name=repo_name)
self.addons = addons
self.profile = profile
self.cmdargs = cmdargs
def install(self, date=datetime.date.today()):
if not self.app.download(date=date):
print "could not find nightly from " + str(date)
return False # download failed
print "Starting nightly\n"
self.app.install()
def start(self, date=datetime.date.today()):
self.install(date)
if self.profile:
profile = self.app.profileClass(profile=self.profile,
addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
self.runner = Runner(binary=self.app.binary,
cmdargs=self.cmdargs,
profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
def stop(self):
self.runner.stop()
def getAppInfo(self):
return self.app.getAppInfo()
def start(self):
profile = self.profile
if not profile:
prefs = {"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": 2828}
profile = {"preferences": prefs, "restore":False}
print "starting runner"
self.runner = Runner.create(binary=self.bin, profile_args=profile, cmdargs=['-no-remote'])
self.runner.start()
def start(self, date=datetime.date.today()):
if not self.app.download(date=date):
print "could not find nightly from " + str(date)
return False # download failed
self.app.install()
if self.profile:
profile = self.app.profileClass(profile=self.profile, addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
print "running nightly from " + str(date) + "\n"
self.runner = Runner(binary=self.app.binary, cmdargs=self.cmdargs, profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
def _start(self,
profile=None,
addons=(),
cmdargs=(),
preferences=None,
adb_profile_dir=None):
profile = self._create_profile(profile=profile,
addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print()
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print()
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def start(self):
profile_path = self.profile
profile_args = {"preferences": self.required_prefs}
if not profile_path:
profile_args["restore"] = False
else:
profile_args["profile"] = profile_path
print "starting runner"
self.runner = Runner.create(binary=self.bin, profile_args=profile_args, cmdargs=["-no-remote"])
self.runner.start()
def start(self):
profile = self.profile
if not profile:
prefs = {"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": 2828,
"browser.warnOnQuit": False}
profile = {"preferences": prefs, "restore":False}
print "starting runner"
self.runner = Runner.create(binary=self.bin, profile_args=profile, cmdargs=['-no-remote'])
self.runner.start()
def run(self):
self.profile_dir = os.path.join(tempfile.mkdtemp(suffix='.gaiaunittest'),
'profile')
shutil.copytree(self.profile, self.profile_dir)
self.runner = Runner.create(binary=self.binary,
profile_args={'profile': self.profile_dir},
clean_profile=False,
cmdargs=['--runapp', 'Test Agent'])
self.runner.start()
def run(self):
self.profile_dir = os.path.join(
tempfile.mkdtemp(suffix='.gaiaunittest'), 'profile')
shutil.copytree(self.profile, self.profile_dir)
self.runner = Runner.create(binary=self.binary,
profile_args={'profile': self.profile_dir},
clean_profile=False,
cmdargs=['--runapp', 'Test Agent'])
self.runner.start()
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
self._logger.info("Launching %s" % self.binary)
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def run(self):
self.profile_dir = os.path.join(tempfile.mkdtemp(suffix=".gaiaunittest"), "profile")
shutil.copytree(self.profile, self.profile_dir)
self.runner = Runner.create(
binary=self.binary,
profile_args={"profile": self.profile_dir},
clean_profile=False,
cmdargs=["--runapp", "Test Agent"],
)
self.runner.start()
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs, profile=profile)
self.runner.start()
return True
def start(self, date=datetime.date.today()):
self.install(date)
if self.profile:
profile = self.app.profileClass(profile=self.profile, addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
self.runner = Runner(binary=self.app.binary, cmdargs=self.cmdargs, profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
class NightlyRunner(object):
def __init__(self, addons=None, appname="firefox", repo_name=None,
profile=None, cmdargs=[]):
if appname.lower() == 'thunderbird':
self.app = ThunderbirdNightly(repo_name=repo_name)
elif appname.lower() == 'mobile':
self.app = FennecNightly(repo_name=repo_name)
else:
self.app = FirefoxNightly(repo_name=repo_name)
self.addons = addons
self.profile = profile
self.cmdargs = cmdargs
def install(self, date=datetime.date.today()):
if not self.app.download(date=date):
print "could not find nightly from " + str(date)
return False # download failed
print "Starting nightly\n"
self.app.install()
def start(self, date=datetime.date.today()):
self.install(date)
if self.profile:
profile = self.app.profileClass(profile=self.profile, addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
self.runner = Runner(binary=self.app.binary, cmdargs=self.cmdargs, profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
def stop(self):
self.runner.stop()
def getAppInfo(self):
return self.app.getAppInfo()
def _start(self, profile=None, addons=(), cmdargs=()):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir), self.app_name)
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile,
addons=addons,
preferences=preferences)
self._logger.info("Launching %s" % self.binary)
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def _stop(self):
self.runner.stop()
def __del__(self):
try:
Launcher.__del__(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return mozversion.get_version(binary=self.binary)
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name)
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
self._logger.info("Launching %s" % self.binary)
process_args = {'processOutputLine': [self._logger.debug]}
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile,
process_args=process_args)
self.runner.start()
def _stop(self):
self.runner.stop()
def __del__(self):
try:
Launcher.__del__(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return mozversion.get_version(binary=self.binary)
def run(self):
self.profile_dir = os.path.join(tempfile.mkdtemp(suffix='.gaiaunittest'),
'profile')
shutil.copytree(self.profile, self.profile_dir)
cmdargs = ['--runapp', 'Test Agent']
if self.browser_arg:
cmdargs += list(self.browser_arg)
self.runner = Runner.create(binary=self.binary,
profile_args={'profile': self.profile_dir},
clean_profile=False,
cmdargs=cmdargs,
symbols_path=self.symbols_path)
self.runner.start()
def start(self, date=datetime.date.today()):
if not self.app.download(date=date):
print "could not find nightly from " + str(date)
return False # download failed
self.app.install()
if self.profile:
profile = self.app.profileClass(profile=self.profile, addons=self.addons)
elif len(self.addons):
profile = self.app.profileClass(addons=self.addons)
else:
profile = self.app.profileClass()
print "running nightly from " + str(date) + "\n"
self.runner = Runner(binary=self.app.binary, cmdargs=self.cmdargs, profile=profile)
self.runner.names = [self.app.processName]
self.runner.start()
return True
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
})
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
self._logger.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print
self._logger.error(
"Error while waiting process, consider filing a bug.",
exc_info=True
)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print
self._logger.warning('Process exited with code %s'
% exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if "-jsdebugger" in self.app_args:
profile_args["preferences"].update(
{
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.debugger.chrome-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
}
)
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
profile_args["restore"] = False
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {"processOutputLine": [NullOutput()]}
if self.gecko_log == "-":
process_args["stream"] = sys.stdout
else:
if self.gecko_log is None:
self.gecko_log = "gecko.log"
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
if platform.system() is "Windows":
# NOTE: windows has a weird filesystem where it happily 'closes'
# the file, but complains if you try to delete it. You get a
# 'file still in use' error. Sometimes you can wait a bit and
# a retry will succeed.
# If all retries fail, we'll just continue without removing
# the file. In this case, if we are restarting the instance,
# then the new logs just get appended to the old file.
tries = 0
while tries < 10:
try:
os.remove(self.gecko_log)
break
except WindowsError as e:
if e.errno == errno.EACCES:
tries += 1
time.sleep(0.5)
else:
raise e
else:
os.remove(self.gecko_log)
process_args["logfile"] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({"MOZ_CRASHREPORTER": "1", "MOZ_CRASHREPORTER_NO_REPORT": "1"})
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=["-no-remote", "-marionette"] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args,
)
self.runner.start()
class GeckoInstance(object):
required_prefs = {
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.displayedE10SPrompt": 5,
"browser.sessionstore.resume_from_crash": False,
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False,
"browser.tabs.remote.autostart": False,
"browser.urlbar.userMadeSearchSuggestionsChoice": True,
"browser.warnOnQuit": False,
"datareporting.healthreport.logging.consoleEnabled": False,
"datareporting.healthreport.service.enabled": False,
"datareporting.healthreport.service.firstRun": False,
"datareporting.healthreport.uploadEnabled": False,
"datareporting.policy.dataSubmissionEnabled": False,
"datareporting.policy.dataSubmissionPolicyAccepted": False,
"dom.ipc.reportProcessHangs": False,
"focusmanager.testmode": True,
"marionette.defaultPrefs.enabled": True,
"startup.homepage_welcome_url": "about:blank",
"toolkit.telemetry.enabled": False,
# Until Bug 1238095 is fixed, we have to enable CPOWs in order
# for Marionette tests to work properly.
"dom.ipc.cpows.forbid-unsafe-from-browser": False,
}
def __init__(self,
host,
port,
bin,
profile=None,
addons=None,
app_args=None,
symbols_path=None,
gecko_log=None,
prefs=None,
workspace=None,
verbose=0):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
# Alternative to default temporary directory
self.workspace = workspace
# Check if it is a Profile object or a path to profile
self.profile = None
self.addons = addons
if isinstance(profile, Profile):
self.profile = profile
else:
self.profile_path = profile
self.prefs = prefs
self.required_prefs = deepcopy(GeckoInstance.required_prefs)
if prefs:
self.required_prefs.update(prefs)
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
if gecko_log != '-':
if gecko_log is None:
gecko_log = 'gecko.log'
elif os.path.isdir(gecko_log):
fname = 'gecko-%d.log' % time.time()
gecko_log = os.path.join(gecko_log, fname)
gecko_log = os.path.realpath(gecko_log)
if os.access(gecko_log, os.F_OK):
os.remove(gecko_log)
self.gecko_log = gecko_log
self.verbose = verbose
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"][
"marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if self.verbose:
level = "TRACE" if self.verbose >= 2 else "DEBUG"
profile_args["preferences"]["marionette.logging"] = level
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel":
"jsdebugger",
"devtools.debugger.remote-enabled":
True,
"devtools.chrome.enabled":
True,
"devtools.debugger.prompt-connection":
False,
"marionette.debugging.clicktostart":
True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
if self.workspace:
profile_args['profile'] = tempfile.mkdtemp(
suffix='.mozrunner-{:.0f}'.format(time.time()),
dir=self.workspace)
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile_name = '{}-{:.0f}'.format(
os.path.basename(self.profile_path), time.time())
if self.workspace:
profile_args["path_to"] = os.path.join(
self.workspace, profile_name)
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({
'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
})
self.runner = Runner(binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] +
self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def close(self, restart=False):
if not restart:
self.profile = None
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None, clean=True):
self.close(restart=True)
if clean:
self.profile.cleanup()
self.profile = None
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class GeckoInstance(object):
required_prefs = {
"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": 2828,
"marionette.logging": True,
"startup.homepage_welcome_url": "about:blank",
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.sessionstore.resume_from_crash": False,
"browser.warnOnQuit": False
}
def __init__(self,
host,
port,
bin,
profile,
app_args=None,
symbols_path=None,
gecko_log=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
self.profile_path = profile
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": self.required_prefs}
if not self.profile_path:
profile_args["restore"] = False
profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile = Profile.clone(**profile_args)
if self.gecko_log is None:
self.gecko_log = 'gecko.log'
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
os.remove(self.gecko_log)
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({
'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
})
self.runner = Runner(binary=self.bin,
profile=profile,
cmdargs=['-no-remote', '-marionette'] +
self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args={
'processOutputLine': [NullOutput()],
'logfile': self.gecko_log
})
self.runner.start()
def check_for_crashes(self):
return self.runner.check_for_crashes()
def close(self):
if self.runner:
self.runner.stop()
self.runner.cleanup()
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = safe_mkdtemp()
try:
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir), self.app_name)
except Exception:
rmtree(self.tempdir)
raise
def _disableUpdateByPolicy(self):
updatePolicy = {'policies': {'DisableAppUpdate': True}}
installdir = os.path.dirname(self.binary)
if mozinfo.os == 'mac':
# macOS has the following filestructure:
# binary at:
# PackageName.app/Contents/MacOS/firefox
# we need policies.json in:
# PackageName.app/Contents/Resources/distribution
installdir = os.path.normpath(
os.path.join(installdir, '..', 'Resources'))
os.mkdir(os.path.join(installdir, 'distribution'))
policyFile = os.path.join(installdir, 'distribution', 'policies.json')
with open(policyFile, 'w') as fp:
json.dump(updatePolicy, fp, indent=2)
def _start(self,
profile=None,
addons=(),
cmdargs=(),
preferences=None,
adb_profile_dir=None):
profile = self._create_profile(profile=profile,
addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print()
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print()
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def _wait(self):
return self.runner.wait()
def _stop(self):
if mozinfo.os == "win" and self.app_name == 'firefox':
# for some reason, stopping the runner may hang on windows. For
# example restart the browser in safe mode, it will hang for a
# couple of minutes. As a workaround, we call that in a thread and
# wait a bit for the completion. If the stop() can't complete we
# forgot about that thread.
thread = Thread(target=self.runner.stop)
thread.daemon = True
thread.start()
thread.join(0.7)
else:
self.runner.stop()
# release the runner since it holds a profile reference
del self.runner
def cleanup(self):
try:
Launcher.cleanup(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return safe_get_version(binary=self.binary)
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"][
"marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel":
"jsdebugger",
"devtools.debugger.remote-enabled":
True,
"devtools.debugger.chrome-enabled":
True,
"devtools.chrome.enabled":
True,
"devtools.debugger.prompt-connection":
False,
"marionette.debugging.clicktostart":
True,
})
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
profile_args["restore"] = False
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.goanna_log == '-':
process_args['stream'] = sys.stdout
else:
if self.goanna_log is None:
self.goanna_log = 'goanna.log'
elif os.path.isdir(self.goanna_log):
fname = "goanna-%d.log" % time.time()
self.goanna_log = os.path.join(self.goanna_log, fname)
self.goanna_log = os.path.realpath(self.goanna_log)
if os.access(self.goanna_log, os.F_OK):
if platform.system() is 'Windows':
# NOTE: windows has a weird filesystem where it happily 'closes'
# the file, but complains if you try to delete it. You get a
# 'file still in use' error. Sometimes you can wait a bit and
# a retry will succeed.
# If all retries fail, we'll just continue without removing
# the file. In this case, if we are restarting the instance,
# then the new logs just get appended to the old file.
tries = 0
while tries < 10:
try:
os.remove(self.goanna_log)
break
except WindowsError as e:
if e.errno == errno.EACCES:
tries += 1
time.sleep(0.5)
else:
raise e
else:
os.remove(self.goanna_log)
process_args['logfile'] = self.goanna_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({
'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
})
self.runner = Runner(binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] +
self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
class Nightly(object):
name = None # abstract base class
_monthlinks = {}
lastdest = None
tempdir = None
app_name = None
profile_class = Profile
build_base_repo_name = "firefox"
@staticmethod
def _get_os_regex_suffix(bits, with_ext=True):
if mozinfo.os == "win":
if bits == 64:
raise errors.Win64NoAvailableBuildError()
suffix, ext = ".*win32", ".zip"
elif mozinfo.os == "linux":
if bits == 64:
suffix, ext = ".*linux-x86_64", ".tar.bz2"
else:
suffix, ext = ".*linux-i686", ".tar.bz2"
elif mozinfo.os == "mac":
suffix, ext = r".*mac.*", "\.dmg"
if with_ext:
return '%s%s' % (suffix, ext)
else:
return suffix
@staticmethod
def _get_build_regex(name, bits, with_ext=True):
name_prefix = name if ".*%s" % name is not None else ''
suffix = Nightly._get_os_regex_suffix(bits, with_ext)
return "%s%s" % (name_prefix, suffix)
def __init__(self, inbound_branch=None, bits=mozinfo.bits, persist=None):
self.inbound_branch = inbound_branch
self.bits = bits
self.persist = persist
self.build_regex = self._get_build_regex(self.name, bits) + "$"
self.build_info_regex = \
self._get_build_regex(self.name, bits, with_ext=False) + "\.txt$"
def get_inbound_branch(self, date):
raise NotImplementedError
# cleanup functions
def remove_tempdir(self):
if self.tempdir:
rmtree(self.tempdir)
self.tempdir = None
def remove_lastdest(self):
if self.lastdest:
os.remove(self.lastdest)
self.lastdest = None
def cleanup(self):
self.remove_tempdir()
if not self.persist:
self.remove_lastdest()
__del__ = cleanup
# installation functions
def get_destination(self, url, date):
inbound_branch = self.get_inbound_branch(date)
dest = os.path.basename(url)
if self.persist is not None:
if hasattr(date, "strftime"):
date_str = date.strftime("%Y-%m-%d")
else:
date_str = date # Might be just a number with inbound
dest = os.path.join(
self.persist, "%s--%s--%s" % (date_str, inbound_branch, dest))
return dest
def download(self, date=datetime.date.today(), dest=None):
url = self.get_build_url(date)
if url:
if not dest:
dest = self.get_destination(url, date)
if not self.persist:
self.remove_lastdest()
download_url(url, dest)
self.dest = self.lastdest = dest
return True
else:
return False
def install(self):
if not self.name:
raise NotImplementedError("Can't invoke abstract base class")
self.remove_tempdir()
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=self.dest, dest=self.tempdir), self.name)
return True
def get_build_info(self, date):
url = self._get_build_url(date, self.build_info_regex, 'builds info')
if url is not None:
print "Getting %s" % url
response = requests.get(url)
if response.status_code == 200:
for line in response.text.splitlines():
if '/rev/' in line:
# returns [repository, changeset]
return line.split('/rev/')
def get_build_url(self, datestamp):
return self._get_build_url(datestamp, self.build_regex, 'builds')
def _get_build_url(self, datestamp, regex, what):
url = "http://ftp.mozilla.org/pub/mozilla.org/" + \
self.build_base_repo_name + "/nightly/"
year = str(datestamp.year)
month = "%02d" % datestamp.month
day = "%02d" % datestamp.day
inbound_branch = self.get_inbound_branch(datestamp)
url += year + "/" + month + "/"
link_regex = '^' + year + '-' + month + '-' + day + '-' \
+ r'[\d-]+' + inbound_branch + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = url_links(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
matches = []
for dirlink in monthlinks:
if re.match(link_regex, dirlink):
# now parse the page for the correct build url
for link in url_links(url + dirlink, regex=regex):
matches.append(url + dirlink + link)
if not matches:
print "Tried to get %s from %s that match '%s' but didn't find any." % \
(what, url, self.build_regex)
return None
else:
return sorted(matches)[-1] # the most recent build url
# functions for invoking nightly
def get_app_info(self):
return mozversion.get_version(binary=self.binary)
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
class GeckoInstance(object):
required_prefs = {"marionette.defaultPrefs.enabled": True,
"marionette.logging": True,
"browser.displayedE10SPrompt": 5,
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.sessionstore.resume_from_crash": False,
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False,
"browser.warnOnQuit": False,
"dom.ipc.reportProcessHangs": False,
"focusmanager.testmode": True,
"startup.homepage_welcome_url": "about:blank"}
def __init__(self, host, port, bin, profile=None, addons=None,
app_args=None, symbols_path=None, gecko_log=None, prefs=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
# Check if it is a Profile object or a path to profile
self.profile = None
self.addons = addons
if isinstance(profile, Profile):
self.profile = profile
else:
self.profile_path = profile
self.prefs = prefs
self.required_prefs = deepcopy(GeckoInstance.required_prefs)
if prefs:
self.required_prefs.update(prefs)
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
if gecko_log != '-':
if gecko_log is None:
gecko_log = 'gecko.log'
elif os.path.isdir(gecko_log):
fname = 'gecko-%d.log' % time.time()
gecko_log = os.path.join(gecko_log, fname)
gecko_log = os.path.realpath(gecko_log)
if os.access(gecko_log, os.F_OK):
os.remove(gecko_log)
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def close(self, restart=False):
if not restart:
self.profile = None
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None, clean=True):
self.close(restart=True)
if clean:
self.profile.cleanup()
self.profile = None
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class GeckoInstance(object):
required_prefs = {
"marionette.defaultPrefs.enabled": True,
"marionette.logging": True,
"browser.displayedE10SPrompt": 5,
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.sessionstore.resume_from_crash": False,
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False,
"browser.warnOnQuit": False,
"dom.ipc.reportProcessHangs": False,
"focusmanager.testmode": True,
"startup.homepage_welcome_url": "about:blank"
}
def __init__(self,
host,
port,
bin,
profile=None,
addons=None,
app_args=None,
symbols_path=None,
gecko_log=None,
prefs=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
# Check if it is a Profile object or a path to profile
self.profile = None
self.addons = addons
if isinstance(profile, Profile):
self.profile = profile
else:
self.profile_path = profile
self.prefs = prefs
self.required_prefs = deepcopy(GeckoInstance.required_prefs)
if prefs:
self.required_prefs.update(prefs)
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
if gecko_log != '-':
if gecko_log is None:
gecko_log = 'gecko.log'
elif os.path.isdir(gecko_log):
fname = 'gecko-%d.log' % time.time()
gecko_log = os.path.join(gecko_log, fname)
gecko_log = os.path.realpath(gecko_log)
if os.access(gecko_log, os.F_OK):
os.remove(gecko_log)
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"][
"marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel":
"jsdebugger",
"devtools.debugger.remote-enabled":
True,
"devtools.chrome.enabled":
True,
"devtools.debugger.prompt-connection":
False,
"marionette.debugging.clicktostart":
True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({
'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1',
})
self.runner = Runner(binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] +
self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def close(self, restart=False):
if not restart:
self.profile = None
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None, clean=True):
self.close(restart=True)
if clean:
self.profile.cleanup()
self.profile = None
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class Nightly(object):
name = None # abstract base class
_monthlinks = {}
lastdest = None
tempdir = None
@staticmethod
def _getBuildRegex(bits):
if mozinfo.os == "win":
if bits == 64:
# XXX this should actually throw an error to be consumed by the caller
print "No builds available for 64 bit Windows (try specifying --bits=32)"
sys.exit()
return ".*win32.zip"
elif mozinfo.os == "linux":
if bits == 64:
return ".*linux-x86_64.tar.bz2"
else:
return ".*linux-i686.tar.bz2"
elif mozinfo.os == "mac":
return ".*mac.*\.dmg"
def __init__(self, repo_name=None, bits=mozinfo.bits, persist=None):
self.buildRegex = self._getBuildRegex(bits)
self.persist = persist
self.repo_name = repo_name
### cleanup functions
def remove_tempdir(self):
if self.tempdir:
rmtree(self.tempdir)
self.tempdir = None
def remove_lastdest(self):
if self.lastdest:
os.remove(self.lastdest)
self.lastdest = None
def cleanup(self):
self.remove_tempdir()
if not self.persist:
self.remove_lastdest()
__del__ = cleanup
### installation functions
def get_destination(self, url, date):
repo_name = self.repo_name or self.getRepoName(date)
dest = os.path.basename(url)
if self.persist is not None:
date_str = date.strftime("%Y-%m-%d")
dest = os.path.join(self.persist, "%s--%s--%s"%(date_str, repo_name, dest))
return dest
def download(self, date=datetime.date.today(), dest=None):
url = self.getBuildUrl(date)
if url:
if not dest:
dest = self.get_destination(url, date)
if not self.persist:
self.remove_lastdest()
self.dest = self.lastdest = dest
download_url(url, dest)
return True
else:
return False
def install(self):
if not self.name:
raise NotImplementedError("Can't invoke abstract base class")
self.remove_tempdir()
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(mozinstall.install(src=self.dest, dest=self.tempdir), self.name)
return True
def getBuildUrl(self, datestamp):
if self.appName == 'fennec':
repo = 'mobile'
else:
repo = 'firefox'
url = "http://ftp.mozilla.org/pub/mozilla.org/" + repo + "/nightly/"
year = str(datestamp.year)
month = "%02d" % datestamp.month
day = "%02d" % datestamp.day
repo_name = self.repo_name or self.getRepoName(datestamp)
url += year + "/" + month + "/"
linkRegex = '^' + year + '-' + month + '-' + day + '-' + '[\d-]+' + repo_name + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = urlLinks(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
for dirlink in monthlinks:
dirhref = dirlink.get("href")
if re.match(linkRegex, dirhref):
# now parse the page for the correct build url
for link in urlLinks(url + dirhref):
href = link.get("href")
if re.match(self.buildRegex, href):
return url + dirhref + href
### functions for invoking nightly
def getAppInfo(self):
parser = ConfigParser()
ini_file = os.path.join(os.path.dirname(self.binary), "application.ini")
parser.read(ini_file)
try:
changeset = parser.get('App', 'SourceStamp')
repo = parser.get('App', 'SourceRepository')
return (repo, changeset)
except:
return None
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs, profile=profile)
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
class GeckoInstance(object):
required_prefs = {"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": 2828,
"marionette.logging": True,
"startup.homepage_welcome_url": "about:blank",
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.sessionstore.resume_from_crash": False,
"browser.warnOnQuit": False,
"browser.displayedE10SPrompt": 5,
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False}
def __init__(self, host, port, bin, profile, app_args=None, symbols_path=None,
gecko_log=None, prefs=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
self.profile_path = profile
self.prefs = prefs
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
if self.prefs:
profile_args["preferences"].update(self.prefs)
if not self.profile_path:
profile_args["restore"] = False
profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile = Profile.clone(**profile_args)
if self.gecko_log is None:
self.gecko_log = 'gecko.log'
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
if platform.system() is 'Windows':
# NOTE: windows has a weird filesystem where it happily 'closes'
# the file, but complains if you try to delete it. You get a
# 'file still in use' error. Sometimes you can wait a bit and
# a retry will succeed.
# If all retries fail, we'll just continue without removing
# the file. In this case, if we are restarting the instance,
# then the new logs just get appended to the old file.
tries = 0
while tries < 10:
try:
os.remove(self.gecko_log)
break
except WindowsError as e:
if e.errno == errno.EACCES:
tries += 1
time.sleep(0.5)
else:
raise e
else:
os.remove(self.gecko_log)
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args={
'processOutputLine': [NullOutput()],
'logfile': self.gecko_log})
self.runner.start()
def close(self):
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None):
self.close()
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class GeckoInstance(object):
required_prefs = {
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.displayedE10SPrompt": 5,
"browser.sessionstore.resume_from_crash": False,
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False,
"browser.tabs.remote.autostart": False,
"browser.urlbar.userMadeSearchSuggestionsChoice": True,
"browser.warnOnQuit": False,
"datareporting.healthreport.logging.consoleEnabled": False,
"datareporting.healthreport.service.enabled": False,
"datareporting.healthreport.service.firstRun": False,
"datareporting.healthreport.uploadEnabled": False,
"datareporting.policy.dataSubmissionEnabled": False,
"datareporting.policy.dataSubmissionPolicyAccepted": False,
"dom.ipc.reportProcessHangs": False,
"focusmanager.testmode": True,
"marionette.defaultPrefs.enabled": True,
"startup.homepage_welcome_url": "about:blank",
"toolkit.telemetry.enabled": False,
}
def __init__(self, host, port, bin, profile=None, addons=None,
app_args=None, symbols_path=None, gecko_log=None, prefs=None,
workspace=None, verbose=0):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
# Alternative to default temporary directory
self.workspace = workspace
# Check if it is a Profile object or a path to profile
self.profile = None
self.addons = addons
if isinstance(profile, Profile):
self.profile = profile
else:
self.profile_path = profile
self.prefs = prefs
self.required_prefs = deepcopy(GeckoInstance.required_prefs)
if prefs:
self.required_prefs.update(prefs)
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
if gecko_log != '-':
if gecko_log is None:
gecko_log = 'gecko.log'
elif os.path.isdir(gecko_log):
fname = 'gecko-%d.log' % time.time()
gecko_log = os.path.join(gecko_log, fname)
gecko_log = os.path.realpath(gecko_log)
if os.access(gecko_log, os.F_OK):
os.remove(gecko_log)
self.gecko_log = gecko_log
self.verbose = verbose
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if self.verbose:
level = "TRACE" if self.verbose >= 2 else "DEBUG"
profile_args["preferences"]["marionette.logging"] = level
if '-jsdebugger' in self.app_args:
profile_args["preferences"].update({
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
})
if self.addons:
profile_args['addons'] = self.addons
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
if self.workspace:
profile_args['profile'] = tempfile.mkdtemp(
suffix='.mozrunner-{:.0f}'.format(time.time()),
dir=self.workspace)
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile_name = '{}-{:.0f}'.format(
os.path.basename(self.profile_path),
time.time()
)
if self.workspace:
profile_args["path_to"] = os.path.join(self.workspace,
profile_name)
self.profile = Profile.clone(**profile_args)
process_args = {
'processOutputLine': [NullOutput()],
}
if self.gecko_log == '-':
process_args['stream'] = sys.stdout
else:
process_args['logfile'] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args)
self.runner.start()
def close(self, restart=False):
if not restart:
self.profile = None
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None, clean=True):
self.close(restart=True)
if clean:
self.profile.cleanup()
self.profile = None
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
try:
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name
)
except:
rmtree(self.tempdir)
raise
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True
)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def _wait(self):
return self.runner.wait()
def _stop(self):
self.runner.stop()
# release the runner since it holds a profile reference
del self.runner
def cleanup(self):
try:
Launcher.cleanup(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return mozversion.get_version(binary=self.binary)
class Nightly(object):
name = None # abstract base class
def __init__(self, repo_name=None, bits=mozinfo.bits, persist=None):
if mozinfo.os == "win":
if bits == 64:
# XXX this should actually throw an error to be consumed by the caller
print "No nightly builds available for 64 bit Windows"
sys.exit()
self.buildRegex = ".*win32.zip"
elif mozinfo.os == "linux":
if bits == 64:
self.buildRegex = ".*linux-x86_64.tar.bz2"
else:
self.buildRegex = ".*linux-i686.tar.bz2"
elif mozinfo.os == "mac":
self.buildRegex = ".*mac.*\.dmg"
self.persist = persist
self.repo_name = repo_name
self._monthlinks = {}
self.lastdest = None
self.tempdir = None
### cleanup functions
def remove_tempdir(self):
if self.tempdir:
rmtree(self.tempdir)
self.tempdir = None
def remove_lastdest(self):
if self.lastdest:
os.remove(self.lastdest)
self.lastdest = None
def cleanup(self):
self.remove_tempdir()
if not self.persist:
self.remove_lastdest()
__del__ = cleanup
### installation functions
def get_destination(self, url, date):
repo_name = self.repo_name or self.getRepoName(date)
dest = os.path.basename(url)
if self.persist is not None:
date_str = date.strftime("%Y-%m-%d")
dest = os.path.join(self.persist, "%s--%s--%s"%(date_str, repo_name, dest))
return dest
def download(self, date=datetime.date.today(), dest=None):
url = self.getBuildUrl(date)
if url:
if not dest:
dest = self.get_destination(url, date)
if not self.persist:
self.remove_lastdest()
self.dest = self.lastdest = dest
download_url(url, dest)
return True
else:
return False
def install(self):
if not self.name:
raise NotImplementedError("Can't invoke abstract base class")
self.remove_tempdir()
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(mozinstall.install(src=self.dest, dest=self.tempdir), self.name)
return True
def getBuildUrl(self, date):
url = "http://ftp.mozilla.org/pub/mozilla.org/" + self.appName + "/nightly/"
year = str(date.year)
month = "%02d" % date.month
day = "%02d" % date.day
repo_name = self.repo_name or self.getRepoName(date)
url += year + "/" + month + "/"
linkRegex = '^' + year + '-' + month + '-' + day + '-' + '[\d-]+' + repo_name + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = urlLinks(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
for dirlink in monthlinks:
dirhref = dirlink.get("href")
if re.match(linkRegex, dirhref):
# now parse the page for the correct build url
for link in urlLinks(url + dirhref):
href = link.get("href")
if re.match(self.buildRegex, href):
return url + dirhref + href
return False
### functions for invoking nightly
def getAppInfo(self):
parser = ConfigParser()
ini_file = os.path.join(os.path.dirname(self.binary), "application.ini")
parser.read(ini_file)
try:
changeset = parser.get('App', 'SourceStamp')
repo = parser.get('App', 'SourceRepository')
return (repo, changeset)
except:
return ("", "")
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs, profile=profile)
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
class FirefoxLauncher:
def __init__(self, binary_path):
self.binary = binary_path
self._stopping = False
self.env = dict(os.environ)
# init gecko profiler
self.env.update({
'MOZ_PROFILER_STARTUP': '1',
'MOZ_PROFILER_STARTUP_INTERVAL': str(1),
'MOZ_PROFILER_STARTUP_ENTRIES': str(1000000)
})
def create_firefox_profile(self,
profile=None,
addons=(),
preferences=None,
clone=True):
if profile:
if not os.path.exists(profile):
LOG.warning(
"Creating directory '%s' to put the profile in there" %
profile)
os.makedirs(profile)
# since the user gave an empty dir for the profile,
# let's keep it on the disk in any case.
clone = False
if clone:
# mozprofile makes some changes in the profile that can not
# be undone. Let's clone the profile to not have side effect
# on existing profile.
# see https://bugzilla.mozilla.org/show_bug.cgi?id=999009
profile = FirefoxProfile.clone(profile,
addons=addons,
preferences=preferences)
else:
profile = FirefoxProfile(profile,
addons=addons,
preferences=preferences)
elif len(addons):
profile = FirefoxProfile(addons=addons, preferences=preferences)
else:
profile = FirefoxProfile(preferences=preferences)
return profile
def get_browser_viewport(self, x1=0, y1=0, x2=1024, y2=768):
# capture screen
img_obj = ImageGrab.grab((x1, y1, x2, y2))
# analyze viewport
current_viewport = find_image_viewport(img_obj)
return current_viewport
def is_browser_window_show_up(self):
expected_viewport_width = 1000
expected_viewport_height = 650
for _ in range(30):
# move window position
window_obj = WindowObject([
'Firefox.app', 'FirefoxNightly.app',
'FirefoxDeveloperEdition.app', 'Firefox Nightly'
])
window_obj.move_window_pos(0,
0,
window_height=768,
window_width=1024)
# get viewport
current_viewport = self.get_browser_viewport()
if current_viewport[
"height"] >= expected_viewport_height or current_viewport[
"width"] >= expected_viewport_width:
break
def start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self.create_firefox_profile(profile=profile,
addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def stop(self):
if mozinfo.os == "win":
# for some reason, stopping the runner may hang on windows. For
# example restart the browser in safe mode, it will hang for a
# couple of minutes. As a workaround, we call that in a thread and
# wait a bit for the completion. If the stop() can't complete we
# forgot about that thread.
thread = Thread(target=self.runner.stop)
thread.daemon = True
thread.start()
thread.join(0.7)
else:
self.runner.stop()
# release the runner since it holds a profile reference
del self.runner
class GeckoInstance(object):
required_prefs = {
"marionette.defaultPrefs.enabled": True,
"marionette.logging": True,
"startup.homepage_welcome_url": "about:blank",
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.sessionstore.resume_from_crash": False,
"browser.warnOnQuit": False,
"browser.displayedE10SPrompt": 5,
"browser.displayedE10SPrompt.1": 5,
"browser.displayedE10SPrompt.2": 5,
"browser.displayedE10SPrompt.3": 5,
"browser.displayedE10SPrompt.4": 5,
"browser.tabs.remote.autostart.1": False,
"browser.tabs.remote.autostart.2": False,
"dom.ipc.reportProcessHangs": False,
}
def __init__(self, host, port, bin, profile=None, app_args=None, symbols_path=None, gecko_log=None, prefs=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
# Check if it is a Profile object or a path to profile
self.profile = None
if isinstance(profile, Profile):
self.profile = profile
else:
self.profile_path = profile
self.prefs = prefs
self.required_prefs = deepcopy(GeckoInstance.required_prefs)
if prefs:
self.required_prefs.update(prefs)
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": deepcopy(self.required_prefs)}
profile_args["preferences"]["marionette.defaultPrefs.port"] = self.marionette_port
if self.prefs:
profile_args["preferences"].update(self.prefs)
if "-jsdebugger" in self.app_args:
profile_args["preferences"].update(
{
"devtools.browsertoolbox.panel": "jsdebugger",
"devtools.debugger.remote-enabled": True,
"devtools.debugger.chrome-enabled": True,
"devtools.chrome.enabled": True,
"devtools.debugger.prompt-connection": False,
"marionette.debugging.clicktostart": True,
}
)
if hasattr(self, "profile_path") and self.profile is None:
if not self.profile_path:
profile_args["restore"] = False
self.profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
self.profile = Profile.clone(**profile_args)
process_args = {"processOutputLine": [NullOutput()]}
if self.gecko_log == "-":
process_args["stream"] = sys.stdout
else:
if self.gecko_log is None:
self.gecko_log = "gecko.log"
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
if platform.system() is "Windows":
# NOTE: windows has a weird filesystem where it happily 'closes'
# the file, but complains if you try to delete it. You get a
# 'file still in use' error. Sometimes you can wait a bit and
# a retry will succeed.
# If all retries fail, we'll just continue without removing
# the file. In this case, if we are restarting the instance,
# then the new logs just get appended to the old file.
tries = 0
while tries < 10:
try:
os.remove(self.gecko_log)
break
except WindowsError as e:
if e.errno == errno.EACCES:
tries += 1
time.sleep(0.5)
else:
raise e
else:
os.remove(self.gecko_log)
process_args["logfile"] = self.gecko_log
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({"MOZ_CRASHREPORTER": "1", "MOZ_CRASHREPORTER_NO_REPORT": "1"})
self.runner = Runner(
binary=self.bin,
profile=self.profile,
cmdargs=["-no-remote", "-marionette"] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args=process_args,
)
self.runner.start()
def close(self, restart=False):
if not restart:
self.profile = None
if self.runner:
self.runner.stop()
self.runner.cleanup()
def restart(self, prefs=None, clean=True):
if clean:
self.profile.cleanup()
self.profile = None
self.close(restart=True)
if prefs:
self.prefs = prefs
else:
self.prefs = None
self.start()
class Nightly(object):
def __init__(self, repo_name=None):
platform=get_platform()
if platform['name'] == "Windows":
if platform['bits'] == '64':
print "No nightly builds available for 64 bit Windows"
sys.exit()
self.buildRegex = ".*win32.zip"
self.processName = self.name + ".exe"
self.binary = "moznightlyapp/" + self.name + "/" + self.name + ".exe"
elif platform['name'] == "Linux":
self.processName = self.name + "-bin"
self.binary = "moznightlyapp/" + self.name + "/" + self.name
if platform['bits'] == '64':
self.buildRegex = ".*linux-x86_64.tar.bz2"
else:
self.buildRegex = ".*linux-i686.tar.bz2"
elif platform['name'] == "Mac":
self.buildRegex = ".*mac.*\.dmg"
self.processName = self.name + "-bin"
self.binary = "moznightlyapp/Mozilla.app/Contents/MacOS/" + self.name + "-bin"
self.repo_name = repo_name
self._monthlinks = {}
self.lastdest = None
def cleanup(self):
rmtree('moznightlyapp')
if self.lastdest:
os.remove(self.lastdest)
__del__ = cleanup
def download(self, date=datetime.date.today(), dest=None):
url = self.getBuildUrl(date)
if url:
if not dest:
dest = os.path.basename(url)
print "Downloading nightly from %s" % date
if self.lastdest:
os.remove(self.lastdest)
download_url(url, dest)
self.dest = self.lastdest = dest
return True
else:
return False
def install(self):
rmtree("moznightlyapp")
subprocess._cleanup = lambda : None # mikeal's fix for subprocess threading bug
MozInstaller(src=self.dest, dest="moznightlyapp", dest_app="Mozilla.app")
return True
@staticmethod
def urlLinks(url):
res = [] # do not return a generator but an array, so we can store it for later use
h = httplib2.Http();
resp, content = h.request(url, "GET")
if resp.status != 200:
return res
soup = BeautifulSoup(content)
for link in soup.findAll('a'):
res.append(link)
return res
def getBuildUrl(self, date):
url = "http://ftp.mozilla.org/pub/mozilla.org/" + self.appName + "/nightly/"
year = str(date.year)
month = "%02d" % date.month
day = "%02d" % date.day
repo_name = self.repo_name or self.getRepoName(date)
url += year + "/" + month + "/"
linkRegex = '^' + year + '-' + month + '-' + day + '-' + '[\d-]+' + repo_name + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = self.urlLinks(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
for dirlink in monthlinks:
dirhref = dirlink.get("href")
if re.match(linkRegex, dirhref):
# now parse the page for the correct build url
for link in self.urlLinks(url + dirhref):
href = link.get("href")
if re.match(self.buildRegex, href):
return url + dirhref + href
return False
def getAppInfo(self):
parser = ConfigParser()
ini_file = os.path.join(os.path.dirname(self.binary), "application.ini")
parser.read(ini_file)
try:
changeset = parser.get('App', 'SourceStamp')
repo = parser.get('App', 'SourceRepository')
return (repo, changeset)
except:
return ("", "")
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs, profile=profile)
self.runner.names = [self.processName]
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
try:
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name
)
except Exception:
rmtree(self.tempdir)
raise
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True
)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def _wait(self):
return self.runner.wait()
def _stop(self):
if mozinfo.os == "win" and self.app_name == 'firefox':
# for some reason, stopping the runner may hang on windows. For
# example restart the browser in safe mode, it will hang for a
# couple of minutes. As a workaround, we call that in a thread and
# wait a bit for the completion. If the stop() can't complete we
# forgot about that thread.
thread = Thread(target=self.runner.stop)
thread.daemon = True
thread.start()
thread.join(0.7)
else:
self.runner.stop()
# release the runner since it holds a profile reference
del self.runner
def cleanup(self):
try:
Launcher.cleanup(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return safe_get_version(binary=self.binary)
class MozRunnerLauncher(Launcher):
tempdir = None
runner = None
app_name = 'undefined'
binary = None
def _install(self, dest):
self.tempdir = safe_mkdtemp()
try:
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name
)
except Exception:
rmtree(self.tempdir)
raise
def _disableUpdateByPolicy(self):
updatePolicy = {
'policies': {
'DisableAppUpdate': True
}
}
installdir = os.path.dirname(self.binary)
if mozinfo.os == 'mac':
# macOS has the following filestructure:
# binary at:
# PackageName.app/Contents/MacOS/firefox
# we need policies.json in:
# PackageName.app/Contents/Resources/distribution
installdir = os.path.normpath(
os.path.join(installdir, '..', 'Resources')
)
os.mkdir(os.path.join(installdir, 'distribution'))
policyFile = os.path.join(
installdir, 'distribution', 'policies.json'
)
with open(policyFile, 'w') as fp:
json.dump(updatePolicy, fp, indent=2)
def _start(self, profile=None, addons=(), cmdargs=(), preferences=None,
adb_profile_dir=None):
profile = self._create_profile(profile=profile, addons=addons,
preferences=preferences)
LOG.info("Launching %s" % self.binary)
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
def _on_exit():
# if we are stopping the process do not log anything.
if not self._stopping:
# mozprocess (behind mozrunner) fire 'onFinish'
# a bit early - let's ensure the process is finished.
# we have to call wait() directly on the subprocess
# instance of the ProcessHandler, else on windows
# None is returned...
# TODO: search that bug and fix that in mozprocess or
# mozrunner. (likely mozproces)
try:
exitcode = self.runner.process_handler.proc.wait()
except Exception:
print
LOG.error(
"Error while waiting process, consider filing a bug.",
exc_info=True
)
return
if exitcode != 0:
# first print a blank line, to be sure we don't
# write on an already printed line without EOL.
print
LOG.warning('Process exited with code %s' % exitcode)
self.runner.process_args = {
'processOutputLine': [get_default_logger("process").info],
'onFinish': _on_exit,
}
self.runner.start()
def _wait(self):
return self.runner.wait()
def _stop(self):
if mozinfo.os == "win" and self.app_name == 'firefox':
# for some reason, stopping the runner may hang on windows. For
# example restart the browser in safe mode, it will hang for a
# couple of minutes. As a workaround, we call that in a thread and
# wait a bit for the completion. If the stop() can't complete we
# forgot about that thread.
thread = Thread(target=self.runner.stop)
thread.daemon = True
thread.start()
thread.join(0.7)
else:
self.runner.stop()
# release the runner since it holds a profile reference
del self.runner
def cleanup(self):
try:
Launcher.cleanup(self)
finally:
# always remove tempdir
if self.tempdir is not None:
rmtree(self.tempdir)
def get_app_info(self):
return safe_get_version(binary=self.binary)
class Nightly(object):
name = None # abstract base class
_monthlinks = {}
lastdest = None
tempdir = None
app_name = None
profile_class = Profile
build_base_repo_name = "firefox"
@staticmethod
def _get_os_regex_suffix(bits, with_ext=True):
if mozinfo.os == "win":
if bits == 64:
suffix, ext = ".*win64-x86_64", ".zip"
else:
suffix, ext = ".*win32", ".zip"
elif mozinfo.os == "linux":
if bits == 64:
suffix, ext = ".*linux-x86_64", ".tar.bz2"
else:
suffix, ext = ".*linux-i686", ".tar.bz2"
elif mozinfo.os == "mac":
suffix, ext = r".*mac.*", "\.dmg"
if with_ext:
return '%s%s' % (suffix, ext)
else:
return suffix
@staticmethod
def _get_build_regex(name, bits, with_ext=True):
name_prefix = name if ".*%s" % name is not None else ''
suffix = Nightly._get_os_regex_suffix(bits, with_ext)
return "%s%s" % (name_prefix, suffix)
def __init__(self, inbound_branch=None, bits=mozinfo.bits, persist=None):
self.inbound_branch = inbound_branch
self.bits = bits
self.persist = persist
self.build_regex = self._get_build_regex(self.name, bits) + "$"
self.build_info_regex = \
self._get_build_regex(self.name, bits, with_ext=False) + "\.txt$"
def get_inbound_branch(self, date):
raise NotImplementedError
# cleanup functions
def remove_tempdir(self):
if self.tempdir:
rmtree(self.tempdir)
self.tempdir = None
def remove_lastdest(self):
if self.lastdest:
os.remove(self.lastdest)
self.lastdest = None
def cleanup(self):
self.remove_tempdir()
if not self.persist:
self.remove_lastdest()
__del__ = cleanup
# installation functions
def get_destination(self, url, date):
inbound_branch = self.get_inbound_branch(date)
dest = os.path.basename(url)
if self.persist is not None:
if hasattr(date, "strftime"):
date_str = date.strftime("%Y-%m-%d")
else:
date_str = date # Might be just a number with inbound
dest = os.path.join(self.persist,
"%s--%s--%s" % (date_str, inbound_branch, dest))
return dest
def download(self, date=datetime.date.today(), dest=None):
url = self.get_build_url(date)
if url:
if not dest:
dest = self.get_destination(url, date)
if not self.persist:
self.remove_lastdest()
if os.path.exists(dest):
print "Using local file: %s" % dest
else:
download_url(url, dest)
self.dest = self.lastdest = dest
return True
else:
return False
def install(self):
if not self.name:
raise NotImplementedError("Can't invoke abstract base class")
self.remove_tempdir()
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=self.dest, dest=self.tempdir),
self.name)
return True
def get_build_info(self, date):
url = self._get_build_url(date, self.build_info_regex, 'builds info')
if url is not None:
print "Getting %s" % url
response = requests.get(url)
if response.status_code == 200:
for line in response.text.splitlines():
if '/rev/' in line:
# returns [repository, changeset]
return line.split('/rev/')
def get_build_url(self, datestamp):
return self._get_build_url(datestamp, self.build_regex, 'builds')
def _get_build_url(self, datestamp, regex, what):
url = "http://ftp.mozilla.org/pub/mozilla.org/" + \
self.build_base_repo_name + "/nightly/"
year = str(datestamp.year)
month = "%02d" % datestamp.month
day = "%02d" % datestamp.day
inbound_branch = self.get_inbound_branch(datestamp)
url += year + "/" + month + "/"
link_regex = '^' + year + '-' + month + '-' + day + '-' \
+ r'[\d-]+' + inbound_branch + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = url_links(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
matches = []
for dirlink in monthlinks:
if re.match(link_regex, dirlink):
# now parse the page for the correct build url
for link in url_links(url + dirlink, regex=regex):
matches.append(url + dirlink + link)
if not matches:
print "Tried to get %s from %s that match '%s' but didn't find any." % \
(what, url, self.build_regex)
return None
else:
return sorted(matches)[-1] # the most recent build url
# functions for invoking nightly
def get_app_info(self):
return mozversion.get_version(binary=self.binary)
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profile_class(profile=profile, addons=addons)
elif len(addons):
profile = self.profile_class(addons=addons)
else:
profile = self.profile_class()
self.runner = Runner(binary=self.binary, cmdargs=cmdargs,
profile=profile)
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
class GeckoInstance(object):
required_prefs = {"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": 2828,
"marionette.logging": True,
"startup.homepage_welcome_url": "about:blank",
"browser.shell.checkDefaultBrowser": False,
"browser.startup.page": 0,
"browser.sessionstore.resume_from_crash": False,
"browser.warnOnQuit": False}
def __init__(self, host, port, bin, profile, app_args=None, symbols_path=None,
gecko_log=None):
self.marionette_host = host
self.marionette_port = port
self.bin = bin
self.profile_path = profile
self.app_args = app_args or []
self.runner = None
self.symbols_path = symbols_path
self.gecko_log = gecko_log
def start(self):
profile_args = {"preferences": self.required_prefs}
if not self.profile_path:
profile_args["restore"] = False
profile = Profile(**profile_args)
else:
profile_args["path_from"] = self.profile_path
profile = Profile.clone(**profile_args)
if self.gecko_log is None:
self.gecko_log = 'gecko.log'
elif os.path.isdir(self.gecko_log):
fname = "gecko-%d.log" % time.time()
self.gecko_log = os.path.join(self.gecko_log, fname)
self.gecko_log = os.path.realpath(self.gecko_log)
if os.access(self.gecko_log, os.F_OK):
os.remove(self.gecko_log)
env = os.environ.copy()
# environment variables needed for crashreporting
# https://developer.mozilla.org/docs/Environment_variables_affecting_crash_reporting
env.update({ 'MOZ_CRASHREPORTER': '1',
'MOZ_CRASHREPORTER_NO_REPORT': '1', })
self.runner = Runner(
binary=self.bin,
profile=profile,
cmdargs=['-no-remote', '-marionette'] + self.app_args,
env=env,
symbols_path=self.symbols_path,
process_args={
'processOutputLine': [NullOutput()],
'logfile': self.gecko_log})
self.runner.start()
def check_for_crashes(self):
return self.runner.check_for_crashes()
def close(self):
if self.runner:
self.runner.stop()
self.runner.cleanup()
class Nightly(object):
def __init__(self, repo_name=None):
platform = get_platform()
if platform['name'] == "Windows":
if platform['bits'] == '64':
print "No nightly builds available for 64 bit Windows"
sys.exit()
self.buildRegex = ".*win32.zip"
self.processName = self.name + ".exe"
self.binary = "moznightlyapp/" + self.name + "/" + self.name + ".exe"
elif platform['name'] == "Linux":
self.processName = self.name + "-bin"
self.binary = "moznightlyapp/" + self.name + "/" + self.name
if platform['bits'] == '64':
self.buildRegex = ".*linux-x86_64.tar.bz2"
else:
self.buildRegex = ".*linux-i686.tar.bz2"
elif platform['name'] == "Mac":
self.buildRegex = ".*mac.*\.dmg"
self.processName = self.name + "-bin"
self.binary = "moznightlyapp/Mozilla.app/Contents/MacOS/" + self.name + "-bin"
self.repo_name = repo_name
self._monthlinks = {}
self.lastdest = None
def cleanup(self):
rmtree('moznightlyapp')
if self.lastdest:
os.remove(self.lastdest)
__del__ = cleanup
def download(self, date=datetime.date.today(), dest=None):
url = self.getBuildUrl(date)
if url:
if not dest:
dest = os.path.basename(url)
print "Downloading nightly from %s" % date
if self.lastdest:
os.remove(self.lastdest)
download_url(url, dest)
self.dest = self.lastdest = dest
return True
else:
return False
def install(self):
rmtree("moznightlyapp")
subprocess._cleanup = lambda: None # mikeal's fix for subprocess threading bug
MozInstaller(src=self.dest,
dest="moznightlyapp",
dest_app="Mozilla.app")
return True
@staticmethod
def urlLinks(url):
res = [
] # do not return a generator but an array, so we can store it for later use
h = httplib2.Http()
resp, content = h.request(url, "GET")
if resp.status != 200:
return res
soup = BeautifulSoup(content)
for link in soup.findAll('a'):
res.append(link)
return res
def getBuildUrl(self, date):
url = "http://ftp.mozilla.org/pub/mozilla.org/" + self.appName + "/nightly/"
year = str(date.year)
month = "%02d" % date.month
day = "%02d" % date.day
repo_name = self.repo_name or self.getRepoName(date)
url += year + "/" + month + "/"
linkRegex = '^' + year + '-' + month + '-' + day + '-' + '[\d-]+' + repo_name + '/$'
cachekey = year + '-' + month
if cachekey in self._monthlinks:
monthlinks = self._monthlinks[cachekey]
else:
monthlinks = self.urlLinks(url)
self._monthlinks[cachekey] = monthlinks
# first parse monthly list to get correct directory
for dirlink in monthlinks:
dirhref = dirlink.get("href")
if re.match(linkRegex, dirhref):
# now parse the page for the correct build url
for link in self.urlLinks(url + dirhref):
href = link.get("href")
if re.match(self.buildRegex, href):
return url + dirhref + href
return False
def getAppInfo(self):
parser = ConfigParser()
ini_file = os.path.join(os.path.dirname(self.binary),
"application.ini")
parser.read(ini_file)
try:
changeset = parser.get('App', 'SourceStamp')
repo = parser.get('App', 'SourceRepository')
return (repo, changeset)
except:
return ("", "")
def start(self, profile, addons, cmdargs):
if profile:
profile = self.profileClass(profile=profile, addons=addons)
elif len(addons):
profile = self.profileClass(addons=addons)
else:
profile = self.profileClass()
self.runner = Runner(binary=self.binary,
cmdargs=cmdargs,
profile=profile)
self.runner.names = [self.processName]
self.runner.start()
return True
def stop(self):
self.runner.stop()
def wait(self):
self.runner.wait()
