def test_symlink_objdir(self):
"""Objdir that is a symlink is loaded properly."""
d = os.path.realpath(tempfile.mkdtemp())
try:
topobjdir_real = os.path.join(d, 'objdir')
topobjdir_link = os.path.join(d, 'objlink')
os.mkdir(topobjdir_real)
os.symlink(topobjdir_real, topobjdir_link)
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write('mk_add_options MOZ_OBJDIR=%s' % topobjdir_link)
mozinfo = os.path.join(topobjdir_real, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=d,
mozconfig=mozconfig,
), fh)
os.chdir(topobjdir_link)
obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
self.assertEqual(obj.topobjdir, topobjdir_real)
os.chdir(topobjdir_real)
obj = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
self.assertEqual(obj.topobjdir, topobjdir_real)
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def test_objdir_mismatch(self):
"""Ensure MachCommandBase throwing on objdir mismatch."""
d = os.path.realpath(tempfile.mkdtemp())
try:
real_topobjdir = os.path.join(d, 'real-objdir')
os.makedirs(real_topobjdir)
topobjdir = os.path.join(d, 'objdir')
os.makedirs(topobjdir)
topsrcdir = os.path.join(d, 'srcdir')
os.makedirs(topsrcdir)
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write('mk_add_options MOZ_OBJDIR=%s' % real_topobjdir)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=topsrcdir,
mozconfig=mozconfig,
), fh)
os.chdir(topobjdir)
with self.assertRaises(ObjdirMismatchException):
MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def main(platform):
build = MozbuildObject.from_environment()
topsrcdir = build.topsrcdir
distdir = build.distdir
srcdir = os.path.join(topsrcdir, "b2g", "simulator")
app_buildid = open(os.path.join(build.topobjdir, "config", "buildid")).read().strip()
# The simulator uses a shorter version string,
# it only keeps the major version digits A.B
# whereas MOZ_B2G_VERSION is A.B.C.D
b2g_version = build.config_environment.defines["MOZ_B2G_VERSION"].replace('"', '')
version = ".".join(str(n) for n in LooseVersion(b2g_version).version[0:2])
# Build a gaia profile specific to the simulator in order to:
# - disable the FTU
# - set custom prefs to enable devtools debugger server
# - set custom settings to disable lockscreen and screen timeout
# - only ship production apps
gaia_path = build.config_environment.substs["GAIADIR"]
builder = GaiaBuilder(build, gaia_path)
builder.clean()
env = {
"NOFTU": "1",
"GAIA_APP_TARGET": "production",
"SETTINGS_PATH": os.path.join(srcdir, "custom-settings.json")
}
builder.profile(env)
builder.override_prefs(os.path.join(srcdir, "custom-prefs.js"))
# Substitute version strings in the package manifest overload file
manifest_overload = os.path.join(build.topobjdir, "b2g", "simulator", "package-overload.json")
process_package_overload(os.path.join(srcdir, "package-overload.json.in"),
manifest_overload,
version,
app_buildid)
# Build the simulator addon xpi
xpi_name = XPI_NAME % {"version": version, "platform": platform}
xpi_path = os.path.join(distdir, xpi_name)
update_path = "%s/%s" % (version, platform)
update_link = UPDATE_LINK % {"update_path": update_path, "xpi_name": xpi_name}
update_url = UPDATE_URL % {"update_path": update_path}
subprocess.check_call([
build.virtualenv_manager.python_path, os.path.join(topsrcdir, "addon-sdk", "source", "bin", "cfx"), "xpi", \
"--pkgdir", srcdir, \
"--manifest-overload", manifest_overload, \
"--strip-sdk", \
"--update-link", update_link, \
"--update-url", update_url, \
"--static-args", "{\"label\": \"Firefox OS %s\"}" % version, \
"--output-file", xpi_path \
])
# Ship b2g-desktop, but prevent its gaia profile to be shipped in the xpi
add_dir_to_zip(xpi_path, os.path.join(distdir, "b2g"), "b2g", ("gaia", "B2G.app/Contents/MacOS/gaia"))
# Then ship our own gaia profile
add_dir_to_zip(xpi_path, os.path.join(gaia_path, "profile"), "profile")
def mozregression_create_parser():
# Create the mozregression command line parser.
# if mozregression is not installed, or not up to date, it will
# first be installed.
cmd = MozbuildObject.from_environment()
cmd._activate_virtualenv()
mozregression = mozregression_import()
if not mozregression:
# mozregression is not here at all, install it
cmd.virtualenv_manager.install_pip_package('mozregression')
print("mozregression was installed. please re-run your"
" command. If you keep getting this message please "
" manually run: 'pip install -U mozregression'.")
else:
# check if there is a new release available
release = mozregression.new_release_on_pypi()
if release:
print(release)
# there is one, so install it. Note that install_pip_package
# does not work here, so just run pip directly.
cmd.virtualenv_manager._run_pip([
'install',
'mozregression==%s' % release
])
print("mozregression was updated to version %s. please"
" re-run your command." % release)
else:
# mozregression is up to date, return the parser.
return mozregression.parser()
# exit if we updated or installed mozregression because
# we may have already imported mozregression and running it
# as this may cause issues.
sys.exit(0)
def get_parser():
here = os.path.abspath(os.path.dirname(__file__))
build_obj = MozbuildObject.from_environment(cwd=here)
if conditions.is_android(build_obj):
return reftestcommandline.RemoteArgumentsParser()
else:
return reftestcommandline.DesktopArgumentsParser()
def get_properties_db_from_xpcshell(self):
"""Generate the static css properties db for devtools from an xpcshell script."""
build = MozbuildObject.from_environment()
# Get the paths
script_path = resolve_path(self.topsrcdir,
'devtools/shared/css/generated/generate-properties-db.js')
gre_path = resolve_path(self.topobjdir, 'dist/bin')
browser_path = resolve_path(self.topobjdir, 'dist/bin/browser')
xpcshell_path = build.get_binary_path(what='xpcshell')
print(browser_path)
sub_env = dict(os.environ)
if sys.platform.startswith('linux'):
sub_env["LD_LIBRARY_PATH"] = gre_path
# Run the xcpshell script, and set the appdir flag to the browser path so that
# we have the proper dependencies for requiring the loader.
contents = subprocess.check_output([xpcshell_path, '-g', gre_path,
'-a', browser_path, script_path],
env = sub_env)
# Extract just the output between the delimiters as the xpcshell output can
# have extra output that we don't want.
contents = contents.split('DEVTOOLS_CSS_DB_DELIMITER')[1]
return json.loads(contents)
def test_relative_objdir(self):
"""Relative defined objdirs are loaded properly."""
d = os.path.realpath(tempfile.mkdtemp())
try:
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write('mk_add_options MOZ_OBJDIR=./objdir')
topobjdir = mozpath.join(d, 'objdir')
os.mkdir(topobjdir)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=d,
mozconfig=mozconfig,
), fh)
os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8')
child = os.path.join(topobjdir, 'foo', 'bar')
os.makedirs(child)
os.chdir(child)
obj = MozbuildObject.from_environment(
detect_virtualenv_mozinfo=False)
self.assertEqual(obj.topobjdir, topobjdir)
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def setup_argument_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
build_path = os.path.join(build_obj.topobjdir, 'build')
if build_path not in sys.path:
sys.path.append(build_path)
mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest')
with warnings.catch_warnings():
warnings.simplefilter('ignore')
import imp
path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py')
if not os.path.exists(path):
path = os.path.join(here, "runtests.py")
with open(path, 'r') as fh:
imp.load_module('mochitest', fh, path,
('.py', 'r', imp.PY_SOURCE))
from mochitest_options import MochitestArgumentParser
if conditions.is_android(build_obj):
# On Android, check for a connected device (and offer to start an
# emulator if appropriate) before running tests. This check must
# be done in this admittedly awkward place because
# MochitestArgumentParser initialization fails if no device is found.
from mozrunner.devices.android_device import verify_android_device
verify_android_device(build_obj, install=True, xre=True)
global parser
parser = MochitestArgumentParser()
return parser
def find_and_update_from_json(*dirs):
"""
Find a mozinfo.json file, load it, and update the info with the
contents.
:param dirs: Directories in which to look for the file. They will be
searched after first looking in the root of the objdir
if the current script is being run from a Mozilla objdir.
Returns the full path to mozinfo.json if it was found, or None otherwise.
"""
# First, see if we're in an objdir
try:
from mozbuild.base import MozbuildObject, BuildEnvironmentNotFoundException
from mozbuild.mozconfig import MozconfigFindException
build = MozbuildObject.from_environment()
json_path = _os.path.join(build.topobjdir, "mozinfo.json")
if _os.path.isfile(json_path):
update(json_path)
return json_path
except ImportError:
pass
except (BuildEnvironmentNotFoundException, MozconfigFindException):
pass
for d in dirs:
d = _os.path.abspath(d)
json_path = _os.path.join(d, "mozinfo.json")
if _os.path.isfile(json_path):
update(json_path)
return json_path
return None
def get_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
if conditions.is_android(build_obj):
return parser_remote()
elif conditions.is_b2g(build_obj):
return parser_b2g()
else:
return parser_desktop()
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._cmd = MozbuildObject.from_environment()
# This is a 'group' dependency - All rules that list this as an output
# will be built before any rules that list this as an input.
self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
def test_objdir_is_srcdir_rejected(self):
"""Ensure the srcdir configurations are rejected."""
d = os.path.realpath(tempfile.mkdtemp())
try:
# The easiest way to do this is to create a mozinfo.json with data
# that will never happen.
mozinfo = os.path.join(d, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump({'topsrcdir': d}, fh)
os.chdir(d)
with self.assertRaises(BadEnvironmentException):
MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
finally:
shutil.rmtree(d)
def test_info(self, **params):
import which
from mozbuild.base import MozbuildObject
self.branches = params['branches']
self.start = params['start']
self.end = params['end']
self.show_info = params['show_info']
self.show_results = params['show_results']
self.show_durations = params['show_durations']
self.show_bugs = params['show_bugs']
self.verbose = params['verbose']
if (not self.show_info and
not self.show_results and
not self.show_durations and
not self.show_bugs):
# by default, show everything
self.show_info = True
self.show_results = True
self.show_durations = True
self.show_bugs = True
here = os.path.abspath(os.path.dirname(__file__))
build_obj = MozbuildObject.from_environment(cwd=here)
self._hg = None
if conditions.is_hg(build_obj):
if self._is_windows():
self._hg = which.which('hg.exe')
else:
self._hg = which.which('hg')
self._git = None
if conditions.is_git(build_obj):
if self._is_windows():
self._git = which.which('git.exe')
else:
self._git = which.which('git')
for test_name in params['test_names']:
print("===== %s =====" % test_name)
self.test_name = test_name
if len(self.test_name) < 6:
print("'%s' is too short for a test name!" % self.test_name)
continue
if self.show_info:
self.set_test_name()
if self.show_results:
self.report_test_results()
if self.show_durations:
self.report_test_durations()
if self.show_bugs:
self.report_bugs()
def main(platform):
build = MozbuildObject.from_environment()
topsrcdir = build.topsrcdir
distdir = build.distdir
srcdir = os.path.join(topsrcdir, "b2g", "simulator")
app_buildid = open(os.path.join(build.topobjdir, "config", "buildid")).read().strip()
# The simulator uses a shorter version string,
# it only keeps the major version digits A.B
# whereas MOZ_B2G_VERSION is A.B.C.D
b2g_version = build.config_environment.defines["MOZ_B2G_VERSION"].replace('"', "")
version = ".".join(str(n) for n in LooseVersion(b2g_version).version[0:2])
# Build a gaia profile specific to the simulator in order to:
# - disable the FTU
# - set custom prefs to enable devtools debugger server
# - set custom settings to disable lockscreen and screen timeout
# - only ship production apps
gaia_path = build.config_environment.substs["GAIADIR"]
builder = GaiaBuilder(build, gaia_path)
builder.clean()
env = {"NOFTU": "1", "GAIA_APP_TARGET": "production", "SETTINGS_PATH": os.path.join(srcdir, "custom-settings.json")}
builder.profile(env)
builder.override_prefs(os.path.join(srcdir, "custom-prefs.js"))
# Build the simulator addon xpi
xpi_name = XPI_NAME % {"version": version, "platform": platform}
xpi_path = os.path.join(distdir, xpi_name)
update_path = "%s/%s" % (version, platform)
update_url = UPDATE_URL % {"update_path": update_path}
# Preprocess some files...
manifest = os.path.join(build.topobjdir, "b2g", "simulator", "install.rdf")
preprocess_file(os.path.join(srcdir, "install.rdf.in"), manifest, version, app_buildid, update_url)
with JarWriter(xpi_path, optimize=False) as zip:
# Ship addon files into the .xpi
add_file_to_zip(zip, manifest, "install.rdf")
add_file_to_zip(zip, os.path.join(srcdir, "bootstrap.js"), "bootstrap.js")
add_file_to_zip(zip, os.path.join(srcdir, "icon.png"), "icon.png")
add_file_to_zip(zip, os.path.join(srcdir, "icon64.png"), "icon64.png")
# Ship b2g-desktop, but prevent its gaia profile to be shipped in the xpi
add_dir_to_zip(
zip,
os.path.join(distdir, "b2g"),
"b2g",
("gaia", "B2G.app/Contents/MacOS/gaia", "B2G.app/Contents/Resources/gaia"),
)
# Then ship our own gaia profile
add_dir_to_zip(zip, os.path.join(gaia_path, "profile"), "profile")
def make_dmg(source_directory, output_dmg):
build = MozbuildObject.from_environment()
extra_files = [
(os.path.join(build.distdir, 'branding', 'dsstore'), '.DS_Store'),
(os.path.join(build.distdir, 'branding', 'background.png'),
'.background/background.png'),
(os.path.join(build.distdir, 'branding', 'disk.icns'),
'.VolumeIcon.icns'),
]
volume_name = build.substs['MOZ_APP_DISPLAYNAME']
dmg.create_dmg(source_directory, output_dmg, volume_name, extra_files)
def scheduled_by_push(self, repository, revision):
changed_files = files_changed.get_changed_files(repository, revision)
mbo = MozbuildObject.from_environment()
# the decision task has a sparse checkout, so, mozbuild_reader will use
# a MercurialRevisionFinder with revision '.', which should be the same
# as `revision`; in other circumstances, it will use a default reader
rdr = mbo.mozbuild_reader(config_mode='empty')
components = set()
for p, m in rdr.files_info(changed_files).items():
components |= set(m['SCHEDULES'].components)
return components
def test_filesystem_traversal_reading(self):
"""Reading moz.build according to filesystem traversal works.
We attempt to read every known moz.build file via filesystem traversal.
If this test fails, it means that metadata extraction will fail.
"""
mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
config = mb.config_environment
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
paths, contexts = reader.read_relevant_mozbuilds(all_paths)
self.assertEqual(set(paths), all_paths)
self.assertGreaterEqual(len(contexts), len(paths))
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
config = EmptyConfig(build_obj.topsrcdir)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
def main(argv):
logging.basicConfig(format='%(levelname)s - %(threadName)s - %(message)s')
parser = argparse.ArgumentParser(
description='Upload generated source files in ARTIFACT to BUCKET in S3.')
parser.add_argument('artifact',
help='generated-sources artifact from build task')
args = parser.parse_args(argv)
region, bucket = get_s3_region_and_bucket()
config = MozbuildObject.from_environment()
config._activate_virtualenv()
config.virtualenv_manager.install_pip_package('boto3==1.4.4')
with timed() as elapsed:
do_work(region=region, bucket=bucket, artifact=args.artifact)
log.info('Finished in {:.03f}s'.format(elapsed()))
return 0
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._cmd = MozbuildObject.from_environment()
self._manifest_entries = OrderedDefaultDict(set)
self._compile_env_gen_files = (
'*.c',
'*.cpp',
'*.h',
'*.inc',
'*.py',
'*.rs',
)
# These are 'group' dependencies - All rules that list these as an output
# will be built before any rules that list this as an input.
self._installed_idls = '$(MOZ_OBJ_ROOT)/<installed-idls>'
self._installed_files = '$(MOZ_OBJ_ROOT)/<installed-files>'
def test_objdir_config_status(self):
"""Ensure @CONFIG_GUESS@ is handled when loading mozconfig."""
base = self.get_base()
cmd = base._normalize_command(
[os.path.join(topsrcdir, 'build', 'autoconf', 'config.guess')],
True)
guess = subprocess.check_output(cmd, cwd=topsrcdir).strip()
# There may be symlinks involved, so we use real paths to ensure
# path consistency.
d = os.path.realpath(tempfile.mkdtemp())
try:
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/@CONFIG_GUESS@')
print('Wrote mozconfig %s' % mozconfig)
topobjdir = os.path.join(d, 'foo', guess)
os.makedirs(topobjdir)
# Create a fake topsrcdir.
guess_path = os.path.join(d, 'build', 'autoconf', 'config.guess')
os.makedirs(os.path.dirname(guess_path))
shutil.copy(os.path.join(topsrcdir, 'build', 'autoconf',
'config.guess',), guess_path)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=d,
mozconfig=mozconfig,
), fh)
os.environ[b'MOZCONFIG'] = mozconfig.encode('utf-8')
os.chdir(topobjdir)
obj = MozbuildObject.from_environment(
detect_virtualenv_mozinfo=False)
self.assertEqual(obj.topobjdir, mozpath.normsep(topobjdir))
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def test_orphan_file_patterns(self):
if sys.platform == 'win32':
raise unittest.SkipTest('failing on windows builds')
mb = MozbuildObject.from_environment(detect_virtualenv_mozinfo=False)
try:
config = mb.config_environment
except Exception as e:
if e.message == 'config.status not available. Run configure.':
raise unittest.SkipTest('failing without config.status')
raise
if config.substs['MOZ_BUILD_APP'] == 'js':
raise unittest.SkipTest('failing in Spidermonkey builds')
reader = BuildReader(config)
all_paths = self._mozbuilds(reader)
_, contexts = reader.read_relevant_mozbuilds(all_paths)
finder = FileFinder(config.topsrcdir, find_executables=False,
ignore=['obj*'])
def pattern_exists(pat):
return [p for p in finder.find(pat)] != []
for ctx in contexts:
if not isinstance(ctx, Files):
continue
relsrcdir = ctx.relsrcdir
if not pattern_exists(os.path.join(relsrcdir, ctx.pattern)):
self.fail("The pattern '%s' in a Files() entry in "
"'%s' corresponds to no files in the tree.\n"
"Please update this entry." %
(ctx.pattern, ctx.main_path))
test_files = ctx['IMPACTED_TESTS'].files
for p in test_files:
if not pattern_exists(os.path.relpath(p.full_path, config.topsrcdir)):
self.fail("The pattern '%s' in a dependent tests entry "
"in '%s' corresponds to no files in the tree.\n"
"Please update this entry." %
(p, ctx.main_path))
def post_dispatch_handler(context, handler, instance, result,
start_time, end_time, args):
"""Perform global operations after command dispatch.
For now, we will use this to handle build system telemetry.
"""
# Don't do anything when...
if should_skip_dispatch(context, handler):
return
# We have not opted-in to telemetry
if not context.settings.build.telemetry:
return
from mozbuild.telemetry import gather_telemetry
from mozbuild.base import MozbuildObject
if not isinstance(instance, MozbuildObject):
instance = MozbuildObject.from_environment()
try:
substs = instance.substs
except Exception:
substs = {}
# We gather telemetry for every operation...
gather_telemetry(command=handler.name, success=(result == 0),
start_time=start_time, end_time=end_time,
mach_context=context, substs=substs,
paths=[instance.topsrcdir, instance.topobjdir])
# But only submit about every n-th operation
if random.randint(1, TELEMETRY_SUBMISSION_FREQUENCY) != 1:
return
with open(os.devnull, 'wb') as devnull:
subprocess.Popen([sys.executable,
os.path.join(topsrcdir, 'build',
'submit_telemetry_data.py'),
get_state_dir()[0]],
stdout=devnull, stderr=devnull)
def setup_argument_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
build_path = os.path.join(build_obj.topobjdir, 'build')
if build_path not in sys.path:
sys.path.append(build_path)
mochitest_dir = os.path.join(build_obj.topobjdir, '_tests', 'testing', 'mochitest')
with warnings.catch_warnings():
warnings.simplefilter('ignore')
import imp
path = os.path.join(build_obj.topobjdir, mochitest_dir, 'runtests.py')
with open(path, 'r') as fh:
imp.load_module('mochitest', fh, path,
('.py', 'r', imp.PY_SOURCE))
from mochitest_options import MochitestArgumentParser
return MochitestArgumentParser()
def test_objdir_config_status(self):
"""Ensure @CONFIG_GUESS@ is handled when loading mozconfig."""
base = self.get_base()
guess = base._config_guess
# There may be symlinks involved, so we use real paths to ensure
# path consistency.
d = os.path.realpath(tempfile.mkdtemp())
try:
mozconfig = os.path.join(d, 'mozconfig')
with open(mozconfig, 'wt') as fh:
fh.write('mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/foo/@CONFIG_GUESS@')
print('Wrote mozconfig %s' % mozconfig)
topobjdir = os.path.join(d, 'foo', guess)
os.makedirs(topobjdir)
# Create a fake topsrcdir.
guess_path = os.path.join(d, 'build', 'autoconf', 'config.guess')
os.makedirs(os.path.dirname(guess_path))
shutil.copy(os.path.join(topsrcdir, 'build', 'autoconf',
'config.guess',), guess_path)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(
topsrcdir=d,
mozconfig=mozconfig,
), fh)
os.environ[b'MOZCONFIG'] = mozconfig
os.chdir(topobjdir)
obj = MozbuildObject.from_environment()
self.assertEqual(obj.topobjdir, topobjdir)
finally:
shutil.rmtree(d)
def gen_test_backend():
build_obj = MozbuildObject.from_environment()
try:
config = build_obj.config_environment
except BuildEnvironmentNotFoundException:
print("No build detected, test metadata may be incomplete.")
# If 'JS_STANDALONE' is set, tests that don't require an objdir won't
# be picked up due to bug 1345209.
substs = EmptyConfig.default_substs
if 'JS_STANDALONE' in substs:
del substs['JS_STANDALONE']
config = EmptyConfig(build_obj.topsrcdir, substs)
config.topobjdir = build_obj.topobjdir
reader = BuildReader(config)
emitter = TreeMetadataEmitter(config)
backend = TestManifestBackend(config)
context = reader.read_topsrcdir()
data = emitter.emit(context, emitfn=emitter._process_test_manifests)
backend.consume(data)
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import mozinfo
import moznetwork
import optparse
import os
import tempfile
from automationutils import addCommonOptions, isURL
from mozprofile import DEFAULT_PORTS
here = os.path.abspath(os.path.dirname(__file__))
try:
from mozbuild.base import MozbuildObject
build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
build_obj = None
__all__ = ["MochitestOptions", "B2GOptions"]
VMWARE_RECORDING_HELPER_BASENAME = "vmwarerecordinghelper"
class MochitestOptions(optparse.OptionParser):
"""Usage instructions for runtests.py.
All arguments are optional.
If --chrome is specified, chrome tests will be run instead of web content tests.
If --browser-chrome is specified, browser-chrome tests will be run instead of web content tests.
See <http://mochikit.com/doc/html/MochiKit/Logging.html> for details on the logging levels.
"""
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._cmd = MozbuildObject.from_environment()
def profile_data_dir():
build = MozbuildObject.from_environment(cwd=here)
return os.path.join(build.topsrcdir, "testing", "profiles")
def __init__(self, **kwargs):
super(ReftestArgumentsParser, self).__init__(**kwargs)
# Try to import a MozbuildObject. Success indicates that we are
# running from a source tree. This allows some defaults to be set
# from the source tree.
try:
from mozbuild.base import MozbuildObject
self.build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
self.build_obj = None
self.add_argument("--xre-path",
action="store",
type=str,
dest="xrePath",
# individual scripts will set a sane default
default=None,
help="absolute path to directory containing XRE (probably xulrunner)")
self.add_argument("--symbols-path",
action="store",
type=str,
dest="symbolsPath",
default=None,
help="absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
self.add_argument("--debugger",
action="store",
dest="debugger",
help="use the given debugger to launch the application")
self.add_argument("--debugger-args",
action="store",
dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_argument("--debugger-interactive",
action="store_true",
dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_argument("--appname",
action="store",
type=str,
dest="app",
default=None,
help="absolute path to application, overriding default")
self.add_argument("--extra-profile-file",
action="append",
dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_argument("--timeout",
action="store",
dest="timeout",
type=int,
default=5 * 60, # 5 minutes per bug 479518
help="reftest will timeout in specified number of seconds. [default %(default)s].")
self.add_argument("--leak-threshold",
action="store",
type=int,
dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_argument("--utility-path",
action="store",
type=str,
dest="utilityPath",
default=self.build_obj.bindir if self.build_obj else None,
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
self.add_argument("--total-chunks",
type=int,
dest="totalChunks",
help="how many chunks to split the tests up into")
self.add_argument("--this-chunk",
type=int,
dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
self.add_argument("--log-file",
action="store",
type=str,
dest="logFile",
default=None,
help="file to log output to in addition to stdout")
self.add_argument("--skip-slow-tests",
dest="skipSlowTests",
action="store_true",
default=False,
help="skip tests marked as slow when running")
self.add_argument("--ignore-window-size",
dest="ignoreWindowSize",
action="store_true",
default=False,
help="ignore the window size, which may cause spurious failures and passes")
self.add_argument("--install-extension",
action="append",
dest="extensionsToInstall",
default=[],
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
self.add_argument("--setenv",
action="append",
type=str,
default=[],
dest="environment",
metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
self.add_argument("--filter",
action="store",
type=str,
dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
self.add_argument("--shuffle",
action="store_true",
default=False,
dest="shuffle",
help="run reftests in random order")
self.add_argument("--run-until-failure",
action="store_true",
default=False,
dest="runUntilFailure",
help="stop running on the first failure. Useful for RR recordings.")
self.add_argument("--repeat",
action="store",
type=int,
default=0,
dest="repeat",
help="number of times the select test(s) will be executed. Useful for "
"finding intermittent failures.")
self.add_argument("--focus-filter-mode",
action="store",
type=str,
dest="focusFilterMode",
default="all",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
self.add_argument("--e10s",
action="store_true",
default=False,
dest="e10s",
help="enables content processes")
self.add_argument("--setpref",
action="append",
type=str,
default=[],
dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
self.add_argument("--reftest-extension-path",
action="store",
dest="reftestExtensionPath",
help="Path to the reftest extension")
self.add_argument("--special-powers-extension-path",
action="store",
dest="specialPowersExtensionPath",
help="Path to the special powers extension")
self.add_argument("--suite",
choices=["reftest", "crashtest", "jstestbrowser"],
default=None,
help=argparse.SUPPRESS)
self.add_argument("tests",
metavar="TEST_PATH",
nargs="*",
help="Path to test file, manifest file, or directory containing tests")
mozlog.commandline.add_logging_group(self)
def main():
config = MozbuildObject.from_environment()
config.activate_virtualenv()
logging.basicConfig()
parser = argparse.ArgumentParser(
description=
'Upload symbols in ZIP using token from Taskcluster secrets service.')
parser.add_argument(
'archive', help='Symbols archive file - URL or path to local file')
parser.add_argument('--ignore-missing',
help='No error on missing files',
action='store_true')
args = parser.parse_args()
def check_file_exists(url):
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
try:
resp = requests.head(url, allow_redirects=True)
return resp.status_code == requests.codes.ok
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
return False
zip_path = args.archive
if args.archive.endswith('.tar.zst'):
from mozpack.files import File
from mozpack.mozjar import JarWriter
import gzip
import tarfile
import tempfile
config._ensure_zstd()
import zstandard
def prepare_zip_from(archive, tmpdir):
if archive.startswith('http'):
resp = requests.get(archive, allow_redirects=True, stream=True)
resp.raise_for_status()
reader = resp.raw
# Work around taskcluster generic-worker possibly gzipping the tar.zst.
if resp.headers.get('Content-Encoding') == 'gzip':
reader = gzip.GzipFile(fileobj=reader)
else:
reader = open(archive, 'rb')
ctx = zstandard.ZstdDecompressor()
uncompressed = ctx.stream_reader(reader)
with tarfile.open(mode='r|',
fileobj=uncompressed,
bufsize=1024 * 1024) as tar:
while True:
info = tar.next()
if info is None:
break
log.info(info.name)
data = tar.extractfile(info)
path = os.path.join(tmpdir, info.name.lstrip('/'))
if info.name.endswith('.dbg'):
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wb') as fh:
with gzip.GzipFile(fileobj=fh,
mode='wb',
compresslevel=5) as c:
shutil.copyfileobj(data, c)
jar.add(info.name + '.gz', File(path), compress=False)
elif info.name.endswith('.dSYM.tar'):
import bz2
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wb') as fh:
c = bz2.BZ2Compressor()
while True:
buf = data.read(16384)
if not buf:
break
fh.write(c.compress(buf))
fh.write(c.flush())
jar.add(info.name + '.bz2', File(path), compress=False)
elif info.name.endswith('.pdb'):
import subprocess
makecab = os.environ.get('MAKECAB', 'makecab')
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'wb') as fh:
shutil.copyfileobj(data, fh)
subprocess.check_call([
makecab, '-D', 'CompressionType=MSZIP', path,
path + '_'
],
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
jar.add(info.name[:-1] + '_',
File(path + '_'),
compress=False)
else:
jar.add(info.name, data)
reader.close()
tmpdir = tempfile.TemporaryDirectory()
zip_path = os.path.join(tmpdir.name, 'symbols.zip')
log.info('Preparing symbol archive "{0}" from "{1}"'.format(
zip_path, args.archive))
is_existing = False
try:
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
with JarWriter(zip_path, compress_level=5) as jar:
try:
prepare_zip_from(args.archive, tmpdir.name)
is_existing = True
break
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
except Exception:
os.remove(zip_path)
raise
elif args.archive.startswith('http'):
is_existing = check_file_exists(args.archive)
else:
is_existing = os.path.isfile(args.archive)
if not is_existing:
if args.ignore_missing:
log.info('Archive file "{0}" does not exist!'.format(args.archive))
return 0
else:
log.error('Error: archive file "{0}" does not exist!'.format(
args.archive))
return 1
secret_name = os.environ.get('SYMBOL_SECRET')
if secret_name is not None:
auth_token = get_taskcluster_secret(secret_name)
elif 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' in os.environ:
token_file = os.environ['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
log.error('SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.
format(token_file))
return 1
auth_token = open(token_file, 'r').read().strip()
else:
log.error(
'You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE '
'environment variables!')
return 1
# Allow overwriting of the upload url with an environmental variable
if 'SOCORRO_SYMBOL_UPLOAD_URL' in os.environ:
url = os.environ['SOCORRO_SYMBOL_UPLOAD_URL']
else:
url = DEFAULT_URL
log.info('Uploading symbol file "{0}" to "{1}"'.format(zip_path, url))
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
log.info('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
if zip_path.startswith('http'):
zip_arg = {'data': {'url': zip_path}}
else:
zip_arg = {'files': {'symbols.zip': open(zip_path, 'rb')}}
r = requests.post(
url,
headers={'Auth-Token': auth_token},
allow_redirects=False,
# Allow a longer read timeout because uploading by URL means the server
# has to fetch the entire zip file, which can take a while. The load balancer
# in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
timeout=(300, 300),
**zip_arg)
# 429 or any 5XX is likely to be a transient failure.
# Break out for success or other error codes.
if r.ok or (r.status_code < 500 and r.status_code != 429):
break
print_error(r)
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
else:
log.warn('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
log.info('Uploaded successfully!')
return 0
print_error(r)
return 1
def get_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
if conditions.is_android(build_obj) or build_obj.substs.get('MOZ_BUILD_APP') == 'b2g':
return parser_remote()
else:
return parser_desktop()
def environment(self, verbose=False):
import platform
print('platform:\n\t%s' % platform.platform())
print('python version:\n\t%s' % sys.version)
print('python prefix:\n\t%s' % sys.prefix)
print('mach cwd:\n\t%s' % self._mach_context.cwd)
print('os cwd:\n\t%s' % os.getcwd())
print('mach directory:\n\t%s' % self._mach_context.topdir)
print('state directory:\n\t%s' % self._mach_context.state_dir)
try:
mb = MozbuildObject.from_environment(cwd=self._mach_context.cwd)
except ObjdirMismatchException as e:
print('Ambiguous object directory detected. We detected that '
'both %s and %s could be object directories. This is '
'typically caused by having a mozconfig pointing to a '
'different object directory from the current working '
'directory. To solve this problem, ensure you do not have a '
'default mozconfig in searched paths.' %
(e.objdir1, e.objdir2))
return 1
mozconfig = None
try:
mozconfig = mb.mozconfig
print('mozconfig path:\n\t%s' % mozconfig['path'])
except MozconfigFindException as e:
print('Unable to find mozconfig: %s' % e.message)
return 1
except MozconfigLoadException as e:
print('Error loading mozconfig: %s' % e.path)
print(e.message)
if e.output:
print('mozconfig evaluation output:')
for line in e.output:
print(line)
return 1
print('object directory:\n\t%s' % mb.topobjdir)
if mozconfig:
print('mozconfig configure args:')
if mozconfig['configure_args']:
for arg in mozconfig['configure_args']:
print('\t%s' % arg)
print('mozconfig extra make args:')
if mozconfig['make_extra']:
for arg in mozconfig['make_extra']:
print('\t%s' % arg)
print('mozconfig make flags:')
if mozconfig['make_flags']:
for arg in mozconfig['make_flags']:
print('\t%s' % arg)
config = None
try:
config = mb.config_environment
except Exception:
pass
if config:
print('config topsrcdir:\n\t%s' % config.topsrcdir)
print('config topobjdir:\n\t%s' % config.topobjdir)
if verbose:
print('config substitutions:')
for k in sorted(config.substs):
print('\t%s: %s' % (k, config.substs[k]))
print('config defines:')
for k in sorted(config.defines):
print('\t%s' % k)
def main(platform):
build = MozbuildObject.from_environment()
topsrcdir = build.topsrcdir
distdir = build.distdir
srcdir = os.path.join(topsrcdir, "b2g", "simulator")
app_buildid = open(os.path.join(build.topobjdir, "config",
"buildid")).read().strip()
# The simulator uses a shorter version string,
# it only keeps the major version digits A.B
# whereas MOZ_B2G_VERSION is A.B.C.D
b2g_version = build.config_environment.defines["MOZ_B2G_VERSION"].replace(
'"', '')
version = ".".join(str(n) for n in LooseVersion(b2g_version).version[0:2])
# Build a gaia profile specific to the simulator in order to:
# - disable the FTU
# - set custom prefs to enable devtools debugger server
# - set custom settings to disable lockscreen and screen timeout
# - only ship production apps
gaia_path = build.config_environment.substs["GAIADIR"]
builder = GaiaBuilder(build, gaia_path)
builder.clean()
env = {
"NOFTU": "1",
"GAIA_APP_TARGET": "production",
"SETTINGS_PATH": os.path.join(srcdir, "custom-settings.json")
}
builder.profile(env)
builder.override_prefs(os.path.join(srcdir, "custom-prefs.js"))
# Build the simulator addon xpi
xpi_name = XPI_NAME % {"version": version, "platform": platform}
xpi_path = os.path.join(distdir, xpi_name)
update_path = "%s/%s" % (version, platform)
update_url = UPDATE_URL % {"update_path": update_path}
# Preprocess some files...
manifest = os.path.join(build.topobjdir, "b2g", "simulator", "install.rdf")
preprocess_file(os.path.join(srcdir, "install.rdf.in"), manifest, version,
app_buildid, update_url)
options_file = os.path.join(build.topobjdir, "b2g", "simulator",
"options.xul")
preprocess_file(os.path.join(srcdir, "options.xul.in"), options_file,
version, app_buildid, update_url)
with JarWriter(xpi_path, optimize=False) as zip:
# Ship addon files into the .xpi
add_dir_to_zip(zip, os.path.join(srcdir, "lib"), "lib")
add_file_to_zip(zip, manifest, "install.rdf")
add_file_to_zip(zip, os.path.join(srcdir, "bootstrap.js"),
"bootstrap.js")
add_file_to_zip(zip, options_file, "options.xul")
add_file_to_zip(zip, os.path.join(srcdir, "icon.png"), "icon.png")
add_file_to_zip(zip, os.path.join(srcdir, "icon64.png"), "icon64.png")
# Ship b2g-desktop, but prevent its gaia profile to be shipped in the xpi
add_dir_to_zip(zip, os.path.join(distdir, "b2g"), "b2g",
("gaia", "B2G.app/Contents/MacOS/gaia"))
# Then ship our own gaia profile
add_dir_to_zip(zip, os.path.join(gaia_path, "profile"), "profile")
def __init__(self, verbose):
self.verbose = verbose
here = os.path.abspath(os.path.dirname(__file__))
self.build_obj = MozbuildObject.from_environment(cwd=here)
self.total_activedata_seconds = 0
from mozdevice import DroidADB, DroidSUT
from mozprofile import DEFAULT_PORTS
import mozinfo
import mozlog
import moznetwork
here = os.path.abspath(os.path.dirname(__file__))
try:
from mozbuild.base import (
MozbuildObject,
MachCommandConditions as conditions,
)
build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
build_obj = None
conditions = None
def get_default_valgrind_suppression_files():
# We are trying to locate files in the source tree. So if we
# don't know where the source tree is, we must give up.
#
# When this is being run by |mach mochitest --valgrind ...|, it is
# expected that |build_obj| is not None, and so the logic below will
# select the correct suppression files.
#
# When this is run from mozharness, |build_obj| is None, and we expect
# that testing/mozharness/configs/unittests/linux_unittests.py will
def _finalize_telemetry_legacy(context, instance, handler, success, start_time,
end_time, topsrcdir):
"""Record and submit legacy telemetry.
Parameterized by the raw gathered telemetry, this function handles persisting and
submission of the data.
This has been designated as "legacy" telemetry because modern telemetry is being
submitted with "Glean".
"""
from mozboot.util import get_state_dir
from mozbuild.base import MozbuildObject
from mozbuild.telemetry import gather_telemetry
from mach.telemetry import (is_telemetry_enabled,
is_applicable_telemetry_environment)
if not (is_applicable_telemetry_environment()
and is_telemetry_enabled(context.settings)):
return
if not isinstance(instance, MozbuildObject):
instance = MozbuildObject.from_environment()
command_attrs = getattr(context, 'command_attrs', {})
# We gather telemetry for every operation.
data = gather_telemetry(command=handler.name,
success=success,
start_time=start_time,
end_time=end_time,
mach_context=context,
instance=instance,
command_attrs=command_attrs)
if data:
telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
try:
os.mkdir(telemetry_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
try:
os.mkdir(outgoing_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(os.path.join(outgoing_dir,
str(uuid.uuid4()) + '.json'), 'w') as f:
json.dump(data, f, sort_keys=True)
# The user is performing a maintenance command, skip the upload
if handler.name in (
'bootstrap',
'doctor',
'mach-commands',
'vcs-setup',
'create-mach-environment',
'install-moz-phab',
# We call mach environment in client.mk which would cause the
# data submission to block the forward progress of make.
'environment'):
return False
if 'TEST_MACH_TELEMETRY_NO_SUBMIT' in os.environ:
# In our telemetry tests, we want telemetry to be collected for analysis, but
# we don't want it submitted.
return False
state_dir = get_state_dir()
machpath = os.path.join(instance.topsrcdir, 'mach')
with open(os.devnull, 'wb') as devnull:
subprocess.Popen([
sys.executable, machpath, 'python', '--no-virtualenv',
os.path.join(topsrcdir, 'build', 'submit_telemetry_data.py'),
state_dir
],
stdout=devnull,
stderr=devnull)
import threading
import os
import sys
import shutil
import tempfile
from datetime import datetime
from mozbuild.base import MozbuildObject
from buildconfig import substs
PORT = 8888
if __name__ == '__main__':
cli = CLI()
debug_args, interactive = cli.debugger_arguments()
build = MozbuildObject.from_environment()
httpd = MozHttpd(port=PORT,
docroot=os.path.join(build.topsrcdir, "build", "pgo"))
httpd.start(block=False)
locations = ServerLocations()
locations.add_host(host='127.0.0.1',
port=PORT,
options='primary,privileged')
#TODO: mozfile.TemporaryDirectory
profilePath = tempfile.mkdtemp()
try:
#TODO: refactor this into mozprofile
prefpath = os.path.join(build.topsrcdir, "testing", "profiles", "prefs_general.js")
prefs = {}
def get_parser():
build_obj = MozbuildObject.from_environment(cwd=here)
if conditions.is_android(build_obj):
return parser_remote()
else:
return parser_desktop()
def main():
config = MozbuildObject.from_environment()
config._activate_virtualenv()
import redo
import requests
logging.basicConfig()
parser = argparse.ArgumentParser(
description=
'Upload symbols in ZIP using token from Taskcluster secrets service.')
parser.add_argument('zip',
help='Symbols zip file - URL or path to local file')
args = parser.parse_args()
if not args.zip.startswith('http') and not os.path.isfile(args.zip):
log.error('Error: zip file "{0}" does not exist!'.format(args.zip))
return 1
secret_name = os.environ.get('SYMBOL_SECRET')
if secret_name is not None:
auth_token = get_taskcluster_secret(secret_name)
elif 'SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE' in os.environ:
token_file = os.environ['SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE']
if not os.path.isfile(token_file):
log.error('SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE "{0}" does not exist!'.
format(token_file))
return 1
auth_token = open(token_file, 'r').read().strip()
else:
log.error(
'You must set the SYMBOL_SECRET or SOCORRO_SYMBOL_UPLOAD_TOKEN_FILE '
'environment variables!')
return 1
# Allow overwriting of the upload url with an environmental variable
if 'SOCORRO_SYMBOL_UPLOAD_URL' in os.environ:
url = os.environ['SOCORRO_SYMBOL_UPLOAD_URL']
else:
url = DEFAULT_URL
log.info('Uploading symbol file "{0}" to "{1}"'.format(args.zip, url))
for i, _ in enumerate(redo.retrier(attempts=MAX_RETRIES), start=1):
log.info('Attempt %d of %d...' % (i, MAX_RETRIES))
try:
if args.zip.startswith('http'):
zip_arg = {'data': {'url': args.zip}}
else:
zip_arg = {'files': {'symbols.zip': open(args.zip, 'rb')}}
r = requests.post(
url,
headers={'Auth-Token': auth_token},
allow_redirects=False,
# Allow a longer read timeout because uploading by URL means the server
# has to fetch the entire zip file, which can take a while. The load balancer
# in front of symbols.mozilla.org has a 300 second timeout, so we'll use that.
timeout=(10, 300),
**zip_arg)
# 500 is likely to be a transient failure.
# Break out for success or other error codes.
if r.status_code < 500:
break
print_error(r)
except requests.exceptions.RequestException as e:
log.error('Error: {0}'.format(e))
log.info('Retrying...')
else:
log.warn('Maximum retries hit, giving up!')
return 1
if r.status_code >= 200 and r.status_code < 300:
log.info('Uploaded successfully!')
return 0
print_error(r)
return 1
import logging
import os
import sys
from collections import defaultdict
from mozbuild.base import MozbuildObject
from mozlint.pathutils import findobject
from mozlint.parser import Parser
from mozlint.result import ResultSummary
from mozlog.structuredlog import StructuredLogger
from mozpack import path
import pytest
here = path.abspath(path.dirname(__file__))
build = MozbuildObject.from_environment(cwd=here,
virtualenv_name="python-test")
lintdir = path.dirname(here)
sys.path.insert(0, lintdir)
logger = logging.getLogger("mozlint")
def pytest_generate_tests(metafunc):
"""Finds, loads and returns the config for the linter name specified by the
LINTER global variable in the calling module.
This implies that each test file (that uses this fixture) should only be
used to test a single linter. If no LINTER variable is defined, the test
will fail.
"""
if "config" in metafunc.fixturenames:
def __init__(self, **kwargs):
super(ReftestArgumentsParser, self).__init__(**kwargs)
# Try to import a MozbuildObject. Success indicates that we are
# running from a source tree. This allows some defaults to be set
# from the source tree.
try:
from mozbuild.base import MozbuildObject
self.build_obj = MozbuildObject.from_environment(cwd=here)
except ImportError:
self.build_obj = None
self.add_argument(
"--xre-path",
action="store",
type=str,
dest="xrePath",
# individual scripts will set a sane default
default=None,
help=
"absolute path to directory containing XRE (probably xulrunner)")
self.add_argument(
"--symbols-path",
action="store",
type=str,
dest="symbolsPath",
default=None,
help=
"absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols"
)
self.add_argument(
"--debugger",
action="store",
dest="debugger",
help="use the given debugger to launch the application")
self.add_argument("--debugger-args",
action="store",
dest="debuggerArgs",
help="pass the given args to the debugger _before_ "
"the application on the command line")
self.add_argument("--debugger-interactive",
action="store_true",
dest="debuggerInteractive",
help="prevents the test harness from redirecting "
"stdout and stderr for interactive debuggers")
self.add_argument(
"--appname",
action="store",
type=str,
dest="app",
default=None,
help="absolute path to application, overriding default")
self.add_argument("--extra-profile-file",
action="append",
dest="extraProfileFiles",
default=[],
help="copy specified files/dirs to testing profile")
self.add_argument(
"--timeout",
action="store",
dest="timeout",
type=int,
default=5 * 60, # 5 minutes per bug 479518
help=
"reftest will timeout in specified number of seconds. [default %(default)s]."
)
self.add_argument("--leak-threshold",
action="store",
type=int,
dest="defaultLeakThreshold",
default=0,
help="fail if the number of bytes leaked in default "
"processes through refcounted objects (or bytes "
"in classes with MOZ_COUNT_CTOR and MOZ_COUNT_DTOR) "
"is greater than the given number")
self.add_argument(
"--utility-path",
action="store",
type=str,
dest="utilityPath",
default=self.build_obj.bindir if self.build_obj else None,
help="absolute path to directory containing utility "
"programs (xpcshell, ssltunnel, certutil)")
self.add_argument("--total-chunks",
type=int,
dest="totalChunks",
help="how many chunks to split the tests up into")
self.add_argument(
"--this-chunk",
type=int,
dest="thisChunk",
help="which chunk to run between 1 and --total-chunks")
self.add_argument("--log-file",
action="store",
type=str,
dest="logFile",
default=None,
help="file to log output to in addition to stdout")
self.add_argument("--skip-slow-tests",
dest="skipSlowTests",
action="store_true",
default=False,
help="skip tests marked as slow when running")
self.add_argument(
"--ignore-window-size",
dest="ignoreWindowSize",
action="store_true",
default=False,
help=
"ignore the window size, which may cause spurious failures and passes"
)
self.add_argument(
"--install-extension",
action="append",
dest="extensionsToInstall",
default=[],
help="install the specified extension in the testing profile. "
"The extension file's name should be <id>.xpi where <id> is "
"the extension's id as indicated in its install.rdf. "
"An optional path can be specified too.")
self.add_argument("--setenv",
action="append",
type=str,
default=[],
dest="environment",
metavar="NAME=VALUE",
help="sets the given variable in the application's "
"environment")
self.add_argument(
"--filter",
action="store",
type=str,
dest="filter",
help="specifies a regular expression (as could be passed to the JS "
"RegExp constructor) to test against URLs in the reftest manifest; "
"only test items that have a matching test URL will be run.")
self.add_argument("--shuffle",
action="store_true",
default=False,
dest="shuffle",
help="run reftests in random order")
self.add_argument(
"--run-until-failure",
action="store_true",
default=False,
dest="runUntilFailure",
help="stop running on the first failure. Useful for RR recordings."
)
self.add_argument(
"--repeat",
action="store",
type=int,
default=0,
dest="repeat",
help=
"number of times the select test(s) will be executed. Useful for "
"finding intermittent failures.")
self.add_argument(
"--focus-filter-mode",
action="store",
type=str,
dest="focusFilterMode",
default="all",
help="filters tests to run by whether they require focus. "
"Valid values are `all', `needs-focus', or `non-needs-focus'. "
"Defaults to `all'.")
self.add_argument("--e10s",
action="store_true",
default=False,
dest="e10s",
help="enables content processes")
self.add_argument("--setpref",
action="append",
type=str,
default=[],
dest="extraPrefs",
metavar="PREF=VALUE",
help="defines an extra user preference")
self.add_argument("--reftest-extension-path",
action="store",
dest="reftestExtensionPath",
help="Path to the reftest extension")
self.add_argument("--special-powers-extension-path",
action="store",
dest="specialPowersExtensionPath",
help="Path to the special powers extension")
self.add_argument("--suite",
choices=["reftest", "crashtest", "jstestbrowser"],
default=None,
help=argparse.SUPPRESS)
self.add_argument(
"tests",
metavar="TEST_PATH",
nargs="*",
help=
"Path to test file, manifest file, or directory containing tests")
def post_dispatch_handler(context, handler, instance, result,
start_time, end_time, depth, args):
"""Perform global operations after command dispatch.
For now, we will use this to handle build system telemetry.
"""
# Don't write telemetry data if this mach command was invoked as part of another
# mach command.
if depth != 1 or os.environ.get('MACH_MAIN_PID') != str(os.getpid()):
return
from mozbuild.telemetry import is_telemetry_enabled
if not is_telemetry_enabled(context.settings):
return
from mozbuild.telemetry import gather_telemetry
from mozbuild.base import MozbuildObject
import mozpack.path as mozpath
if not isinstance(instance, MozbuildObject):
instance = MozbuildObject.from_environment()
try:
substs = instance.substs
except Exception:
substs = {}
command_attrs = getattr(context, 'command_attrs', {})
# We gather telemetry for every operation.
paths = {
instance.topsrcdir: '$topsrcdir/',
instance.topobjdir: '$topobjdir/',
mozpath.normpath(os.path.expanduser('~')): '$HOME/',
}
# This might override one of the existing entries, that's OK.
# We don't use a sigil here because we treat all arguments as potentially relative
# paths, so we'd like to get them back as they were specified.
paths[mozpath.normpath(os.getcwd())] = ''
data = gather_telemetry(command=handler.name, success=(result == 0),
start_time=start_time, end_time=end_time,
mach_context=context, substs=substs,
command_attrs=command_attrs, paths=paths)
if data:
telemetry_dir = os.path.join(get_state_dir(), 'telemetry')
try:
os.mkdir(telemetry_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
try:
os.mkdir(outgoing_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
'w') as f:
json.dump(data, f, sort_keys=True)
if should_skip_telemetry_submission(handler):
return True
state_dir = get_state_dir()
machpath = os.path.join(instance.topsrcdir, 'mach')
with open(os.devnull, 'wb') as devnull:
subprocess.Popen([sys.executable, machpath, 'python',
'--no-virtualenv',
os.path.join(topsrcdir, 'build',
'submit_telemetry_data.py'),
state_dir],
stdout=devnull, stderr=devnull)
print("Importing private key {} to database".format(key_pem))
status = runUtil(pk12util, [
"-i", p12, "-d", "dbm:" + srcDir, "-w", pwfile.name, "-k",
pwfile.name
])
if status:
return status
if key_type == "ca":
shutil.copyfile(cert_pem,
os.path.join(srcDir, "{}.ca".format(name)))
elif key_type == "client":
shutil.copyfile(
p12, os.path.join(srcDir, "{}.client".format(name)))
elif key_type == "server":
pass # Nothing to do for server keys
else:
raise Exception(
"State error: Unknown keyspec key_type: {}".format(
key_type))
return 0
build = MozbuildObject.from_environment()
certdir = os.path.join(build.topsrcdir, "build", "pgo", "certs")
certificateStatus = constructCertDatabase(build, certdir)
if certificateStatus:
print "TEST-UNEXPECTED-FAIL | SSL Server Certificate generation"
sys.exit(certificateStatus)
def _init(self):
CommonBackend._init(self)
self._backend_files = {}
self._cmd = MozbuildObject.from_environment()
def _run(self, argv):
# Load settings as early as possible so things in dispatcher.py
# can use them.
for provider in Registrar.settings_providers:
self.settings.register_provider(provider)
self.load_settings(self.settings_paths)
context = CommandContext(cwd=self.cwd,
settings=self.settings,
log_manager=self.log_manager,
commands=Registrar)
if self.populate_context_handler:
self.populate_context_handler(context)
context = ContextWrapper(context, self.populate_context_handler)
parser = self.get_argument_parser(context)
if not len(argv):
# We don't register the usage until here because if it is globally
# registered, argparse always prints it. This is not desired when
# running with --help.
parser.usage = Mach.USAGE
parser.print_usage()
return 0
try:
args = parser.parse_args(argv)
except NoCommandError:
print(NO_COMMAND_ERROR)
return 1
except UnknownCommandError as e:
suggestion_message = SUGGESTED_COMMANDS_MESSAGE % (
e.verb, ', '.join(
e.suggested_commands)) if e.suggested_commands else ''
print(UNKNOWN_COMMAND_ERROR %
(e.verb, e.command, suggestion_message))
return 1
except UnrecognizedArgumentError as e:
print(UNRECOGNIZED_ARGUMENT_ERROR %
(e.command, ' '.join(e.arguments)))
return 1
# Add JSON logging to a file if requested.
if args.logfile:
self.log_manager.add_json_handler(args.logfile)
# Up the logging level if requested.
log_level = logging.INFO
if args.verbose:
log_level = logging.DEBUG
self.log_manager.register_structured_logger(logging.getLogger('mach'))
write_times = True
if args.log_no_times or 'MACH_NO_WRITE_TIMES' in os.environ:
write_times = False
# Always enable terminal logging. The log manager figures out if we are
# actually in a TTY or are a pipe and does the right thing.
self.log_manager.add_terminal_logging(level=log_level,
write_interval=args.log_interval,
write_times=write_times)
if args.settings_file:
# Argument parsing has already happened, so settings that apply
# to command line handling (e.g alias, defaults) will be ignored.
self.load_settings(args.settings_file)
def _check_debugger(program):
"""Checks if debugger specified in command line is installed.
Uses mozdebug to locate debuggers.
If the call does not raise any exceptions, mach is permitted
to continue execution.
Otherwise, mach execution is halted.
Args:
program (str): debugger program name.
"""
import mozdebug
info = mozdebug.get_debugger_info(program)
if info is None:
print(
"Specified debugger '{}' is not found.\n".format(program) +
"Is it installed? Is it in your PATH?")
sys.exit(1)
# For the codepath where ./mach <test_type> --debugger=<program>,
# assert that debugger value exists first, then check if installed on system.
if (hasattr(args.command_args, "debugger")
and getattr(args.command_args, "debugger") is not None):
_check_debugger(getattr(args.command_args, "debugger"))
# For the codepath where ./mach test --debugger=<program> <test_type>,
# debugger must be specified from command line with the = operator.
# Otherwise, an IndexError is raised, which is converted to an exit code of 1.
elif (hasattr(args.command_args, "extra_args")
and getattr(args.command_args, "extra_args")):
extra_args = getattr(args.command_args, "extra_args")
try:
debugger = [
ea.split("=")[1] for ea in extra_args if "debugger" in ea
]
except IndexError:
print(
"Debugger must be specified with '=' when invoking ./mach test.\n"
+ "Please correct the command and try again.")
sys.exit(1)
if debugger:
_check_debugger(''.join(debugger))
if not hasattr(args, 'mach_handler'):
raise MachError('ArgumentParser result missing mach handler info.')
handler = getattr(args, 'mach_handler')
# if --disable-tests flag was enabled in the mozconfig used to compile
# the build, tests will be disabled.
# instead of trying to run nonexistent tests then reporting a failure,
# this will prevent mach from progressing beyond this point.
if handler.category == 'testing':
from mozbuild.base import BuildEnvironmentNotFoundException
try:
from mozbuild.base import MozbuildObject
# all environments should have an instance of build object.
build = MozbuildObject.from_environment()
if build is not None and hasattr(build, 'mozconfig'):
ac_options = build.mozconfig['configure_args']
if ac_options and '--disable-tests' in ac_options:
print(
'Tests have been disabled by mozconfig with the flag'
+ '"ac_add_options --disable-tests".\n' +
'Remove the flag, and re-compile to enable tests.')
return 1
except BuildEnvironmentNotFoundException:
# likely automation environment, so do nothing.
pass
try:
return Registrar._run_command_handler(
handler,
context=context,
debug_command=args.debug_command,
**vars(args.command_args))
except KeyboardInterrupt as ki:
raise ki
except FailedCommandError as e:
print(e.message)
return e.exit_code
except Exception as e:
exc_type, exc_value, exc_tb = sys.exc_info()
# The first two frames are us and are never used.
stack = traceback.extract_tb(exc_tb)[2:]
# If we have nothing on the stack, the exception was raised as part
# of calling the @Command method itself. This likely means a
# mismatch between @CommandArgument and arguments to the method.
# e.g. there exists a @CommandArgument without the corresponding
# argument on the method. We handle that here until the module
# loader grows the ability to validate better.
if not len(stack):
print(COMMAND_ERROR)
self._print_exception(sys.stdout, exc_type, exc_value,
traceback.extract_tb(exc_tb))
return 1
# Split the frames into those from the module containing the
# command and everything else.
command_frames = []
other_frames = []
initial_file = stack[0][0]
for frame in stack:
if frame[0] == initial_file:
command_frames.append(frame)
else:
other_frames.append(frame)
# If the exception was in the module providing the command, it's
# likely the bug is in the mach command module, not something else.
# If there are other frames, the bug is likely not the mach
# command's fault.
self._print_error_header(argv, sys.stdout)
if len(other_frames):
print(MODULE_ERROR)
else:
print(COMMAND_ERROR)
self._print_exception(sys.stdout, exc_type, exc_value, stack)
return 1
