def run_tests(self, **kwargs):
MachCommandBase._activate_virtualenv(self)
from pathlib import Path
from mozperftest.runner import _setup_path
from mozperftest.utils import install_package, temporary_env
# include in sys.path all deps
_setup_path()
try:
import black # noqa
except ImportError:
# install the tests scripts in the virtualenv
# this step will be removed once Bug 1635573 is done
# so we don't have to hit pypi or our internal mirror at all
pydeps = Path(self.topsrcdir, "third_party", "python")
for name in (
# installing pyrsistent and attrs so we don't pick newer ones
str(pydeps / "pyrsistent"),
str(pydeps / "attrs"),
"coverage",
"black",
"flake8",
):
install_package(self.virtualenv_manager, name)
here = Path(__file__).parent.resolve()
if not ON_TRY:
# formatting the code with black
assert self._run_python_script("black", str(here))
# checking flake8 correctness
assert self._run_python_script("flake8", str(here))
# running pytest with coverage
# coverage is done in three steps:
# 1/ coverage erase => erase any previous coverage data
# 2/ coverage run pytest ... => run the tests and collect info
# 3/ coverage report => generate the report
tests = here / "tests"
import pytest
with temporary_env(COVERAGE_RCFILE=str(here / ".coveragerc")):
assert self._run_python_script("coverage",
"erase",
label="remove old coverage data")
args = [
"run",
pytest.__file__,
"-xs",
str(tests.resolve()),
]
assert self._run_python_script("coverage",
*args,
label="running tests")
if not self._run_python_script("coverage", "report", display=True):
raise ValueError("Coverage is too low!")
def run_tests(self, **kwargs):
MachCommandBase.activate_virtualenv(self)
from pathlib import Path
from mozperftest.utils import temporary_env
with temporary_env(COVERAGE_RCFILE=str(Path(HERE, ".coveragerc")),
RUNNING_TESTS="YES"):
self._run_tests(**kwargs)
def activate_browsertime_virtualenv(command_context, *args, **kwargs):
r"""Activates virtualenv.
This function will also install Pillow and pyssim if needed.
It will raise an error in case the install failed.
"""
MachCommandBase.activate_virtualenv(command_context, *args, **kwargs)
# installing Python deps on the fly
for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
if _need_install(command_context, dep):
command_context.virtualenv_manager._run_pip(["install", dep])
def _activate_virtualenv(self, *args, **kwargs):
r'''Activates virtualenv.
This function will also install Pillow and pyssim if needed.
It will raise an error in case the install failed.
'''
MachCommandBase._activate_virtualenv(self, *args, **kwargs)
# installing Python deps on the fly
for dep in ("Pillow==%s" % PILLOW_VERSION, "pyssim==%s" % PYSSIM_VERSION):
if self._need_install(dep):
self.virtualenv_manager._run_pip(["install", dep])
def _activate_virtualenv(self, *args, **kwargs):
MachCommandBase._activate_virtualenv(self, *args, **kwargs)
try:
self.virtualenv_manager.install_pip_package('Pillow==6.0.0')
except Exception:
print('Could not install Pillow from pip.')
return 1
try:
self.virtualenv_manager.install_pip_package('pyssim==0.4')
except Exception:
print('Could not install pyssim from pip.')
return 1
def test_mach_command_base_inside_objdir(self):
"""Ensure a MachCommandBase constructed from inside the objdir works."""
d = os.path.realpath(tempfile.mkdtemp())
try:
topobjdir = os.path.join(d, 'objdir')
os.makedirs(topobjdir)
topsrcdir = os.path.join(d, 'srcdir')
os.makedirs(topsrcdir)
mozinfo = os.path.join(topobjdir, 'mozinfo.json')
with open(mozinfo, 'wt') as fh:
json.dump(dict(topsrcdir=topsrcdir, ), fh)
os.chdir(topobjdir)
class MockMachContext(object):
pass
context = MockMachContext()
context.cwd = topobjdir
context.topdir = topsrcdir
context.settings = None
context.log_manager = None
o = MachCommandBase(context)
self.assertEqual(o.topobjdir, topobjdir)
self.assertEqual(o.topsrcdir, topsrcdir)
finally:
shutil.rmtree(d)
def _get_command(command=mozperftest.mach_commands.run_perftest):
from mozbuild.base import MozbuildObject
from mozperftest.argparser import PerftestArgumentParser
config = MozbuildObject.from_environment()
class context:
topdir = config.topobjdir
cwd = os.getcwd()
settings = {}
log_manager = mock.Mock()
state_dir = tempfile.mkdtemp()
# used to make arguments passed by the test as
# being set by the user.
def _run_perftest(func):
def _run(command_context, **kwargs):
parser.set_by_user = list(kwargs.keys())
return func(command_context, **kwargs)
return _run
try:
command_context = MachCommandBase(context())
parser = PerftestArgumentParser()
if command == mozperftest.mach_commands.run_perftest:
command = _run_perftest(command)
with mock.patch("mozperftest.mach_commands.get_parser", new=lambda: parser):
yield command, command_context
finally:
shutil.rmtree(context.state_dir)
def test_objdir_mismatch(self):
"""Ensure MachCommandBase throwing on objdir mismatch."""
d = os.path.realpath(tempfile.mkdtemp())
try:
real_topobjdir = os.path.join(d, "real-objdir")
os.makedirs(real_topobjdir)
topobjdir = os.path.join(d, "objdir")
os.makedirs(topobjdir)
topsrcdir = os.path.join(d, "srcdir")
prepare_tmp_topsrcdir(topsrcdir)
mozconfig = os.path.join(d, "mozconfig")
with open(mozconfig, "wt") as fh:
fh.write("mk_add_options MOZ_OBJDIR=%s" % real_topobjdir)
mozinfo = os.path.join(topobjdir, "mozinfo.json")
with open(mozinfo, "wt") as fh:
json.dump(
dict(
topsrcdir=topsrcdir,
mozconfig=mozconfig,
),
fh,
)
os.chdir(topobjdir)
class MockMachContext(object):
pass
context = MockMachContext()
context.cwd = topobjdir
context.topdir = topsrcdir
context.settings = None
context.log_manager = None
context.detect_virtualenv_mozinfo = False
stdout = sys.stdout
sys.stdout = StringIO()
try:
with self.assertRaises(SystemExit):
MachCommandBase(context, None)
self.assertTrue(
sys.stdout.getvalue().startswith(
"Ambiguous object directory detected."
)
)
finally:
sys.stdout = stdout
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def run_perftest(self, **kwargs):
push_to_try = kwargs.pop("push_to_try", False)
if push_to_try:
from pathlib import Path
sys.path.append(str(Path(self.topsrcdir, "tools", "tryselect")))
from tryselect.push import push_to_try
platform = kwargs.pop("try_platform")
if platform not in _TRY_PLATFORMS:
# we can extend platform support here: linux, win, macOs, pixel2
# by adding more jobs in taskcluster/ci/perftest/kind.yml
# then picking up the right one here
raise NotImplementedError("%r not supported yet" % platform)
perftest_parameters = {}
parser = get_perftest_parser()()
for name, value in kwargs.items():
# ignore values that are set to default
if parser.get_default(name) == value:
continue
perftest_parameters[name] = value
parameters = {
"try_task_config": {
"tasks": [_TRY_PLATFORMS[platform]],
"perftest-options": perftest_parameters,
},
"try_mode": "try_task_config",
}
task_config = {"parameters": parameters, "version": 2}
push_to_try("perftest", "perftest", try_task_config=task_config)
return
# run locally
MachCommandBase._activate_virtualenv(self)
from mozperftest.runner import run_tests
run_tests(mach_cmd=self, **kwargs)
def _activate_virtualenv(self, *args, **kwargs):
r'''Activates virtualenv.
This function will also install Pillow and pyssim if needed.
It will raise an error in case the install failed.
'''
MachCommandBase._activate_virtualenv(self, *args, **kwargs)
# installing Python deps on the fly
try:
import PIL
if PIL.__version__ != PILLOW_VERSION:
raise ImportError("Wrong version %s" % PIL.__version__)
except ImportError:
self.virtualenv_manager.install_pip_package('Pillow==%s' % PILLOW_VERSION)
try:
# No __version__ in that package.
# We make the assumption it's fine.
import ssim # noqa
except ImportError:
self.virtualenv_manager.install_pip_package('pyssim==%s' % PYSSIM_VERSION)
def main(argv=sys.argv[1:]):
"""Used when the runner is directly called from the shell
"""
_setup_path()
from mozbuild.base import MachCommandBase, MozbuildObject
from mozperftest import PerftestArgumentParser
from mozboot.util import get_state_dir
config = MozbuildObject.from_environment()
config.topdir = config.topsrcdir
config.cwd = os.getcwd()
config.state_dir = get_state_dir()
mach_cmd = MachCommandBase(config)
parser = PerftestArgumentParser(description="vanilla perftest")
args = parser.parse_args(args=argv)
run_tests(mach_cmd, **dict(args._get_kwargs()))
def main(argv=sys.argv[1:]):
"""Used when the runner is directly called from the shell
"""
_setup_path()
from mozbuild.mozconfig import MozconfigLoader
from mozbuild.base import MachCommandBase, MozbuildObject
from mozperftest import PerftestArgumentParser
from mozboot.util import get_state_dir
from mach.logging import LoggingManager
mozconfig = SRC_ROOT / "browser" / "config" / "mozconfig"
if mozconfig.exists():
os.environ["MOZCONFIG"] = str(mozconfig)
if "--xpcshell-mozinfo" in argv:
mozinfo = argv[argv.index("--xpcshell-mozinfo") + 1]
topobjdir = Path(mozinfo).parent
else:
topobjdir = None
config = MozbuildObject(
str(SRC_ROOT),
None,
LoggingManager(),
topobjdir=topobjdir,
mozconfig=MozconfigLoader.AUTODETECT,
)
config.topdir = config.topsrcdir
config.cwd = os.getcwd()
config.state_dir = get_state_dir()
# This monkey patch forces mozbuild to reuse
# our configuration when it tries to re-create
# it from the environment.
def _here(*args, **kw):
return config
MozbuildObject.from_environment = _here
mach_cmd = MachCommandBase(config)
parser = PerftestArgumentParser(description="vanilla perftest")
args = parser.parse_args(args=argv)
run_tests(mach_cmd, **dict(args._get_kwargs()))
def test_mach_command_base_inside_objdir(self):
"""Ensure a MachCommandBase constructed from inside the objdir works."""
d = os.path.realpath(tempfile.mkdtemp())
try:
topobjdir = os.path.join(d, "objdir")
os.makedirs(topobjdir)
topsrcdir = os.path.join(d, "srcdir")
prepare_tmp_topsrcdir(topsrcdir)
mozinfo = os.path.join(topobjdir, "mozinfo.json")
with open(mozinfo, "wt") as fh:
json.dump(
dict(
topsrcdir=topsrcdir,
),
fh,
)
os.chdir(topobjdir)
class MockMachContext(object):
pass
context = MockMachContext()
context.cwd = topobjdir
context.topdir = topsrcdir
context.settings = None
context.log_manager = None
context.detect_virtualenv_mozinfo = False
o = MachCommandBase(context, None)
self.assertEqual(o.topobjdir, mozpath.normsep(topobjdir))
self.assertEqual(o.topsrcdir, mozpath.normsep(topsrcdir))
finally:
os.chdir(self._old_cwd)
shutil.rmtree(d)
def __init__(self, context):
MachCommandBase.__init__(self, context)
for attr in ('b2g_home', 'device_name'):
setattr(self, attr, getattr(context, attr, None))
def __init__(self, context):
MachCommandBase.__init__(self, context)
for attr in ('device_name', 'target_out'):
setattr(self, attr, getattr(context, attr, None))
def run_tests(self, **kwargs):
MachCommandBase._activate_virtualenv(self)
from pathlib import Path
from mozperftest.runner import _setup_path
from mozperftest.utils import install_package, temporary_env
skip_linters = kwargs.get("skip_linters", False)
# include in sys.path all deps
_setup_path()
try:
import coverage # noqa
except ImportError:
pydeps = Path(self.topsrcdir, "third_party", "python")
vendors = ["coverage"]
if skip_linters:
pypis = []
else:
pypis = ["flake8"]
# if we're not on try we want to install black
if not ON_TRY and not skip_linters:
pypis.append("black")
# these are the deps we are getting from pypi
for dep in pypis:
install_package(self.virtualenv_manager, dep)
# pip-installing dependencies that require compilation or special setup
for dep in vendors:
install_package(self.virtualenv_manager, str(Path(pydeps,
dep)))
here = Path(__file__).parent.resolve()
if not ON_TRY and not skip_linters:
# formatting the code with black
assert self._run_python_script("black", str(here))
# checking flake8 correctness
if not (ON_TRY and sys.platform == "darwin") and not skip_linters:
assert self._run_python_script("flake8", str(here))
# running pytest with coverage
# coverage is done in three steps:
# 1/ coverage erase => erase any previous coverage data
# 2/ coverage run pytest ... => run the tests and collect info
# 3/ coverage report => generate the report
tests_dir = Path(here, "tests").resolve()
tests = kwargs.get("tests", [])
if tests == []:
tests = str(tests_dir)
run_coverage_check = not skip_linters
else:
run_coverage_check = False
def _get_test(test):
if Path(test).exists():
return str(test)
return str(tests_dir / test)
tests = " ".join([_get_test(test) for test in tests])
import pytest
with temporary_env(COVERAGE_RCFILE=str(here / ".coveragerc")):
if run_coverage_check:
assert self._run_python_script(
"coverage", "erase", label="remove old coverage data")
args = [
"run",
pytest.__file__,
"-xs",
tests,
]
assert self._run_python_script("coverage",
*args,
label="running tests")
if run_coverage_check and not self._run_python_script(
"coverage", "report", display=True):
raise ValueError("Coverage is too low!")
def __init__(self, context):
MachCommandBase.__init__(self, context)
for attr in ("b2g_home", "xre_path", "device_name"):
setattr(self, attr, getattr(context, attr, None))
def __init__(self, context):
MachCommandBase.__init__(self, context)
for attr in ('b2g_home', 'xre_path', 'device_name', 'get_build_var'):
setattr(self, attr, getattr(context, attr, None))
def __init__(self, context):
MachCommandBase.__init__(self, context)
def __init__(self, context):
MachCommandBase.__init__(self, context)
self.b2g_home = context.b2g_home
self.device_name = context.device_name
def __init__(self, context):
MachCommandBase.__init__(self, context)
for attr in ('b2g_home', 'xre_path', 'device_name'):
setattr(self, attr, getattr(context, attr, None))
def __init__(self, context):
MachCommandBase.__init__(self, context)
self.remotedir = os.path.join(self.topsrcdir, "remote")
def __init__(self, context):
MachCommandBase.__init__(self, context)
self._python_executable = None
def __init__(self, context):
MachCommandBase.__init__(self, context)
def __init__(self, context):
MachCommandBase.__init__(self, context)
self._python_executable = None