def test_uninstall(self):
""" Test mozinstall's uninstall capabilites """
# Uninstall after installing
if mozinfo.isLinux:
installdir = mozinstall.install(self.bz2, self.tempdir)
mozinstall.uninstall(installdir)
self.assertFalse(os.path.exists(installdir))
elif mozinfo.isWin:
# Exe installer for Windows
installdir_exe = mozinstall.install(self.exe,
os.path.join(self.tempdir, 'exe'))
mozinstall.uninstall(installdir_exe)
self.assertFalse(os.path.exists(installdir_exe))
# Zip installer for Windows
installdir_zip = mozinstall.install(self.zipfile,
os.path.join(self.tempdir, 'zip'))
mozinstall.uninstall(installdir_zip)
self.assertFalse(os.path.exists(installdir_zip))
elif mozinfo.isMac:
installdir = mozinstall.install(self.dmg, self.tempdir)
mozinstall.uninstall(installdir)
self.assertFalse(os.path.exists(installdir))
def install_build(self, url):
try:
self.logger.info("Installing build from url: %s" % url)
buildfile = os.path.abspath("b2gtarball.tar.gz")
urllib.urlretrieve(url, buildfile)
except:
self.logger.exception("Failed to download build")
try:
self.logger.info("Untarring build")
# Extract to the same local directory where we downloaded the build
# to. This defaults to the local directory where our script runs
dest = os.path.join(os.path.dirname(buildfile), 'downloadedbuild')
if (os.access(dest, os.F_OK)):
shutil.rmtree(dest)
install(buildfile, dest)
# This should extract into a qemu directory
qemu = os.path.join(dest, 'qemu')
if os.path.exists(qemu):
return qemu
else:
return None
except:
self.logger.exception("Failed to untar file")
return None
def test_get_binary(self):
""" Test mozinstall's get_binary method """
if mozinfo.isLinux:
installdir = mozinstall.install(self.bz2, self.tempdir)
binary = os.path.join(installdir, 'firefox')
self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox'))
elif mozinfo.isWin:
installdir_exe = mozinstall.install(self.exe,
os.path.join(self.tempdir, 'exe'))
binary_exe = os.path.join(installdir_exe, 'firefox', 'firefox',
'firefox.exe')
self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe,
'firefox'))
installdir_zip = mozinstall.install(self.zipfile,
os.path.join(self.tempdir, 'zip'))
binary_zip = os.path.join(installdir_zip, 'firefox.exe')
self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip,
'firefox'))
elif mozinfo.isMac:
installdir = mozinstall.install(self.dmg, self.tempdir)
binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox')
self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox'))
def install(self, dest=None, channel="nightly"):
"""Install Firefox."""
branch = {
"nightly": "mozilla-central",
"beta": "mozilla-beta",
"stable": "mozilla-stable"
}
scraper = {
"nightly": "daily",
"beta": "release",
"stable": "release"
}
version = {
"stable": "latest",
"beta": "latest-beta",
"nightly": "latest"
}
if channel not in branch:
raise ValueError("Unrecognised release channel: %s" % channel)
from mozdownload import FactoryScraper
import mozinstall
platform = {
"Linux": "linux",
"Windows": "win",
"Darwin": "mac"
}.get(uname[0])
if platform is None:
raise ValueError("Unable to construct a valid Firefox package name for current platform")
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers", channel)
filename = FactoryScraper(scraper[channel],
branch=branch[channel],
version=version[channel],
destination=dest).download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError:
if platform == "mac" and os.path.exists(os.path.join(dest, "Firefox Nightly.app")):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, "Firefox Nightly.app"))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def download_b2g_sdk(b2g_sdk):
system = platform.system()
if system == "Linux":
url = "https://queue.taskcluster.net/v1/task/YamDhuDgTWa_kWXcSedDHA/artifacts/public/build/target.linux-x86_64.tar.bz2"
elif system == "Darwin":
url = "http://ftp.mozilla.org/pub/mozilla.org/b2g/nightly/2015/09/2015-09-02-03-02-03-mozilla-central/b2g-43.0a1.en-US.mac64.dmg"
elif system == "Windows":
url = "http://ftp.mozilla.org/pub/mozilla.org/b2g/nightly/2015/09/2015-09-02-03-02-03-mozilla-central/b2g-43.0a1.en-US.win32.zip"
else:
raise Exception('Unable to download b2g_sdk for %s' % system)
if not os.path.isdir(b2g_sdk):
os.mkdir(b2g_sdk)
b2g_path = os.path.join(b2g_sdk, "b2g")
if not os.path.isdir(b2g_path):
file_path = os.path.join(b2g_sdk, os.path.basename(urlparse.urlparse(url).path))
import requests
with open(file_path, "wb") as b2g:
print("Downloading %s" % url)
response = requests.get(url, stream=True)
total_length = response.headers.get("content-length")
if total_length is None: # no content length header
b2g.write(response.content)
else:
download_length = 0
total_length = int(total_length)
for data in response.iter_content(8192):
download_length += len(data)
b2g.write(data)
print("\r%10d / %10d [%3.2f%%]" %
(download_length,
total_length,
download_length * 100. / total_length),
end = "")
b2g.close()
print()
print("Extract %s..." % file_path)
import mozinstall
mozinstall.install(file_path, os.path.join(b2g_sdk))
if system == "Darwin":
os.symlink(os.path.join(b2g_sdk, "B2G.app", "Contents", "MacOS"), b2g_path)
return b2g_path
def startTestRunner(runner_class, options, tests):
install_folder = None
try:
# Prepare the workspace path so that all temporary data can be stored inside it.
if options.workspace_path:
path = os.path.expanduser(options.workspace_path)
options.workspace = os.path.abspath(path)
if not os.path.exists(options.workspace):
os.makedirs(options.workspace)
else:
options.workspace = tempfile.mkdtemp(".{}".format(os.path.basename(sys.argv[0])))
options.logger.info('Using workspace for temporary data: "{}"'.format(options.workspace))
# If the specified binary is an installer it needs to be installed
if options.installer:
installer = os.path.realpath(options.installer)
dest_folder = os.path.join(options.workspace, "binary")
options.logger.info('Installing application "%s" to "%s"' % (installer, dest_folder))
install_folder = mozinstall.install(installer, dest_folder)
options.binary = mozinstall.get_binary(install_folder, "firefox")
runner = runner_class(**vars(options))
runner.run_tests(tests)
finally:
# Ensure to uninstall the binary if it has been installed before
if install_folder and os.path.exists(install_folder):
options.logger.info('Uninstalling application at "%s"' % install_folder)
mozinstall.uninstall(install_folder)
return runner
def prepare_application(self, binary):
# Prepare the binary for the test run
if application.is_installer(self.binary, self.options.application):
install_path = os.path.join(self.workspace, 'binary')
print "*** Installing build: %s" % self.binary
self._folder = mozinstall.install(self.binary, install_path)
binary_name = APPLICATION_BINARY_NAMES[self.options.application]
self._application = mozinstall.get_binary(self._folder,
binary_name)
else:
if os.path.isdir(self.binary):
self._folder = self.binary
else:
if mozinfo.isMac:
# Ensure that self._folder is the app bundle on OS X
p = re.compile('.*\.app/')
self._folder = p.search(self.binary).group()
else:
self._folder = os.path.dirname(self.binary)
binary_name = APPLICATION_BINARY_NAMES[self.options.application]
self._application = mozinstall.get_binary(self._folder,
binary_name)
def install(self):
if not self.name:
raise NotImplementedError("Can't invoke abstract base class")
self.remove_tempdir()
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(mozinstall.install(src=self.dest, dest=self.tempdir), self.name)
return True
def test_install(self):
""" Test mozinstall's install capability """
if mozinfo.isLinux:
installdir = mozinstall.install(self.bz2, self.tempdir)
self.assertEqual(os.path.join(self.tempdir, "firefox"), installdir)
elif mozinfo.isWin:
installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, "exe"))
self.assertEqual(os.path.join(self.tempdir, "exe", "firefox"), installdir_exe)
installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, "zip"))
self.assertEqual(os.path.join(self.tempdir, "zip", "firefox"), installdir_zip)
elif mozinfo.isMac:
installdir = mozinstall.install(self.dmg, self.tempdir)
self.assertEqual(os.path.join(os.path.realpath(self.tempdir), "FirefoxStub.app"), installdir)
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
try:
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name
)
except:
rmtree(self.tempdir)
raise
def test_get_binary(self):
""" Test mozinstall's get_binary method """
if mozinfo.isLinux:
installdir = mozinstall.install(self.bz2, self.tempdir)
binary = os.path.join(installdir, "firefox")
self.assertEqual(binary, mozinstall.get_binary(installdir, "firefox"))
elif mozinfo.isWin:
installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, "exe"))
binary_exe = os.path.join(installdir_exe, "firefox", "firefox", "firefox.exe")
self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe, "firefox"))
installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, "zip"))
binary_zip = os.path.join(installdir_zip, "firefox.exe")
self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip, "firefox"))
elif mozinfo.isMac:
installdir = mozinstall.install(self.dmg, self.tempdir)
binary = os.path.join(installdir, "Contents", "MacOS", "firefox")
self.assertEqual(binary, mozinstall.get_binary(installdir, "firefox"))
def firefox(pytestconfig, tmpdir_factory):
binary = os.getenv('MOZREGRESSION_BINARY',
pytestconfig.getoption('firefox'))
if binary is None:
cache_dir = str(pytestconfig.cache.makedir('firefox'))
scraper = FactoryScraper('daily', destination=cache_dir)
build_path = scraper.download()
install_path = str(tmpdir_factory.mktemp('firefox'))
install_dir = mozinstall.install(src=build_path, dest=install_path)
binary = mozinstall.get_binary(install_dir, 'firefox')
version = mozversion.get_version(binary)
if hasattr(pytestconfig, '_metadata'):
pytestconfig._metadata.update(version)
return binary
def install(self, dest=None):
"""Install Firefox."""
from mozdownload import FactoryScraper
import mozinstall
platform = {
"Linux": "linux",
"Windows": "win",
"Darwin": "mac"
}.get(uname[0])
if platform is None:
raise ValueError("Unable to construct a valid Firefox package name for current platform")
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers")
filename = FactoryScraper("daily", branch="mozilla-central", destination=dest).download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError as e:
if platform == "mac" and os.path.exists(os.path.join(dest, "Firefox Nightly.app")):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, "Firefox Nightly.app"))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def binary():
"""Return a Firefox binary"""
try:
return build.get_binary_path()
except Exception:
pass
app = 'firefox'
bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app)
if os.path.isdir(bindir):
try:
return mozinstall.get_binary(bindir, app_name=app)
except Exception:
pass
if 'GECKO_INSTALLER_URL' in os.environ:
bindir = mozinstall.install(
os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP'])
return mozinstall.get_binary(bindir, app_name='firefox')
def prepare_application(self, binary):
# Prepare the binary for the test run
if mozinstall.is_installer(self.binary):
install_path = tempfile.mkdtemp(".binary")
print "Install build: %s" % self.binary
self._folder = mozinstall.install(self.binary, install_path)
self._application = mozinstall.get_binary(self._folder, self.options.application)
else:
if os.path.isdir(self.binary):
self._folder = self.binary
else:
if sys.platform == "darwin":
# Ensure that self._folder is the app bundle on OS X
p = re.compile(".*\.app/")
self._folder = p.search(self.binary).group()
else:
self._folder = os.path.dirname(self.binary)
self._application = mozinstall.get_binary(self._folder, self.options.application)
def setup_env(self):
# Download the build and tests package
self.build = self.download(url=self.build_url)
self.packaged_tests = self.download(url=self.tests_url)
# Unzip the packaged tests
with zipfile.ZipFile(self.packaged_tests) as zip:
zip.extractall('tests')
# Create the virtual environment
current_location = os.getcwd()
create_venv_script = os.path.abspath(os.path.join('tests', 'tps',
'create_venv.py'))
tps_env = os.path.abspath('tps-env')
# Bug 1030768, setup.py has to be called from it's own location
os.chdir(os.path.abspath(os.path.join('tests', 'tps')))
subprocess.check_call(['python', create_venv_script,
'--username', self.username,
'--password', self.password, tps_env])
os.chdir(current_location)
dir = 'Scripts' if sys.platform == 'win32' else 'bin'
env_activate_file = os.path.join(tps_env, dir, 'activate_this.py')
# Activate the environment and set the VIRTUAL_ENV os variable
execfile(env_activate_file, dict(__file__=env_activate_file))
os.environ['VIRTUAL_ENV'] = tps_env
# Install the fxa-python-client
subprocess.check_call(['pip', 'install', 'fxa-python-client'])
# After activating the environment we import the mozinstall module
import mozinstall
# Install Firefox and get the binary
self.build_path = mozinstall.install(self.build,
os.path.join(current_location,
'binary'))
self.binary = mozinstall.get_binary(self.build_path, 'firefox')
def download_build(self, installdir='downloadedbuild', appname='firefox'):
self.installdir = os.path.abspath(installdir)
buildName = os.path.basename(self.url)
pathToBuild = os.path.join(os.path.dirname(os.path.abspath(__file__)),
buildName)
# delete the build if it already exists
if os.access(pathToBuild, os.F_OK):
os.remove(pathToBuild)
# download the build
print "downloading build"
self.download_url(self.url, pathToBuild)
# install the build
print "installing %s" % pathToBuild
shutil.rmtree(self.installdir, True)
binary = mozinstall.install(src=pathToBuild, dest=self.installdir)
# remove the downloaded archive
os.remove(pathToBuild)
return binary
def startTestRunner(runner_class, options, tests):
install_folder = None
try:
# If the specified binary is an installer it needs to be installed
if options.installer:
installer = os.path.realpath(options.installer)
dest_folder = tempfile.mkdtemp()
options.logger.info('Installing application "%s" to "%s"' % (installer,
dest_folder))
install_folder = mozinstall.install(installer, dest_folder)
options.binary = mozinstall.get_binary(install_folder, 'firefox')
runner = runner_class(**vars(options))
runner.run_tests(tests)
finally:
# Ensure to uninstall the binary if it has been installed before
if install_folder and os.path.exists(install_folder):
options.logger.info('Uninstalling application at "%s"' % install_folder)
mozinstall.uninstall(install_folder)
return runner
import mozinstall
import os
import sys
import tempfile
tempdir = tempfile.mkdtemp()
firefox_exe = sys.argv[1]
install_folder = mozinstall.install(src=firefox_exe, dest=tempdir)
binary = mozinstall.get_binary(install_folder, 'Firefox')
if not os.path.exists("build"):
os.makedirs("build")
f = open("build/fflocation.bat", "w")
f.write("set FIREFOX_BINARY=%s" % binary)
f.close()
f = open("build/fflocation.txt", "w")
f.write(binary)
f.close()
f = open("build/templocation.txt", "w")
f.write(tempdir)
f.close()
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely:
# InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
# File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
# return artifacts.install_from(source, self.distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
# return self.install_from_hg(source, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
# return self.install_from_url(url, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
# return self.install_from_file(filename, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
# mozinstall.install(filename, tempdir)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
# install_dir = _install_dmg(src, dest)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
# subprocess.call('hdiutil detach %s -quiet' % appDir,
# TODO: Extract the bundle name from the archive (it may differ
# from MOZ_MACBUNDLE_NAME).
bundle_name = 'Nightly.app'
source = mozpath.join(tempdir, bundle_name)
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = ('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libsoftokn3.dylib',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = ('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'webapprt-stub',
])
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': path},
'Adding {path} to processed archive')
writer.add(os.path.basename(p).encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
root, paths = paths_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': path},
'Adding {path} to processed archive')
writer.add(p.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
oldcwd = os.getcwd()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
if self._substs['HOST_OS_ARCH'] == 'Linux':
# This is a cross build, use hfsplus and dmg tools to extract the dmg.
os.chdir(tempdir)
with open(os.devnull, 'wb') as devnull:
subprocess.check_call([
self._substs['DMG_TOOL'],
'extract',
filename,
'extracted_img',
], stdout=devnull)
subprocess.check_call([
self._substs['HFS_TOOL'],
'extracted_img',
'extractall'
], stdout=devnull)
else:
mozinstall.install(filename, tempdir)
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = ('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libmozavutil.dylib',
'libmozavcodec.dylib',
'libsoftokn3.dylib',
'pingsender',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/org.mozilla.updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = [
('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/minidump-analyzer',
]),
('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'**/interfaces.xpt',
]),
]
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
for root, paths in paths_keep_path:
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
finally:
os.chdir(oldcwd)
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
oldcwd = os.getcwd()
try:
self.log(logging.INFO, 'artifact',
{'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
if self._substs['HOST_OS_ARCH'] == 'Linux':
# This is a cross build, use hfsplus and dmg tools to extract the dmg.
os.chdir(tempdir)
with open(os.devnull, 'wb') as devnull:
subprocess.check_call([
self._substs['DMG_TOOL'],
'extract',
filename,
'extracted_img',
], stdout=devnull)
subprocess.check_call([
self._substs['HFS_TOOL'],
'extracted_img',
'extractall'
], stdout=devnull)
else:
mozinstall.install(filename, tempdir)
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = [
('Contents/MacOS', [
'crashreporter.app/Contents/MacOS/minidump-analyzer',
]),
('Contents/Resources', [
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
]),
]
with JarWriter(file=processed_filename, compress_level=5) as writer:
root, paths = self.paths_no_keep_path
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'), f, mode=f.mode)
for root, paths in paths_keep_path:
finder = UnpackFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact',
{'path': p},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode)
finally:
os.chdir(oldcwd)
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact',
{'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir),
self.app_name)
kwargs.update(scraper_options.get(options.type, {}))
if options.type == 'nightly':
# DailyScraper generally chooses the right extension based on the platform,
# but it looks for a .exe installer on Windows by default, and B2G nightlies
# for Windows come only in the .zip variant, so specify that extension.
if platform == "win32":
kwargs.update({ 'extension': 'zip' })
build = DailyScraper(**kwargs)
elif options.type == 'specific':
build = DirectScraper(options.url, **kwargs)
else:
raise NotImplementedError('type %s not supported' % options.type)
print "Initiating B2G download."
build.download()
# Install B2G Desktop to addon's data directory.
installer = os.path.abspath(build.target)
# Remove the existing installation, then install.
platformdir = os.path.join(datadir, platform)
shutil.rmtree(os.path.join(platformdir, installdirname), True)
mozinstall.install(installer, platformdir)
# Clean up.
#shutil.rmtree(tmpdir)
def _install(self, dest):
self.tempdir = tempfile.mkdtemp()
self.binary = mozinstall.get_binary(
mozinstall.install(src=dest, dest=self.tempdir), self.app_name)
def install(self, dest=None, channel="nightly"):
"""Install Firefox."""
branch = {
"nightly": "mozilla-central",
"beta": "mozilla-beta",
"stable": "mozilla-stable"
}
scraper = {
"nightly": "daily",
"beta": "release",
"stable": "release"
}
version = {
"stable": "latest",
"beta": "latest-beta",
"nightly": "latest"
}
application_name = {
"stable": "Firefox.app",
"beta": "Firefox.app",
"nightly": "Firefox Nightly.app"
}
if channel not in branch:
raise ValueError("Unrecognised release channel: %s" % channel)
from mozdownload import FactoryScraper
import mozinstall
platform = {
"Linux": "linux",
"Windows": "win",
"Darwin": "mac"
}.get(uname[0])
if platform is None:
raise ValueError("Unable to construct a valid Firefox package name for current platform")
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers", channel)
filename = FactoryScraper(scraper[channel],
branch=branch[channel],
version=version[channel],
destination=dest).download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError:
if platform == "mac" and os.path.exists(os.path.join(dest, application_name[channel])):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, application_name[channel]))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def get_test_candidate(self):
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param:
build: str with firefox build
:return:
Installation path for the Firefox App
"""
location = ''
candidate_app = ''
if self.args.firefox == 'local':
if Settings.is_mac():
location = '/Applications/Firefox.app/Contents/'
candidate_app = os.path.join(location, 'MacOS', 'firefox')
elif Settings.is_windows():
location = 'C:\\Program Files (x86)\\Mozilla Firefox'
if not os.path.exists(location):
location = 'C:\\Program Files\\Mozilla Firefox'
candidate_app = os.path.join(location, 'firefox.exe')
elif Settings.is_linux():
location = '/usr/lib/firefox'
candidate_app = os.path.join(location, 'firefox')
else:
logger.critical('Platform not supported')
self.finish(code=5)
if not os.path.isdir(location):
logger.critical(
'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/'
)
self.finish(code=5)
return candidate_app
else:
try:
locale = 'ja-JP-mac' if self.args.locale == 'ja' and Settings.is_mac(
) else self.args.locale
type, scraper_details = get_scraper_details(
self.args.firefox, Settings.CHANNELS,
os.path.join(IrisCore.get_working_dir(), 'cache'), locale)
scraper = FactoryScraper(type, **scraper_details)
firefox_dmg = scraper.download()
install_folder = install(src=firefox_dmg,
dest=IrisCore.get_current_run_dir())
binary = get_binary(install_folder, 'Firefox')
channel = get_firefox_channel(binary)
latest_type, latest_scraper_details = get_latest_scraper_details(
channel)
latest_path = FactoryScraper(latest_type,
**latest_scraper_details).filename
self.latest_version = get_version_from_path(latest_path)
logger.info('Latest available version for %s channel is: %s' %
(channel, self.latest_version))
return binary
except errors.NotFoundError:
logger.critical(
'Specified build (%s) has not been found. Closing Iris ...'
% self.args.firefox)
self.finish(5)
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(logging.INFO, 'artifact', {'tempdir': tempdir},
'Unpacking DMG into {tempdir}')
mozinstall.install(
filename,
tempdir) # Doesn't handle already mounted DMG files nicely:
# InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
# File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
# return artifacts.install_from(source, self.distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
# return self.install_from_hg(source, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
# return self.install_from_url(url, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
# return self.install_from_file(filename, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
# mozinstall.install(filename, tempdir)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
# install_dir = _install_dmg(src, dest)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
# subprocess.call('hdiutil detach %s -quiet' % appDir,
bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app'))
if len(bundle_dirs) != 1:
raise ValueError(
'Expected one source bundle, found: {}'.format(
bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = (
'Contents/MacOS',
[
'crashreporter.app/Contents/MacOS/crashreporter',
'firefox',
'firefox-bin',
'libfreebl3.dylib',
'liblgpllibs.dylib',
# 'liblogalloc.dylib',
'libmozglue.dylib',
'libnss3.dylib',
'libnssckbi.dylib',
'libnssdbm3.dylib',
'libplugin_child_interpose.dylib',
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
'libsoftokn3.dylib',
'plugin-container.app/Contents/MacOS/plugin-container',
'updater.app/Contents/MacOS/updater',
# 'xpcshell',
'XUL',
])
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = (
'Contents/Resources',
[
'browser/components/libbrowsercomps.dylib',
'dependentlibs.list',
# 'firefox',
'gmp-clearkey/0.1/libclearkey.dylib',
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
'webapprt-stub',
])
with JarWriter(file=processed_filename,
optimize=False,
compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact', {'path': path},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', os.path.basename(p))
writer.add(destpath.encode('utf-8'),
f,
mode=os.stat(mozpath.join(finder.base,
p)).st_mode)
root, paths = paths_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, 'artifact', {'path': path},
'Adding {path} to processed archive')
destpath = mozpath.join('bin', p)
writer.add(destpath.encode('utf-8'),
f,
mode=os.stat(mozpath.join(finder.base,
p)).st_mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, 'artifact', {'tempdir': tempdir},
'Unable to delete {tempdir}')
pass
def install(self, dest=None, channel="nightly"):
"""Install Firefox."""
import mozinstall
product = {
"nightly": "firefox-nightly-latest-ssl",
"beta": "firefox-beta-latest-ssl",
"stable": "firefox-latest-ssl"
}
os_builds = {
("linux", "x86"): "linux",
("linux", "x86_64"): "linux64",
("win", "x86"): "win",
("win", "x86_64"): "win64",
("macos", "x86_64"): "osx",
}
os_key = (self.platform, uname[4])
if channel not in product:
raise ValueError("Unrecognised release channel: %s" % channel)
if os_key not in os_builds:
raise ValueError("Unsupported platform: %s %s" % os_key)
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers", channel)
if not os.path.exists(dest):
os.makedirs(dest)
url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel],
os_builds[os_key])
self.logger.info("Downloading Firefox from %s" % url)
resp = requests.get(url)
filename = None
content_disposition = resp.headers.get('content-disposition')
if content_disposition:
filenames = re.findall("filename=(.+)", content_disposition)
if filenames:
filename = filenames[0]
if not filename:
filename = urlparse.urlsplit(resp.url).path.rsplit("/", 1)[1]
if not filename:
filename = "firefox.tar.bz2"
installer_path = os.path.join(dest, filename)
with open(installer_path, "w") as f:
f.write(resp.content)
try:
mozinstall.install(installer_path, dest)
except mozinstall.mozinstall.InstallError:
if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app")))
mozinstall.install(filename, dest)
else:
raise
os.remove(installer_path)
return self.find_binary_path(dest)
def process_package_artifact(self, filename, processed_filename):
tempdir = tempfile.mkdtemp()
try:
self.log(logging.INFO, "artifact", {"tempdir": tempdir}, "Unpacking DMG into {tempdir}")
mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely:
# InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)"
# File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install
# return artifacts.install_from(source, self.distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from
# return self.install_from_hg(source, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg
# return self.install_from_url(url, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url
# return self.install_from_file(filename, distdir)
# File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file
# mozinstall.install(filename, tempdir)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install
# install_dir = _install_dmg(src, dest)
# File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg
# subprocess.call('hdiutil detach %s -quiet' % appDir,
bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app"))
if len(bundle_dirs) != 1:
raise ValueError("Expected one source bundle, found: {}".format(bundle_dirs))
[source] = bundle_dirs
# These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c".
paths_no_keep_path = (
"Contents/MacOS",
[
"crashreporter.app/Contents/MacOS/crashreporter",
"firefox",
"firefox-bin",
"libfreebl3.dylib",
"liblgpllibs.dylib",
# 'liblogalloc.dylib',
"libmozglue.dylib",
"libnss3.dylib",
"libnssckbi.dylib",
"libnssdbm3.dylib",
"libplugin_child_interpose.dylib",
# 'libreplace_jemalloc.dylib',
# 'libreplace_malloc.dylib',
"libmozavutil.dylib",
"libmozavcodec.dylib",
"libsoftokn3.dylib",
"plugin-container.app/Contents/MacOS/plugin-container",
"updater.app/Contents/MacOS/org.mozilla.updater",
# 'xpcshell',
"XUL",
],
)
# These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c".
paths_keep_path = (
"Contents/Resources",
[
"browser/components/libbrowsercomps.dylib",
"dependentlibs.list",
# 'firefox',
"gmp-clearkey/0.1/libclearkey.dylib",
# 'gmp-fake/1.0/libfake.dylib',
# 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib',
],
)
with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer:
root, paths = paths_no_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, "artifact", {"path": path}, "Adding {path} to processed archive")
destpath = mozpath.join("bin", os.path.basename(p))
writer.add(destpath.encode("utf-8"), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
root, paths = paths_keep_path
finder = FileFinder(mozpath.join(source, root))
for path in paths:
for p, f in finder.find(path):
self.log(logging.INFO, "artifact", {"path": path}, "Adding {path} to processed archive")
destpath = mozpath.join("bin", p)
writer.add(destpath.encode("utf-8"), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode)
finally:
try:
shutil.rmtree(tempdir)
except (OSError, IOError):
self.log(logging.WARN, "artifact", {"tempdir": tempdir}, "Unable to delete {tempdir}")
pass
#!/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from mozdownload import FactoryScraper
import mozinstall
scraper = FactoryScraper('daily',
branch=os.environ['MOZ_BRANCH'],
locale=os.environ['LOCALE'])
installer_path = scraper.download()
install_path = mozinstall.install(installer_path, 'application')
binary_path = mozinstall.get_binary(install_path, 'firefox')
with open('binary.txt', 'w') as f:
f.write(binary_path)
# Download latest build of B2G Desktop.
scraper_keywords = { 'application': 'b2g',
'branch': 'mozilla-aurora',
'platform': platform,
'locale': 'en-US',
'version': None,
'directory': downloaddir }
kwargs = scraper_keywords.copy()
if platform == "win32":
kwargs.update({ 'extension': 'zip' })
build = DailyScraper(**kwargs)
print "Initiating download B2G Desktop latest build..."
build.download()
# Install B2G Desktop to addon's data directory.
installer = os.path.abspath(build.target)
# Remove the existing installation, then install.
platformdir = os.path.join(datadir, platform)
shutil.rmtree(os.path.join(platformdir, installdirname), True)
mozinstall.install(installer, platformdir)
# Clean up.
#shutil.rmtree(tmpdir)
def prepare_b2g_desktop(self):
file_name = self.download_file(
lambda x: x.startswith(self.device) and x.endswith('%s.%s' % (
self.platform, self.suffix)))
mozinstall.install(file_name, self.metadata['workdir'])
os.remove(file_name)
def prepare_b2g_desktop(self):
file_name = self.download_file(lambda x: x.startswith(self.device)
and x.endswith('%s.%s' % (self.platform, self.suffix)))
mozinstall.install(file_name, self.metadata['workdir'])
os.remove(file_name)
self.repository_path)
self._repository.clone()
except Exception, e:
raise Exception("Failure in setting up the mozmill-tests repository. " +
e.message)
if self.options.addons:
self.prepare_addons()
try:
# Prepare the binary for the test run
if mozinstall.is_installer(self.binary):
install_path = tempfile.mkdtemp(".binary")
print "Install build: %s" % self.binary
self._folder = mozinstall.install(self.binary, install_path)
self._application = mozinstall.get_binary(self._folder,
self.options.application)
else:
# TODO: Ensure that self._folder is the same as from mozinstall
folder = os.path.dirname(self.binary)
self._folder = folder if not os.path.isdir(self.binary) else self.binary
self._application = self.binary
# Prepare the repository
ini = application.ApplicationIni(self._application)
repository_url = ini.get('App', 'SourceRepository')
# Update the mozmill-test repository to match the Gecko branch
branch_name = self._repository.identify_branch(repository_url)
self._repository.update(branch_name)
