def test_uninstall(self): """ Test mozinstall's uninstall capabilites """ # Uninstall after installing if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) mozinstall.uninstall(installdir) self.assertFalse(os.path.exists(installdir)) elif mozinfo.isWin: # Exe installer for Windows installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, 'exe')) mozinstall.uninstall(installdir_exe) self.assertFalse(os.path.exists(installdir_exe)) # Zip installer for Windows installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, 'zip')) mozinstall.uninstall(installdir_zip) self.assertFalse(os.path.exists(installdir_zip)) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) mozinstall.uninstall(installdir) self.assertFalse(os.path.exists(installdir))
def install_build(self, url): try: self.logger.info("Installing build from url: %s" % url) buildfile = os.path.abspath("b2gtarball.tar.gz") urllib.urlretrieve(url, buildfile) except: self.logger.exception("Failed to download build") try: self.logger.info("Untarring build") # Extract to the same local directory where we downloaded the build # to. This defaults to the local directory where our script runs dest = os.path.join(os.path.dirname(buildfile), 'downloadedbuild') if (os.access(dest, os.F_OK)): shutil.rmtree(dest) install(buildfile, dest) # This should extract into a qemu directory qemu = os.path.join(dest, 'qemu') if os.path.exists(qemu): return qemu else: return None except: self.logger.exception("Failed to untar file") return None
def test_get_binary(self): """ Test mozinstall's get_binary method """ if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) binary = os.path.join(installdir, 'firefox') self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox')) elif mozinfo.isWin: installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, 'exe')) binary_exe = os.path.join(installdir_exe, 'firefox', 'firefox', 'firefox.exe') self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe, 'firefox')) installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, 'zip')) binary_zip = os.path.join(installdir_zip, 'firefox.exe') self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip, 'firefox')) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) binary = os.path.join(installdir, 'Contents', 'MacOS', 'firefox') self.assertEqual(binary, mozinstall.get_binary(installdir, 'firefox'))
def install(self, dest=None, channel="nightly"): """Install Firefox.""" branch = { "nightly": "mozilla-central", "beta": "mozilla-beta", "stable": "mozilla-stable" } scraper = { "nightly": "daily", "beta": "release", "stable": "release" } version = { "stable": "latest", "beta": "latest-beta", "nightly": "latest" } if channel not in branch: raise ValueError("Unrecognised release channel: %s" % channel) from mozdownload import FactoryScraper import mozinstall platform = { "Linux": "linux", "Windows": "win", "Darwin": "mac" }.get(uname[0]) if platform is None: raise ValueError("Unable to construct a valid Firefox package name for current platform") if dest is None: # os.getcwd() doesn't include the venv path dest = os.path.join(os.getcwd(), "_venv") dest = os.path.join(dest, "browsers", channel) filename = FactoryScraper(scraper[channel], branch=branch[channel], version=version[channel], destination=dest).download() try: mozinstall.install(filename, dest) except mozinstall.mozinstall.InstallError: if platform == "mac" and os.path.exists(os.path.join(dest, "Firefox Nightly.app")): # mozinstall will fail if nightly is already installed in the venv because # mac installation uses shutil.copy_tree mozinstall.uninstall(os.path.join(dest, "Firefox Nightly.app")) mozinstall.install(filename, dest) else: raise os.remove(filename) return self.find_binary_path(dest)
def download_b2g_sdk(b2g_sdk): system = platform.system() if system == "Linux": url = "https://queue.taskcluster.net/v1/task/YamDhuDgTWa_kWXcSedDHA/artifacts/public/build/target.linux-x86_64.tar.bz2" elif system == "Darwin": url = "http://ftp.mozilla.org/pub/mozilla.org/b2g/nightly/2015/09/2015-09-02-03-02-03-mozilla-central/b2g-43.0a1.en-US.mac64.dmg" elif system == "Windows": url = "http://ftp.mozilla.org/pub/mozilla.org/b2g/nightly/2015/09/2015-09-02-03-02-03-mozilla-central/b2g-43.0a1.en-US.win32.zip" else: raise Exception('Unable to download b2g_sdk for %s' % system) if not os.path.isdir(b2g_sdk): os.mkdir(b2g_sdk) b2g_path = os.path.join(b2g_sdk, "b2g") if not os.path.isdir(b2g_path): file_path = os.path.join(b2g_sdk, os.path.basename(urlparse.urlparse(url).path)) import requests with open(file_path, "wb") as b2g: print("Downloading %s" % url) response = requests.get(url, stream=True) total_length = response.headers.get("content-length") if total_length is None: # no content length header b2g.write(response.content) else: download_length = 0 total_length = int(total_length) for data in response.iter_content(8192): download_length += len(data) b2g.write(data) print("\r%10d / %10d [%3.2f%%]" % (download_length, total_length, download_length * 100. / total_length), end = "") b2g.close() print() print("Extract %s..." % file_path) import mozinstall mozinstall.install(file_path, os.path.join(b2g_sdk)) if system == "Darwin": os.symlink(os.path.join(b2g_sdk, "B2G.app", "Contents", "MacOS"), b2g_path) return b2g_path
def startTestRunner(runner_class, options, tests): install_folder = None try: # Prepare the workspace path so that all temporary data can be stored inside it. if options.workspace_path: path = os.path.expanduser(options.workspace_path) options.workspace = os.path.abspath(path) if not os.path.exists(options.workspace): os.makedirs(options.workspace) else: options.workspace = tempfile.mkdtemp(".{}".format(os.path.basename(sys.argv[0]))) options.logger.info('Using workspace for temporary data: "{}"'.format(options.workspace)) # If the specified binary is an installer it needs to be installed if options.installer: installer = os.path.realpath(options.installer) dest_folder = os.path.join(options.workspace, "binary") options.logger.info('Installing application "%s" to "%s"' % (installer, dest_folder)) install_folder = mozinstall.install(installer, dest_folder) options.binary = mozinstall.get_binary(install_folder, "firefox") runner = runner_class(**vars(options)) runner.run_tests(tests) finally: # Ensure to uninstall the binary if it has been installed before if install_folder and os.path.exists(install_folder): options.logger.info('Uninstalling application at "%s"' % install_folder) mozinstall.uninstall(install_folder) return runner
def prepare_application(self, binary): # Prepare the binary for the test run if application.is_installer(self.binary, self.options.application): install_path = os.path.join(self.workspace, 'binary') print "*** Installing build: %s" % self.binary self._folder = mozinstall.install(self.binary, install_path) binary_name = APPLICATION_BINARY_NAMES[self.options.application] self._application = mozinstall.get_binary(self._folder, binary_name) else: if os.path.isdir(self.binary): self._folder = self.binary else: if mozinfo.isMac: # Ensure that self._folder is the app bundle on OS X p = re.compile('.*\.app/') self._folder = p.search(self.binary).group() else: self._folder = os.path.dirname(self.binary) binary_name = APPLICATION_BINARY_NAMES[self.options.application] self._application = mozinstall.get_binary(self._folder, binary_name)
def install(self): if not self.name: raise NotImplementedError("Can't invoke abstract base class") self.remove_tempdir() self.tempdir = tempfile.mkdtemp() self.binary = mozinstall.get_binary(mozinstall.install(src=self.dest, dest=self.tempdir), self.name) return True
def test_install(self): """ Test mozinstall's install capability """ if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) self.assertEqual(os.path.join(self.tempdir, "firefox"), installdir) elif mozinfo.isWin: installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, "exe")) self.assertEqual(os.path.join(self.tempdir, "exe", "firefox"), installdir_exe) installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, "zip")) self.assertEqual(os.path.join(self.tempdir, "zip", "firefox"), installdir_zip) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) self.assertEqual(os.path.join(os.path.realpath(self.tempdir), "FirefoxStub.app"), installdir)
def _install(self, dest): self.tempdir = tempfile.mkdtemp() try: self.binary = mozinstall.get_binary( mozinstall.install(src=dest, dest=self.tempdir), self.app_name ) except: rmtree(self.tempdir) raise
def test_get_binary(self): """ Test mozinstall's get_binary method """ if mozinfo.isLinux: installdir = mozinstall.install(self.bz2, self.tempdir) binary = os.path.join(installdir, "firefox") self.assertEqual(binary, mozinstall.get_binary(installdir, "firefox")) elif mozinfo.isWin: installdir_exe = mozinstall.install(self.exe, os.path.join(self.tempdir, "exe")) binary_exe = os.path.join(installdir_exe, "firefox", "firefox", "firefox.exe") self.assertEqual(binary_exe, mozinstall.get_binary(installdir_exe, "firefox")) installdir_zip = mozinstall.install(self.zipfile, os.path.join(self.tempdir, "zip")) binary_zip = os.path.join(installdir_zip, "firefox.exe") self.assertEqual(binary_zip, mozinstall.get_binary(installdir_zip, "firefox")) elif mozinfo.isMac: installdir = mozinstall.install(self.dmg, self.tempdir) binary = os.path.join(installdir, "Contents", "MacOS", "firefox") self.assertEqual(binary, mozinstall.get_binary(installdir, "firefox"))
def firefox(pytestconfig, tmpdir_factory): binary = os.getenv('MOZREGRESSION_BINARY', pytestconfig.getoption('firefox')) if binary is None: cache_dir = str(pytestconfig.cache.makedir('firefox')) scraper = FactoryScraper('daily', destination=cache_dir) build_path = scraper.download() install_path = str(tmpdir_factory.mktemp('firefox')) install_dir = mozinstall.install(src=build_path, dest=install_path) binary = mozinstall.get_binary(install_dir, 'firefox') version = mozversion.get_version(binary) if hasattr(pytestconfig, '_metadata'): pytestconfig._metadata.update(version) return binary
def install(self, dest=None): """Install Firefox.""" from mozdownload import FactoryScraper import mozinstall platform = { "Linux": "linux", "Windows": "win", "Darwin": "mac" }.get(uname[0]) if platform is None: raise ValueError("Unable to construct a valid Firefox package name for current platform") if dest is None: # os.getcwd() doesn't include the venv path dest = os.path.join(os.getcwd(), "_venv") dest = os.path.join(dest, "browsers") filename = FactoryScraper("daily", branch="mozilla-central", destination=dest).download() try: mozinstall.install(filename, dest) except mozinstall.mozinstall.InstallError as e: if platform == "mac" and os.path.exists(os.path.join(dest, "Firefox Nightly.app")): # mozinstall will fail if nightly is already installed in the venv because # mac installation uses shutil.copy_tree mozinstall.uninstall(os.path.join(dest, "Firefox Nightly.app")) mozinstall.install(filename, dest) else: raise os.remove(filename) return self.find_binary_path(dest)
def binary(): """Return a Firefox binary""" try: return build.get_binary_path() except Exception: pass app = 'firefox' bindir = os.path.join(os.environ['PYTHON_TEST_TMP'], app) if os.path.isdir(bindir): try: return mozinstall.get_binary(bindir, app_name=app) except Exception: pass if 'GECKO_INSTALLER_URL' in os.environ: bindir = mozinstall.install( os.environ['GECKO_INSTALLER_URL'], os.environ['PYTHON_TEST_TMP']) return mozinstall.get_binary(bindir, app_name='firefox')
def prepare_application(self, binary): # Prepare the binary for the test run if mozinstall.is_installer(self.binary): install_path = tempfile.mkdtemp(".binary") print "Install build: %s" % self.binary self._folder = mozinstall.install(self.binary, install_path) self._application = mozinstall.get_binary(self._folder, self.options.application) else: if os.path.isdir(self.binary): self._folder = self.binary else: if sys.platform == "darwin": # Ensure that self._folder is the app bundle on OS X p = re.compile(".*\.app/") self._folder = p.search(self.binary).group() else: self._folder = os.path.dirname(self.binary) self._application = mozinstall.get_binary(self._folder, self.options.application)
def setup_env(self): # Download the build and tests package self.build = self.download(url=self.build_url) self.packaged_tests = self.download(url=self.tests_url) # Unzip the packaged tests with zipfile.ZipFile(self.packaged_tests) as zip: zip.extractall('tests') # Create the virtual environment current_location = os.getcwd() create_venv_script = os.path.abspath(os.path.join('tests', 'tps', 'create_venv.py')) tps_env = os.path.abspath('tps-env') # Bug 1030768, setup.py has to be called from it's own location os.chdir(os.path.abspath(os.path.join('tests', 'tps'))) subprocess.check_call(['python', create_venv_script, '--username', self.username, '--password', self.password, tps_env]) os.chdir(current_location) dir = 'Scripts' if sys.platform == 'win32' else 'bin' env_activate_file = os.path.join(tps_env, dir, 'activate_this.py') # Activate the environment and set the VIRTUAL_ENV os variable execfile(env_activate_file, dict(__file__=env_activate_file)) os.environ['VIRTUAL_ENV'] = tps_env # Install the fxa-python-client subprocess.check_call(['pip', 'install', 'fxa-python-client']) # After activating the environment we import the mozinstall module import mozinstall # Install Firefox and get the binary self.build_path = mozinstall.install(self.build, os.path.join(current_location, 'binary')) self.binary = mozinstall.get_binary(self.build_path, 'firefox')
def download_build(self, installdir='downloadedbuild', appname='firefox'): self.installdir = os.path.abspath(installdir) buildName = os.path.basename(self.url) pathToBuild = os.path.join(os.path.dirname(os.path.abspath(__file__)), buildName) # delete the build if it already exists if os.access(pathToBuild, os.F_OK): os.remove(pathToBuild) # download the build print "downloading build" self.download_url(self.url, pathToBuild) # install the build print "installing %s" % pathToBuild shutil.rmtree(self.installdir, True) binary = mozinstall.install(src=pathToBuild, dest=self.installdir) # remove the downloaded archive os.remove(pathToBuild) return binary
def startTestRunner(runner_class, options, tests): install_folder = None try: # If the specified binary is an installer it needs to be installed if options.installer: installer = os.path.realpath(options.installer) dest_folder = tempfile.mkdtemp() options.logger.info('Installing application "%s" to "%s"' % (installer, dest_folder)) install_folder = mozinstall.install(installer, dest_folder) options.binary = mozinstall.get_binary(install_folder, 'firefox') runner = runner_class(**vars(options)) runner.run_tests(tests) finally: # Ensure to uninstall the binary if it has been installed before if install_folder and os.path.exists(install_folder): options.logger.info('Uninstalling application at "%s"' % install_folder) mozinstall.uninstall(install_folder) return runner
import mozinstall import os import sys import tempfile tempdir = tempfile.mkdtemp() firefox_exe = sys.argv[1] install_folder = mozinstall.install(src=firefox_exe, dest=tempdir) binary = mozinstall.get_binary(install_folder, 'Firefox') if not os.path.exists("build"): os.makedirs("build") f = open("build/fflocation.bat", "w") f.write("set FIREFOX_BINARY=%s" % binary) f.close() f = open("build/fflocation.txt", "w") f.write(binary) f.close() f = open("build/templocation.txt", "w") f.write(tempdir) f.close()
def process_package_artifact(self, filename, processed_filename): tempdir = tempfile.mkdtemp() try: self.log(logging.INFO, 'artifact', {'tempdir': tempdir}, 'Unpacking DMG into {tempdir}') mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely: # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)" # File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install # return artifacts.install_from(source, self.distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from # return self.install_from_hg(source, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg # return self.install_from_url(url, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url # return self.install_from_file(filename, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file # mozinstall.install(filename, tempdir) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install # install_dir = _install_dmg(src, dest) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg # subprocess.call('hdiutil detach %s -quiet' % appDir, # TODO: Extract the bundle name from the archive (it may differ # from MOZ_MACBUNDLE_NAME). bundle_name = 'Nightly.app' source = mozpath.join(tempdir, bundle_name) # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". paths_no_keep_path = ('Contents/MacOS', [ 'crashreporter.app/Contents/MacOS/crashreporter', 'firefox', 'firefox-bin', 'libfreebl3.dylib', 'liblgpllibs.dylib', # 'liblogalloc.dylib', 'libmozglue.dylib', 'libnss3.dylib', 'libnssckbi.dylib', 'libnssdbm3.dylib', 'libplugin_child_interpose.dylib', # 'libreplace_jemalloc.dylib', # 'libreplace_malloc.dylib', 'libsoftokn3.dylib', 'plugin-container.app/Contents/MacOS/plugin-container', 'updater.app/Contents/MacOS/updater', # 'xpcshell', 'XUL', ]) # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". paths_keep_path = ('Contents/Resources', [ 'browser/components/libbrowsercomps.dylib', 'dependentlibs.list', # 'firefox', 'gmp-clearkey/0.1/libclearkey.dylib', # 'gmp-fake/1.0/libfake.dylib', # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', 'webapprt-stub', ]) with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: root, paths = paths_no_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': path}, 'Adding {path} to processed archive') writer.add(os.path.basename(p).encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) root, paths = paths_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': path}, 'Adding {path} to processed archive') writer.add(p.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) finally: try: shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', {'tempdir': tempdir}, 'Unable to delete {tempdir}') pass
def process_package_artifact(self, filename, processed_filename): tempdir = tempfile.mkdtemp() oldcwd = os.getcwd() try: self.log(logging.INFO, 'artifact', {'tempdir': tempdir}, 'Unpacking DMG into {tempdir}') if self._substs['HOST_OS_ARCH'] == 'Linux': # This is a cross build, use hfsplus and dmg tools to extract the dmg. os.chdir(tempdir) with open(os.devnull, 'wb') as devnull: subprocess.check_call([ self._substs['DMG_TOOL'], 'extract', filename, 'extracted_img', ], stdout=devnull) subprocess.check_call([ self._substs['HFS_TOOL'], 'extracted_img', 'extractall' ], stdout=devnull) else: mozinstall.install(filename, tempdir) bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app')) if len(bundle_dirs) != 1: raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs)) [source] = bundle_dirs # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". paths_no_keep_path = ('Contents/MacOS', [ 'crashreporter.app/Contents/MacOS/crashreporter', 'firefox', 'firefox-bin', 'libfreebl3.dylib', 'liblgpllibs.dylib', # 'liblogalloc.dylib', 'libmozglue.dylib', 'libnss3.dylib', 'libnssckbi.dylib', 'libnssdbm3.dylib', 'libplugin_child_interpose.dylib', # 'libreplace_jemalloc.dylib', # 'libreplace_malloc.dylib', 'libmozavutil.dylib', 'libmozavcodec.dylib', 'libsoftokn3.dylib', 'pingsender', 'plugin-container.app/Contents/MacOS/plugin-container', 'updater.app/Contents/MacOS/org.mozilla.updater', # 'xpcshell', 'XUL', ]) # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". paths_keep_path = [ ('Contents/MacOS', [ 'crashreporter.app/Contents/MacOS/minidump-analyzer', ]), ('Contents/Resources', [ 'browser/components/libbrowsercomps.dylib', 'dependentlibs.list', # 'firefox', 'gmp-clearkey/0.1/libclearkey.dylib', # 'gmp-fake/1.0/libfake.dylib', # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', '**/interfaces.xpt', ]), ] with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: root, paths = paths_no_keep_path finder = UnpackFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': p}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', os.path.basename(p)) writer.add(destpath.encode('utf-8'), f, mode=f.mode) for root, paths in paths_keep_path: finder = UnpackFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': p}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', p) writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode) finally: os.chdir(oldcwd) try: shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', {'tempdir': tempdir}, 'Unable to delete {tempdir}') pass
def process_package_artifact(self, filename, processed_filename): tempdir = tempfile.mkdtemp() oldcwd = os.getcwd() try: self.log(logging.INFO, 'artifact', {'tempdir': tempdir}, 'Unpacking DMG into {tempdir}') if self._substs['HOST_OS_ARCH'] == 'Linux': # This is a cross build, use hfsplus and dmg tools to extract the dmg. os.chdir(tempdir) with open(os.devnull, 'wb') as devnull: subprocess.check_call([ self._substs['DMG_TOOL'], 'extract', filename, 'extracted_img', ], stdout=devnull) subprocess.check_call([ self._substs['HFS_TOOL'], 'extracted_img', 'extractall' ], stdout=devnull) else: mozinstall.install(filename, tempdir) bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app')) if len(bundle_dirs) != 1: raise ValueError('Expected one source bundle, found: {}'.format(bundle_dirs)) [source] = bundle_dirs # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". paths_keep_path = [ ('Contents/MacOS', [ 'crashreporter.app/Contents/MacOS/minidump-analyzer', ]), ('Contents/Resources', [ 'browser/components/libbrowsercomps.dylib', 'dependentlibs.list', # 'firefox', 'gmp-clearkey/0.1/libclearkey.dylib', # 'gmp-fake/1.0/libfake.dylib', # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', ]), ] with JarWriter(file=processed_filename, compress_level=5) as writer: root, paths = self.paths_no_keep_path finder = UnpackFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': p}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', os.path.basename(p)) writer.add(destpath.encode('utf-8'), f, mode=f.mode) for root, paths in paths_keep_path: finder = UnpackFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': p}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', p) writer.add(destpath.encode('utf-8'), f.open(), mode=f.mode) finally: os.chdir(oldcwd) try: shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', {'tempdir': tempdir}, 'Unable to delete {tempdir}') pass
def _install(self, dest): self.tempdir = tempfile.mkdtemp() self.binary = mozinstall.get_binary( mozinstall.install(src=dest, dest=self.tempdir), self.app_name)
kwargs.update(scraper_options.get(options.type, {})) if options.type == 'nightly': # DailyScraper generally chooses the right extension based on the platform, # but it looks for a .exe installer on Windows by default, and B2G nightlies # for Windows come only in the .zip variant, so specify that extension. if platform == "win32": kwargs.update({ 'extension': 'zip' }) build = DailyScraper(**kwargs) elif options.type == 'specific': build = DirectScraper(options.url, **kwargs) else: raise NotImplementedError('type %s not supported' % options.type) print "Initiating B2G download." build.download() # Install B2G Desktop to addon's data directory. installer = os.path.abspath(build.target) # Remove the existing installation, then install. platformdir = os.path.join(datadir, platform) shutil.rmtree(os.path.join(platformdir, installdirname), True) mozinstall.install(installer, platformdir) # Clean up. #shutil.rmtree(tmpdir)
def install(self, dest=None, channel="nightly"): """Install Firefox.""" branch = { "nightly": "mozilla-central", "beta": "mozilla-beta", "stable": "mozilla-stable" } scraper = { "nightly": "daily", "beta": "release", "stable": "release" } version = { "stable": "latest", "beta": "latest-beta", "nightly": "latest" } application_name = { "stable": "Firefox.app", "beta": "Firefox.app", "nightly": "Firefox Nightly.app" } if channel not in branch: raise ValueError("Unrecognised release channel: %s" % channel) from mozdownload import FactoryScraper import mozinstall platform = { "Linux": "linux", "Windows": "win", "Darwin": "mac" }.get(uname[0]) if platform is None: raise ValueError("Unable to construct a valid Firefox package name for current platform") if dest is None: # os.getcwd() doesn't include the venv path dest = os.path.join(os.getcwd(), "_venv") dest = os.path.join(dest, "browsers", channel) filename = FactoryScraper(scraper[channel], branch=branch[channel], version=version[channel], destination=dest).download() try: mozinstall.install(filename, dest) except mozinstall.mozinstall.InstallError: if platform == "mac" and os.path.exists(os.path.join(dest, application_name[channel])): # mozinstall will fail if nightly is already installed in the venv because # mac installation uses shutil.copy_tree mozinstall.uninstall(os.path.join(dest, application_name[channel])) mozinstall.install(filename, dest) else: raise os.remove(filename) return self.find_binary_path(dest)
def get_test_candidate(self): """Download and extract a build candidate. Build may either refer to a Firefox release identifier, package, or build directory. :param: build: str with firefox build :return: Installation path for the Firefox App """ location = '' candidate_app = '' if self.args.firefox == 'local': if Settings.is_mac(): location = '/Applications/Firefox.app/Contents/' candidate_app = os.path.join(location, 'MacOS', 'firefox') elif Settings.is_windows(): location = 'C:\\Program Files (x86)\\Mozilla Firefox' if not os.path.exists(location): location = 'C:\\Program Files\\Mozilla Firefox' candidate_app = os.path.join(location, 'firefox.exe') elif Settings.is_linux(): location = '/usr/lib/firefox' candidate_app = os.path.join(location, 'firefox') else: logger.critical('Platform not supported') self.finish(code=5) if not os.path.isdir(location): logger.critical( 'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/' ) self.finish(code=5) return candidate_app else: try: locale = 'ja-JP-mac' if self.args.locale == 'ja' and Settings.is_mac( ) else self.args.locale type, scraper_details = get_scraper_details( self.args.firefox, Settings.CHANNELS, os.path.join(IrisCore.get_working_dir(), 'cache'), locale) scraper = FactoryScraper(type, **scraper_details) firefox_dmg = scraper.download() install_folder = install(src=firefox_dmg, dest=IrisCore.get_current_run_dir()) binary = get_binary(install_folder, 'Firefox') channel = get_firefox_channel(binary) latest_type, latest_scraper_details = get_latest_scraper_details( channel) latest_path = FactoryScraper(latest_type, **latest_scraper_details).filename self.latest_version = get_version_from_path(latest_path) logger.info('Latest available version for %s channel is: %s' % (channel, self.latest_version)) return binary except errors.NotFoundError: logger.critical( 'Specified build (%s) has not been found. Closing Iris ...' % self.args.firefox) self.finish(5)
def process_package_artifact(self, filename, processed_filename): tempdir = tempfile.mkdtemp() try: self.log(logging.INFO, 'artifact', {'tempdir': tempdir}, 'Unpacking DMG into {tempdir}') mozinstall.install( filename, tempdir) # Doesn't handle already mounted DMG files nicely: # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)" # File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install # return artifacts.install_from(source, self.distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from # return self.install_from_hg(source, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg # return self.install_from_url(url, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url # return self.install_from_file(filename, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file # mozinstall.install(filename, tempdir) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install # install_dir = _install_dmg(src, dest) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg # subprocess.call('hdiutil detach %s -quiet' % appDir, bundle_dirs = glob.glob(mozpath.join(tempdir, '*.app')) if len(bundle_dirs) != 1: raise ValueError( 'Expected one source bundle, found: {}'.format( bundle_dirs)) [source] = bundle_dirs # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". paths_no_keep_path = ( 'Contents/MacOS', [ 'crashreporter.app/Contents/MacOS/crashreporter', 'firefox', 'firefox-bin', 'libfreebl3.dylib', 'liblgpllibs.dylib', # 'liblogalloc.dylib', 'libmozglue.dylib', 'libnss3.dylib', 'libnssckbi.dylib', 'libnssdbm3.dylib', 'libplugin_child_interpose.dylib', # 'libreplace_jemalloc.dylib', # 'libreplace_malloc.dylib', 'libsoftokn3.dylib', 'plugin-container.app/Contents/MacOS/plugin-container', 'updater.app/Contents/MacOS/updater', # 'xpcshell', 'XUL', ]) # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". paths_keep_path = ( 'Contents/Resources', [ 'browser/components/libbrowsercomps.dylib', 'dependentlibs.list', # 'firefox', 'gmp-clearkey/0.1/libclearkey.dylib', # 'gmp-fake/1.0/libfake.dylib', # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', 'webapprt-stub', ]) with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: root, paths = paths_no_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': path}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', os.path.basename(p)) writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) root, paths = paths_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, 'artifact', {'path': path}, 'Adding {path} to processed archive') destpath = mozpath.join('bin', p) writer.add(destpath.encode('utf-8'), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) finally: try: shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, 'artifact', {'tempdir': tempdir}, 'Unable to delete {tempdir}') pass
def install(self, dest=None, channel="nightly"): """Install Firefox.""" import mozinstall product = { "nightly": "firefox-nightly-latest-ssl", "beta": "firefox-beta-latest-ssl", "stable": "firefox-latest-ssl" } os_builds = { ("linux", "x86"): "linux", ("linux", "x86_64"): "linux64", ("win", "x86"): "win", ("win", "x86_64"): "win64", ("macos", "x86_64"): "osx", } os_key = (self.platform, uname[4]) if channel not in product: raise ValueError("Unrecognised release channel: %s" % channel) if os_key not in os_builds: raise ValueError("Unsupported platform: %s %s" % os_key) if dest is None: # os.getcwd() doesn't include the venv path dest = os.path.join(os.getcwd(), "_venv") dest = os.path.join(dest, "browsers", channel) if not os.path.exists(dest): os.makedirs(dest) url = "https://download.mozilla.org/?product=%s&os=%s&lang=en-US" % (product[channel], os_builds[os_key]) self.logger.info("Downloading Firefox from %s" % url) resp = requests.get(url) filename = None content_disposition = resp.headers.get('content-disposition') if content_disposition: filenames = re.findall("filename=(.+)", content_disposition) if filenames: filename = filenames[0] if not filename: filename = urlparse.urlsplit(resp.url).path.rsplit("/", 1)[1] if not filename: filename = "firefox.tar.bz2" installer_path = os.path.join(dest, filename) with open(installer_path, "w") as f: f.write(resp.content) try: mozinstall.install(installer_path, dest) except mozinstall.mozinstall.InstallError: if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))): # mozinstall will fail if nightly is already installed in the venv because # mac installation uses shutil.copy_tree mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))) mozinstall.install(filename, dest) else: raise os.remove(installer_path) return self.find_binary_path(dest)
def process_package_artifact(self, filename, processed_filename): tempdir = tempfile.mkdtemp() try: self.log(logging.INFO, "artifact", {"tempdir": tempdir}, "Unpacking DMG into {tempdir}") mozinstall.install(filename, tempdir) # Doesn't handle already mounted DMG files nicely: # InstallError: Failed to install "/Users/nalexander/.mozbuild/package-frontend/b38eeeb54cdcf744-firefox-44.0a1.en-US.mac.dmg (local variable 'appDir' referenced before assignment)" # File "/Users/nalexander/Mozilla/gecko/mobile/android/mach_commands.py", line 250, in artifact_install # return artifacts.install_from(source, self.distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 457, in install_from # return self.install_from_hg(source, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 445, in install_from_hg # return self.install_from_url(url, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 418, in install_from_url # return self.install_from_file(filename, distdir) # File "/Users/nalexander/Mozilla/gecko/python/mozbuild/mozbuild/artifacts.py", line 336, in install_from_file # mozinstall.install(filename, tempdir) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 117, in install # install_dir = _install_dmg(src, dest) # File "/Users/nalexander/Mozilla/gecko/objdir-dce/_virtualenv/lib/python2.7/site-packages/mozinstall/mozinstall.py", line 261, in _install_dmg # subprocess.call('hdiutil detach %s -quiet' % appDir, bundle_dirs = glob.glob(mozpath.join(tempdir, "*.app")) if len(bundle_dirs) != 1: raise ValueError("Expected one source bundle, found: {}".format(bundle_dirs)) [source] = bundle_dirs # These get copied into dist/bin without the path, so "root/a/b/c" -> "dist/bin/c". paths_no_keep_path = ( "Contents/MacOS", [ "crashreporter.app/Contents/MacOS/crashreporter", "firefox", "firefox-bin", "libfreebl3.dylib", "liblgpllibs.dylib", # 'liblogalloc.dylib', "libmozglue.dylib", "libnss3.dylib", "libnssckbi.dylib", "libnssdbm3.dylib", "libplugin_child_interpose.dylib", # 'libreplace_jemalloc.dylib', # 'libreplace_malloc.dylib', "libmozavutil.dylib", "libmozavcodec.dylib", "libsoftokn3.dylib", "plugin-container.app/Contents/MacOS/plugin-container", "updater.app/Contents/MacOS/org.mozilla.updater", # 'xpcshell', "XUL", ], ) # These get copied into dist/bin with the path, so "root/a/b/c" -> "dist/bin/a/b/c". paths_keep_path = ( "Contents/Resources", [ "browser/components/libbrowsercomps.dylib", "dependentlibs.list", # 'firefox', "gmp-clearkey/0.1/libclearkey.dylib", # 'gmp-fake/1.0/libfake.dylib', # 'gmp-fakeopenh264/1.0/libfakeopenh264.dylib', ], ) with JarWriter(file=processed_filename, optimize=False, compress_level=5) as writer: root, paths = paths_no_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, "artifact", {"path": path}, "Adding {path} to processed archive") destpath = mozpath.join("bin", os.path.basename(p)) writer.add(destpath.encode("utf-8"), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) root, paths = paths_keep_path finder = FileFinder(mozpath.join(source, root)) for path in paths: for p, f in finder.find(path): self.log(logging.INFO, "artifact", {"path": path}, "Adding {path} to processed archive") destpath = mozpath.join("bin", p) writer.add(destpath.encode("utf-8"), f, mode=os.stat(mozpath.join(finder.base, p)).st_mode) finally: try: shutil.rmtree(tempdir) except (OSError, IOError): self.log(logging.WARN, "artifact", {"tempdir": tempdir}, "Unable to delete {tempdir}") pass
#!/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os from mozdownload import FactoryScraper import mozinstall scraper = FactoryScraper('daily', branch=os.environ['MOZ_BRANCH'], locale=os.environ['LOCALE']) installer_path = scraper.download() install_path = mozinstall.install(installer_path, 'application') binary_path = mozinstall.get_binary(install_path, 'firefox') with open('binary.txt', 'w') as f: f.write(binary_path)
# Download latest build of B2G Desktop. scraper_keywords = { 'application': 'b2g', 'branch': 'mozilla-aurora', 'platform': platform, 'locale': 'en-US', 'version': None, 'directory': downloaddir } kwargs = scraper_keywords.copy() if platform == "win32": kwargs.update({ 'extension': 'zip' }) build = DailyScraper(**kwargs) print "Initiating download B2G Desktop latest build..." build.download() # Install B2G Desktop to addon's data directory. installer = os.path.abspath(build.target) # Remove the existing installation, then install. platformdir = os.path.join(datadir, platform) shutil.rmtree(os.path.join(platformdir, installdirname), True) mozinstall.install(installer, platformdir) # Clean up. #shutil.rmtree(tmpdir)
def prepare_b2g_desktop(self): file_name = self.download_file( lambda x: x.startswith(self.device) and x.endswith('%s.%s' % ( self.platform, self.suffix))) mozinstall.install(file_name, self.metadata['workdir']) os.remove(file_name)
def prepare_b2g_desktop(self): file_name = self.download_file(lambda x: x.startswith(self.device) and x.endswith('%s.%s' % (self.platform, self.suffix))) mozinstall.install(file_name, self.metadata['workdir']) os.remove(file_name)
self.repository_path) self._repository.clone() except Exception, e: raise Exception("Failure in setting up the mozmill-tests repository. " + e.message) if self.options.addons: self.prepare_addons() try: # Prepare the binary for the test run if mozinstall.is_installer(self.binary): install_path = tempfile.mkdtemp(".binary") print "Install build: %s" % self.binary self._folder = mozinstall.install(self.binary, install_path) self._application = mozinstall.get_binary(self._folder, self.options.application) else: # TODO: Ensure that self._folder is the same as from mozinstall folder = os.path.dirname(self.binary) self._folder = folder if not os.path.isdir(self.binary) else self.binary self._application = self.binary # Prepare the repository ini = application.ApplicationIni(self._application) repository_url = ini.get('App', 'SourceRepository') # Update the mozmill-test repository to match the Gecko branch branch_name = self._repository.identify_branch(repository_url) self._repository.update(branch_name)