def test_factory(self): """Testing various download scenarios for the factory.""" base_kwargs = {"base_url": self.wdir, "destination": self.temp_dir, "log_level": "ERROR"} for test in tests: kwargs = copy.deepcopy(base_kwargs) kwargs.update(test.get("kwargs")) build = FactoryScraper(test["scraper_type"], **kwargs) build.download() dir_content = os.listdir(self.temp_dir) self.assertTrue(test["fname"] in dir_content) mozfile.remove(os.path.join(self.temp_dir, test["fname"]))
def _get_test_candidate(version: str, locale: str) -> str or None: """Download and extract a build candidate. Build may either refer to a Firefox release identifier, package, or build directory. :param: build: str with firefox build :return: Installation path for the Firefox App """ logger.debug("Getting build, version %s, locale %s" % (version, locale)) if version == "local": candidate = PathManager.get_local_firefox_path() if candidate is None: logger.critical("Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/") return candidate elif os.path.isfile(version): return version else: try: s_t, s_d = _get_scraper_details( version, CHANNELS, os.path.join(PathManager.get_working_dir(), "cache"), locale ) scraper = FactoryScraper(s_t, **s_d) firefox_dmg = scraper.download() install_dir = install( src=firefox_dmg, dest=os.path.join( PathManager.get_temp_dir(), "firefox{}{}".format(normalize_str(version), normalize_str(locale)) ), ) return get_binary(install_dir, "Firefox") except errors.NotFoundError: logger.critical("Specified build {} has not been found. Closing Iris ...".format(version)) return None
def get_test_candidate(): """Download and extract a build candidate. Build may either refer to a Firefox release identifier, package, or build directory. :param: build: str with firefox build :return: Installation path for the Firefox App """ if parse_args().firefox == 'local': candidate = get_local_firefox_path() if candidate is None: logger.critical( 'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/' ) else: try: locale = 'ja-JP-mac' if parse_args( ).locale == 'ja' and Settings.is_mac() else parse_args().locale type, scraper_details = get_scraper_details( parse_args().firefox, Settings.CHANNELS, os.path.join(IrisCore.get_working_dir(), 'cache'), locale) scraper = FactoryScraper(type, **scraper_details) firefox_dmg = scraper.download() install_folder = install(src=firefox_dmg, dest=IrisCore.get_current_run_dir()) return get_binary(install_folder, 'Firefox') except errors.NotFoundError: logger.critical( 'Specified build (%s) has not been found. Closing Iris ...' % parse_args().firefox) return None
def run_tests(self, args): settings = config['test_types'][args.type] # In case the log folder does not exist yet we have to create it because # otherwise Marionette will fail for e.g. gecko.log (see bug 1211666) if settings['logs'].get('gecko.log'): try: os.makedirs(os.path.dirname(settings['logs']['gecko.log'])) except OSError: print('Failed to create log folder for {}'.format(settings['logs']['gecko.log'])) print('Downloading the installer: {}'.format(args.installer_url)) scraper = FactoryScraper('direct', url=args.installer_url, retry_attempts=5, retry_delay=30, ) installer_path = scraper.download() command = [ 'firefox-ui-update' if args.type == 'update' else 'firefox-ui-tests', '--installer', installer_path, '--workspace', os.getcwd(), '--log-tbpl', settings['logs']['tbpl.log'], ] if args.type == 'update': # Enable Gecko log to the console because the file would be overwritten # by the second update test command.extend(['--gecko-log', '-']) if args.update_channel: command.extend(['--update-channel', args.update_channel]) if args.update_target_version: command.extend(['--update-target-version', args.update_target_version]) if args.update_target_build_id: command.extend(['--update-target-buildid', args.update_target_build_id]) elif args.type == 'functional': command.extend(['--gecko-log', settings['logs']['gecko.log']]) manifests = [firefox_puppeteer.manifest, firefox_ui_tests.manifest_functional] command.extend(manifests) retval = 0 print('Calling command to execute tests: {}'.format(command)) retval = subprocess.call(command) # Save exit code into file for further processing in report submission try: with file('retval.txt', 'w') as f: f.write(str(retval)) except OSError as e: print('Failed to save return value: {}'.format(e)) # Delete http.log if tests were passing if not retval and settings['logs'].get('http.log'): shutil.rmtree(settings['logs']['http.log'], ignore_errors=True)
def install(self, dest=None, channel="nightly"): """Install Firefox.""" branch = { "nightly": "mozilla-central", "beta": "mozilla-beta", "stable": "mozilla-stable" } scraper = { "nightly": "daily", "beta": "release", "stable": "release" } version = { "stable": "latest", "beta": "latest-beta", "nightly": "latest" } if channel not in branch: raise ValueError("Unrecognised release channel: %s" % channel) from mozdownload import FactoryScraper import mozinstall if self.platform is None: raise ValueError("Unable to construct a valid Firefox package name for current platform") if dest is None: # os.getcwd() doesn't include the venv path dest = os.path.join(os.getcwd(), "_venv") dest = os.path.join(dest, "browsers", channel) scraper = FactoryScraper(scraper[channel], branch=branch[channel], version=version[channel], destination=dest) self.logger.info("Downloading Firefox from %s" % scraper.url) filename = scraper.download() try: mozinstall.install(filename, dest) except mozinstall.mozinstall.InstallError: if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))): # mozinstall will fail if nightly is already installed in the venv because # mac installation uses shutil.copy_tree mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))) mozinstall.install(filename, dest) else: raise os.remove(filename) return self.find_binary_path(dest)
def test_unknown_options(self): """Test that unknown optional options do not break the given scraper.""" base_kwargs = { 'base_url': self.wdir, 'destination': self.temp_dir, 'log_level': 'ERROR', } for test in tests: kwargs = copy.deepcopy(base_kwargs) kwargs.update(test.get('kwargs')) build = FactoryScraper(test['scraper_type'], **kwargs) build.download() dir_content = os.listdir(self.temp_dir) self.assertTrue(test['fname'] in dir_content) mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_unused_options(httpd, tmpdir, scraper_type, builds, filename, args, mocker): """Test that unknown optional options do not break the given scraper.""" base_kwargs = { 'base_url': httpd.get_url(), 'destination': str(tmpdir), 'log_level': 'ERROR', } query_builds_by_revision = \ mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision') if builds: query_builds_by_revision.return_value = builds kwargs = copy.deepcopy(base_kwargs) kwargs.update(args) build = FactoryScraper(scraper_type, **kwargs) build.download() dir_content = os.listdir(str(tmpdir)) assert filename in dir_content
def test_factory(httpd, tmpdir, scraper_type, builds, filename, args, mocker): """Testing various download scenarios for the factory.""" query_builds_by_revision \ = mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision') base_kwargs = { 'base_url': httpd.get_url(), 'destination': str(tmpdir), 'log_level': 'ERROR', } if builds: query_builds_by_revision.return_value = builds kwargs = copy.deepcopy(base_kwargs) kwargs.update(args) build = FactoryScraper(scraper_type, **kwargs) build.download() dir_content = os.listdir(str(tmpdir)) assert filename in dir_content
def download(channel): if channel == 'ALL': download_all() return ch_type = config.get(channel, 'type') ch_version = config.get(channel, 'version') ch_branch = config.get(channel, 'branch') # PLATFORM is uppercased here since the platform comes from the OS-specific # config files, whereas the other flags generically come from channels.ini. ch_platform = env.get(channel, 'PLATFORM') download_filename = env.get(channel, 'DOWNLOAD_FILENAME') download_path = os.path.join(BASE_DIR, download_filename) args = {"channel": channel, "download_path": download_path} print("Downloading {channel} to {download_path}".format(**args)) scraper = FactoryScraper( ch_type, version=ch_version, branch=ch_branch, destination=download_path, platform=ch_platform ) scraper.download() is_recent_file = modification_date(download_path) > SCRIPT_START_TIME firefox_bin = env.get(channel, 'PATH_FIREFOX_BIN_ENV') # If the *.dmg file was downloaded recently, or we don't have the *.app # file installed, install the current Firefox channel. if is_recent_file or not os.path.exists(firefox_bin): install(channel) else: firefox_version = get_firefox_version(channel) args = {"channel": channel, "version": firefox_version} msg = "You have the latest version of {channel} installed ({version})." Log.header(msg.format(**args))
def firefox(pytestconfig, tmpdir_factory): binary = pytestconfig.getoption('firefox') if binary is None: cache_dir = str(pytestconfig.cache.makedir('firefox')) scraper = FactoryScraper('daily', destination=cache_dir) build_path = scraper.download() install_path = str(tmpdir_factory.mktemp('firefox')) install_dir = mozinstall.install(src=build_path, dest=install_path) binary = mozinstall.get_binary(install_dir, 'firefox') version = mozversion.get_version(binary) if hasattr(pytestconfig, '_metadata'): pytestconfig._metadata.update(version) return binary
def test_unknown_options(self, query_builds_by_revision): """Test that unknown optional options do not break the given scraper.""" base_kwargs = { 'base_url': self.wdir, 'destination': self.temp_dir, 'log_level': 'ERROR', } for test in tests: if test.get('builds'): query_builds_by_revision.return_value = test['builds'] kwargs = copy.deepcopy(base_kwargs) kwargs.update(test.get('kwargs')) build = FactoryScraper(test['scraper_type'], **kwargs) build.download() dir_content = os.listdir(self.temp_dir) self.assertTrue(test['fname'] in dir_content) mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_factory(self, query_builds_by_revision): """Testing various download scenarios for the factory.""" base_kwargs = { 'base_url': self.wdir, 'destination': self.temp_dir, 'log_level': 'ERROR', } for test in tests: if test.get('builds'): query_builds_by_revision.return_value = test['builds'] kwargs = copy.deepcopy(base_kwargs) kwargs.update(test.get('kwargs')) build = FactoryScraper(test['scraper_type'], **kwargs) build.download() dir_content = os.listdir(self.temp_dir) self.assertTrue(test['fname'] in dir_content) mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def firefox(pytestconfig, tmpdir_factory): binary = os.getenv('MOZREGRESSION_BINARY', pytestconfig.getoption('firefox')) if binary is None: cache_dir = str(pytestconfig.cache.makedir('firefox')) scraper = FactoryScraper('daily', destination=cache_dir) build_path = scraper.download() install_path = str(tmpdir_factory.mktemp('firefox')) install_dir = mozinstall.install(src=build_path, dest=install_path) binary = mozinstall.get_binary(install_dir, 'firefox') version = mozversion.get_version(binary) if hasattr(pytestconfig, '_metadata'): pytestconfig._metadata.update(version) return binary
def get_test_candidate(self): """Download and extract a build candidate. Build may either refer to a Firefox release identifier, package, or build directory. :param: build: str with firefox build :return: Installation path for the Firefox App """ location = '' candidate_app = '' if self.args.firefox == 'local': if Settings.is_mac(): location = '/Applications/Firefox.app/Contents/' candidate_app = os.path.join(location, 'MacOS', 'firefox') elif Settings.is_windows(): location = 'C:\\Program Files (x86)\\Mozilla Firefox' if not os.path.exists(location): location = 'C:\\Program Files\\Mozilla Firefox' candidate_app = os.path.join(location, 'firefox.exe') elif Settings.is_linux(): location = '/usr/lib/firefox' candidate_app = os.path.join(location, 'firefox') else: logger.critical('Platform not supported') self.finish(code=5) if not os.path.isdir(location): logger.critical( 'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/' ) self.finish(code=5) return candidate_app else: try: locale = 'ja-JP-mac' if self.args.locale == 'ja' and Settings.is_mac( ) else self.args.locale type, scraper_details = get_scraper_details( self.args.firefox, Settings.CHANNELS, os.path.join(IrisCore.get_working_dir(), 'cache'), locale) scraper = FactoryScraper(type, **scraper_details) firefox_dmg = scraper.download() install_folder = install(src=firefox_dmg, dest=IrisCore.get_current_run_dir()) binary = get_binary(install_folder, 'Firefox') channel = get_firefox_channel(binary) latest_type, latest_scraper_details = get_latest_scraper_details( channel) latest_path = FactoryScraper(latest_type, **latest_scraper_details).filename self.latest_version = get_version_from_path(latest_path) logger.info('Latest available version for %s channel is: %s' % (channel, self.latest_version)) return binary except errors.NotFoundError: logger.critical( 'Specified build (%s) has not been found. Closing Iris ...' % self.args.firefox) self.finish(5)
#!/bin/env python # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os from mozdownload import FactoryScraper import mozinstall scraper = FactoryScraper('daily', branch=os.environ['MOZ_BRANCH'], locale=os.environ['LOCALE']) installer_path = scraper.download() install_path = mozinstall.install(installer_path, 'application') binary_path = mozinstall.get_binary(install_path, 'firefox') with open('binary.txt', 'w') as f: f.write(binary_path)