def test_factory(self):
"""Testing various download scenarios for the factory."""
base_kwargs = {"base_url": self.wdir, "destination": self.temp_dir, "log_level": "ERROR"}
for test in tests:
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get("kwargs"))
build = FactoryScraper(test["scraper_type"], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test["fname"] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test["fname"]))
def _get_test_candidate(version: str, locale: str) -> str or None:
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param: build: str with firefox build
:return: Installation path for the Firefox App
"""
logger.debug("Getting build, version %s, locale %s" % (version, locale))
if version == "local":
candidate = PathManager.get_local_firefox_path()
if candidate is None:
logger.critical("Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/")
return candidate
elif os.path.isfile(version):
return version
else:
try:
s_t, s_d = _get_scraper_details(
version, CHANNELS, os.path.join(PathManager.get_working_dir(), "cache"), locale
)
scraper = FactoryScraper(s_t, **s_d)
firefox_dmg = scraper.download()
install_dir = install(
src=firefox_dmg,
dest=os.path.join(
PathManager.get_temp_dir(), "firefox{}{}".format(normalize_str(version), normalize_str(locale))
),
)
return get_binary(install_dir, "Firefox")
except errors.NotFoundError:
logger.critical("Specified build {} has not been found. Closing Iris ...".format(version))
return None
def get_test_candidate():
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param: build: str with firefox build
:return: Installation path for the Firefox App
"""
if parse_args().firefox == 'local':
candidate = get_local_firefox_path()
if candidate is None:
logger.critical(
'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/'
)
else:
try:
locale = 'ja-JP-mac' if parse_args(
).locale == 'ja' and Settings.is_mac() else parse_args().locale
type, scraper_details = get_scraper_details(
parse_args().firefox, Settings.CHANNELS,
os.path.join(IrisCore.get_working_dir(), 'cache'), locale)
scraper = FactoryScraper(type, **scraper_details)
firefox_dmg = scraper.download()
install_folder = install(src=firefox_dmg,
dest=IrisCore.get_current_run_dir())
return get_binary(install_folder, 'Firefox')
except errors.NotFoundError:
logger.critical(
'Specified build (%s) has not been found. Closing Iris ...' %
parse_args().firefox)
return None
def run_tests(self, args):
settings = config['test_types'][args.type]
# In case the log folder does not exist yet we have to create it because
# otherwise Marionette will fail for e.g. gecko.log (see bug 1211666)
if settings['logs'].get('gecko.log'):
try:
os.makedirs(os.path.dirname(settings['logs']['gecko.log']))
except OSError:
print('Failed to create log folder for {}'.format(settings['logs']['gecko.log']))
print('Downloading the installer: {}'.format(args.installer_url))
scraper = FactoryScraper('direct',
url=args.installer_url,
retry_attempts=5,
retry_delay=30,
)
installer_path = scraper.download()
command = [
'firefox-ui-update' if args.type == 'update' else 'firefox-ui-tests',
'--installer', installer_path,
'--workspace', os.getcwd(),
'--log-tbpl', settings['logs']['tbpl.log'],
]
if args.type == 'update':
# Enable Gecko log to the console because the file would be overwritten
# by the second update test
command.extend(['--gecko-log', '-'])
if args.update_channel:
command.extend(['--update-channel', args.update_channel])
if args.update_target_version:
command.extend(['--update-target-version', args.update_target_version])
if args.update_target_build_id:
command.extend(['--update-target-buildid', args.update_target_build_id])
elif args.type == 'functional':
command.extend(['--gecko-log', settings['logs']['gecko.log']])
manifests = [firefox_puppeteer.manifest, firefox_ui_tests.manifest_functional]
command.extend(manifests)
retval = 0
print('Calling command to execute tests: {}'.format(command))
retval = subprocess.call(command)
# Save exit code into file for further processing in report submission
try:
with file('retval.txt', 'w') as f:
f.write(str(retval))
except OSError as e:
print('Failed to save return value: {}'.format(e))
# Delete http.log if tests were passing
if not retval and settings['logs'].get('http.log'):
shutil.rmtree(settings['logs']['http.log'], ignore_errors=True)
def install(self, dest=None, channel="nightly"):
"""Install Firefox."""
branch = {
"nightly": "mozilla-central",
"beta": "mozilla-beta",
"stable": "mozilla-stable"
}
scraper = {
"nightly": "daily",
"beta": "release",
"stable": "release"
}
version = {
"stable": "latest",
"beta": "latest-beta",
"nightly": "latest"
}
if channel not in branch:
raise ValueError("Unrecognised release channel: %s" % channel)
from mozdownload import FactoryScraper
import mozinstall
if self.platform is None:
raise ValueError("Unable to construct a valid Firefox package name for current platform")
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers", channel)
scraper = FactoryScraper(scraper[channel],
branch=branch[channel],
version=version[channel],
destination=dest)
self.logger.info("Downloading Firefox from %s" % scraper.url)
filename = scraper.download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError:
if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app")))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def test_unknown_options(self):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_unused_options(httpd, tmpdir, scraper_type, builds, filename, args, mocker):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
query_builds_by_revision = \
mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def test_factory(httpd, tmpdir, scraper_type, builds, filename, args, mocker):
"""Testing various download scenarios for the factory."""
query_builds_by_revision \
= mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def download(channel):
if channel == 'ALL':
download_all()
return
ch_type = config.get(channel, 'type')
ch_version = config.get(channel, 'version')
ch_branch = config.get(channel, 'branch')
# PLATFORM is uppercased here since the platform comes from the OS-specific
# config files, whereas the other flags generically come from channels.ini.
ch_platform = env.get(channel, 'PLATFORM')
download_filename = env.get(channel, 'DOWNLOAD_FILENAME')
download_path = os.path.join(BASE_DIR, download_filename)
args = {"channel": channel, "download_path": download_path}
print("Downloading {channel} to {download_path}".format(**args))
scraper = FactoryScraper(
ch_type,
version=ch_version,
branch=ch_branch,
destination=download_path,
platform=ch_platform
)
scraper.download()
is_recent_file = modification_date(download_path) > SCRIPT_START_TIME
firefox_bin = env.get(channel, 'PATH_FIREFOX_BIN_ENV')
# If the *.dmg file was downloaded recently, or we don't have the *.app
# file installed, install the current Firefox channel.
if is_recent_file or not os.path.exists(firefox_bin):
install(channel)
else:
firefox_version = get_firefox_version(channel)
args = {"channel": channel, "version": firefox_version}
msg = "You have the latest version of {channel} installed ({version})."
Log.header(msg.format(**args))
def test_unused_options(httpd, tmpdir, scraper_type, builds, filename, args,
mocker):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
query_builds_by_revision = \
mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def firefox(pytestconfig, tmpdir_factory):
binary = pytestconfig.getoption('firefox')
if binary is None:
cache_dir = str(pytestconfig.cache.makedir('firefox'))
scraper = FactoryScraper('daily', destination=cache_dir)
build_path = scraper.download()
install_path = str(tmpdir_factory.mktemp('firefox'))
install_dir = mozinstall.install(src=build_path, dest=install_path)
binary = mozinstall.get_binary(install_dir, 'firefox')
version = mozversion.get_version(binary)
if hasattr(pytestconfig, '_metadata'):
pytestconfig._metadata.update(version)
return binary
def test_unknown_options(self, query_builds_by_revision):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
if test.get('builds'):
query_builds_by_revision.return_value = test['builds']
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_factory(self, query_builds_by_revision):
"""Testing various download scenarios for the factory."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
if test.get('builds'):
query_builds_by_revision.return_value = test['builds']
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def firefox(pytestconfig, tmpdir_factory):
binary = os.getenv('MOZREGRESSION_BINARY',
pytestconfig.getoption('firefox'))
if binary is None:
cache_dir = str(pytestconfig.cache.makedir('firefox'))
scraper = FactoryScraper('daily', destination=cache_dir)
build_path = scraper.download()
install_path = str(tmpdir_factory.mktemp('firefox'))
install_dir = mozinstall.install(src=build_path, dest=install_path)
binary = mozinstall.get_binary(install_dir, 'firefox')
version = mozversion.get_version(binary)
if hasattr(pytestconfig, '_metadata'):
pytestconfig._metadata.update(version)
return binary
def get_test_candidate(self):
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param:
build: str with firefox build
:return:
Installation path for the Firefox App
"""
location = ''
candidate_app = ''
if self.args.firefox == 'local':
if Settings.is_mac():
location = '/Applications/Firefox.app/Contents/'
candidate_app = os.path.join(location, 'MacOS', 'firefox')
elif Settings.is_windows():
location = 'C:\\Program Files (x86)\\Mozilla Firefox'
if not os.path.exists(location):
location = 'C:\\Program Files\\Mozilla Firefox'
candidate_app = os.path.join(location, 'firefox.exe')
elif Settings.is_linux():
location = '/usr/lib/firefox'
candidate_app = os.path.join(location, 'firefox')
else:
logger.critical('Platform not supported')
self.finish(code=5)
if not os.path.isdir(location):
logger.critical(
'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/'
)
self.finish(code=5)
return candidate_app
else:
try:
locale = 'ja-JP-mac' if self.args.locale == 'ja' and Settings.is_mac(
) else self.args.locale
type, scraper_details = get_scraper_details(
self.args.firefox, Settings.CHANNELS,
os.path.join(IrisCore.get_working_dir(), 'cache'), locale)
scraper = FactoryScraper(type, **scraper_details)
firefox_dmg = scraper.download()
install_folder = install(src=firefox_dmg,
dest=IrisCore.get_current_run_dir())
binary = get_binary(install_folder, 'Firefox')
channel = get_firefox_channel(binary)
latest_type, latest_scraper_details = get_latest_scraper_details(
channel)
latest_path = FactoryScraper(latest_type,
**latest_scraper_details).filename
self.latest_version = get_version_from_path(latest_path)
logger.info('Latest available version for %s channel is: %s' %
(channel, self.latest_version))
return binary
except errors.NotFoundError:
logger.critical(
'Specified build (%s) has not been found. Closing Iris ...'
% self.args.firefox)
self.finish(5)
#!/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from mozdownload import FactoryScraper
import mozinstall
scraper = FactoryScraper('daily',
branch=os.environ['MOZ_BRANCH'],
locale=os.environ['LOCALE'])
installer_path = scraper.download()
install_path = mozinstall.install(installer_path, 'application')
binary_path = mozinstall.get_binary(install_path, 'firefox')
with open('binary.txt', 'w') as f:
f.write(binary_path)
