def get_test_candidate():
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param: build: str with firefox build
:return: Installation path for the Firefox App
"""
if parse_args().firefox == 'local':
candidate = get_local_firefox_path()
if candidate is None:
logger.critical(
'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/'
)
else:
try:
locale = 'ja-JP-mac' if parse_args(
).locale == 'ja' and Settings.is_mac() else parse_args().locale
type, scraper_details = get_scraper_details(
parse_args().firefox, Settings.CHANNELS,
os.path.join(IrisCore.get_working_dir(), 'cache'), locale)
scraper = FactoryScraper(type, **scraper_details)
firefox_dmg = scraper.download()
install_folder = install(src=firefox_dmg,
dest=IrisCore.get_current_run_dir())
return get_binary(install_folder, 'Firefox')
except errors.NotFoundError:
logger.critical(
'Specified build (%s) has not been found. Closing Iris ...' %
parse_args().firefox)
return None
def _get_test_candidate(version: str, locale: str) -> str or None:
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param: build: str with firefox build
:return: Installation path for the Firefox App
"""
logger.debug("Getting build, version %s, locale %s" % (version, locale))
if version == "local":
candidate = PathManager.get_local_firefox_path()
if candidate is None:
logger.critical("Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/")
return candidate
elif os.path.isfile(version):
return version
else:
try:
s_t, s_d = _get_scraper_details(
version, CHANNELS, os.path.join(PathManager.get_working_dir(), "cache"), locale
)
scraper = FactoryScraper(s_t, **s_d)
firefox_dmg = scraper.download()
install_dir = install(
src=firefox_dmg,
dest=os.path.join(
PathManager.get_temp_dir(), "firefox{}{}".format(normalize_str(version), normalize_str(locale))
),
)
return get_binary(install_dir, "Firefox")
except errors.NotFoundError:
logger.critical("Specified build {} has not been found. Closing Iris ...".format(version))
return None
def run_tests(self, args):
settings = config['test_types'][args.type]
# In case the log folder does not exist yet we have to create it because
# otherwise Marionette will fail for e.g. gecko.log (see bug 1211666)
if settings['logs'].get('gecko.log'):
try:
os.makedirs(os.path.dirname(settings['logs']['gecko.log']))
except OSError:
print('Failed to create log folder for {}'.format(settings['logs']['gecko.log']))
print('Downloading the installer: {}'.format(args.installer_url))
scraper = FactoryScraper('direct',
url=args.installer_url,
retry_attempts=5,
retry_delay=30,
)
installer_path = scraper.download()
command = [
'firefox-ui-update' if args.type == 'update' else 'firefox-ui-tests',
'--installer', installer_path,
'--workspace', os.getcwd(),
'--log-tbpl', settings['logs']['tbpl.log'],
]
if args.type == 'update':
# Enable Gecko log to the console because the file would be overwritten
# by the second update test
command.extend(['--gecko-log', '-'])
if args.update_channel:
command.extend(['--update-channel', args.update_channel])
if args.update_target_version:
command.extend(['--update-target-version', args.update_target_version])
if args.update_target_build_id:
command.extend(['--update-target-buildid', args.update_target_build_id])
elif args.type == 'functional':
command.extend(['--gecko-log', settings['logs']['gecko.log']])
manifests = [firefox_puppeteer.manifest, firefox_ui_tests.manifest_functional]
command.extend(manifests)
retval = 0
print('Calling command to execute tests: {}'.format(command))
retval = subprocess.call(command)
# Save exit code into file for further processing in report submission
try:
with file('retval.txt', 'w') as f:
f.write(str(retval))
except OSError as e:
print('Failed to save return value: {}'.format(e))
# Delete http.log if tests were passing
if not retval and settings['logs'].get('http.log'):
shutil.rmtree(settings['logs']['http.log'], ignore_errors=True)
def install(self, dest=None, channel="nightly"):
"""Install Firefox."""
branch = {
"nightly": "mozilla-central",
"beta": "mozilla-beta",
"stable": "mozilla-stable"
}
scraper = {
"nightly": "daily",
"beta": "release",
"stable": "release"
}
version = {
"stable": "latest",
"beta": "latest-beta",
"nightly": "latest"
}
if channel not in branch:
raise ValueError("Unrecognised release channel: %s" % channel)
from mozdownload import FactoryScraper
import mozinstall
if self.platform is None:
raise ValueError("Unable to construct a valid Firefox package name for current platform")
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers", channel)
scraper = FactoryScraper(scraper[channel],
branch=branch[channel],
version=version[channel],
destination=dest)
self.logger.info("Downloading Firefox from %s" % scraper.url)
filename = scraper.download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError:
if self.platform == "macos" and os.path.exists(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app"))):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, self.application_name.get(channel, "Firefox Nightly.app")))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def firefox(pytestconfig, tmpdir_factory):
binary = pytestconfig.getoption('firefox')
if binary is None:
cache_dir = str(pytestconfig.cache.makedir('firefox'))
scraper = FactoryScraper('daily', destination=cache_dir)
build_path = scraper.download()
install_path = str(tmpdir_factory.mktemp('firefox'))
install_dir = mozinstall.install(src=build_path, dest=install_path)
binary = mozinstall.get_binary(install_dir, 'firefox')
version = mozversion.get_version(binary)
if hasattr(pytestconfig, '_metadata'):
pytestconfig._metadata.update(version)
return binary
def firefox(pytestconfig, tmpdir_factory):
binary = os.getenv('MOZREGRESSION_BINARY',
pytestconfig.getoption('firefox'))
if binary is None:
cache_dir = str(pytestconfig.cache.makedir('firefox'))
scraper = FactoryScraper('daily', destination=cache_dir)
build_path = scraper.download()
install_path = str(tmpdir_factory.mktemp('firefox'))
install_dir = mozinstall.install(src=build_path, dest=install_path)
binary = mozinstall.get_binary(install_dir, 'firefox')
version = mozversion.get_version(binary)
if hasattr(pytestconfig, '_metadata'):
pytestconfig._metadata.update(version)
return binary
def test_factory(self):
"""Testing various download scenarios for the factory."""
base_kwargs = {"base_url": self.wdir, "destination": self.temp_dir, "log_level": "ERROR"}
for test in tests:
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get("kwargs"))
build = FactoryScraper(test["scraper_type"], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test["fname"] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test["fname"]))
def install(self, dest=None):
"""Install Firefox."""
from mozdownload import FactoryScraper
import mozinstall
if dest is None:
dest = os.getcwd()
filename = FactoryScraper('daily',
branch='mozilla-central',
destination=dest).download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError as e:
if uname[0] == "Darwin":
# mozinstall will fail here if nightly is already installed in the venv
# This only occurs on macOS because shutil.copy_tree() is called in
# mozinstall._install_dmg and will fail if the file already exists.
# copytree isn't used while installing on Windows/linux, so the same error
# won't be thrown if we try to rewrite there.
mozinstall.uninstall(dest + '/Firefox Nightly.app')
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return find_executable("firefox", os.path.join(dest, "firefox"))
def query_file_url(properties, property_overrides=None):
"""Query for the specified build by using mozdownload.
This method uses the properties as received via Mozilla Pulse to query
the build via mozdownload. Use the property overrides to customize the
query, e.g. different build or test package files.
"""
property_overrides = property_overrides or {}
kwargs = {
# General arguments for all types of builds
'build_type': properties.get('build_type'),
'locale': properties.get('locale'),
'platform': properties.get('platform'),
'retry_attempts': 5,
'retry_delay': 30,
# Arguments for daily builds
'branch': properties.get('branch'),
'build_id': properties.get('buildid'),
# Arguments for candidate builds
'build_number': properties.get('build_number'),
'version': properties.get('version'),
}
# Update arguments with given overrides
kwargs.update(property_overrides)
return FactoryScraper(kwargs['build_type'], **kwargs).url
def test_non_release_non_candidate_devedition(httpd, tmpdir):
"""Test that non-relase and non-candidate scraper type for devedition raises exception"""
with pytest.raises(NotSupportedError):
FactoryScraper('daily',
destination=str(tmpdir),
base_url=httpd.get_url(),
application='devedition',
version='60.0b1')
def test_non_daily_fennec(httpd, tmpdir):
"""Test that non-daily scraper_type for fennec raises exception"""
with pytest.raises(NotSupportedError):
FactoryScraper('candidate',
destination=str(tmpdir),
base_url=httpd.get_url(),
application='fennec',
version='60.0b1')
def test_unknown_options(self):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_factory(httpd, tmpdir, scraper_type, builds, filename, args, mocker):
"""Testing various download scenarios for the factory."""
query_builds_by_revision \
= mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def get_firefox_latest_version(binary: str, locale: str) -> str or None:
"""Returns Firefox latest available version."""
if binary is None:
return None
channel = FirefoxUtils.get_firefox_channel(binary)
latest_type, latest_scraper_details = _get_latest_scraper_details(channel)
latest_path = FactoryScraper(latest_type, **latest_scraper_details).filename
latest_version = get_version_from_path(latest_path, locale)
logger.info("Latest available version for {} channel is: {}".format(channel, latest_version))
return latest_version
def test_unused_options(httpd, tmpdir, scraper_type, builds, filename, args, mocker):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
query_builds_by_revision = \
mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def download(channel):
if channel == 'ALL':
download_all()
return
ch_type = config.get(channel, 'type')
ch_version = config.get(channel, 'version')
ch_branch = config.get(channel, 'branch')
# PLATFORM is uppercased here since the platform comes from the OS-specific
# config files, whereas the other flags generically come from channels.ini.
ch_platform = env.get(channel, 'PLATFORM')
download_filename = env.get(channel, 'DOWNLOAD_FILENAME')
download_path = os.path.join(BASE_DIR, download_filename)
args = {"channel": channel, "download_path": download_path}
print("Downloading {channel} to {download_path}".format(**args))
scraper = FactoryScraper(
ch_type,
version=ch_version,
branch=ch_branch,
destination=download_path,
platform=ch_platform
)
scraper.download()
is_recent_file = modification_date(download_path) > SCRIPT_START_TIME
firefox_bin = env.get(channel, 'PATH_FIREFOX_BIN_ENV')
# If the *.dmg file was downloaded recently, or we don't have the *.app
# file installed, install the current Firefox channel.
if is_recent_file or not os.path.exists(firefox_bin):
install(channel)
else:
firefox_version = get_firefox_version(channel)
args = {"channel": channel, "version": firefox_version}
msg = "You have the latest version of {channel} installed ({version})."
Log.header(msg.format(**args))
def test_unused_options(httpd, tmpdir, scraper_type, builds, filename, args,
mocker):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': httpd.get_url(),
'destination': str(tmpdir),
'log_level': 'ERROR',
}
query_builds_by_revision = \
mocker.patch('mozdownload.treeherder.Treeherder.query_builds_by_revision')
if builds:
query_builds_by_revision.return_value = builds
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(args)
build = FactoryScraper(scraper_type, **kwargs)
build.download()
dir_content = os.listdir(str(tmpdir))
assert filename in dir_content
def test_unknown_options(self, query_builds_by_revision):
"""Test that unknown optional options do not break the given scraper."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
if test.get('builds'):
query_builds_by_revision.return_value = test['builds']
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def test_factory(self, query_builds_by_revision):
"""Testing various download scenarios for the factory."""
base_kwargs = {
'base_url': self.wdir,
'destination': self.temp_dir,
'log_level': 'ERROR',
}
for test in tests:
if test.get('builds'):
query_builds_by_revision.return_value = test['builds']
kwargs = copy.deepcopy(base_kwargs)
kwargs.update(test.get('kwargs'))
build = FactoryScraper(test['scraper_type'], **kwargs)
build.download()
dir_content = os.listdir(self.temp_dir)
self.assertTrue(test['fname'] in dir_content)
mozfile.remove(os.path.join(self.temp_dir, test['fname']))
def get_firefox_latest_version(binary):
"""Returns Firefox latest available version."""
if binary is None:
return None
channel = get_firefox_channel(binary)
latest_type, latest_scraper_details = get_latest_scraper_details(channel)
latest_path = FactoryScraper(latest_type,
**latest_scraper_details).filename
latest_version = get_version_from_path(latest_path)
logger.info('Latest available version for %s channel is: %s' %
(channel, latest_version))
return latest_version
def test_unused_options_direct_scraper(httpd, tmpdir):
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
scraper = FactoryScraper(scraper_type='direct',
url=test_url,
destination=str(tmpdir),
platform='win32',
branch='mozilla-central',
build_id='20131001030204',
revision='8fcac92cfcad',
date='2013-10-01',
debug_build=True,
version='23.0.1')
assert scraper.url == test_url
assert scraper.filename == os.path.join(str(tmpdir), filename)
def install(self, dest=None):
"""Install Firefox."""
from mozdownload import FactoryScraper
import mozinstall
platform = {
"Linux": "linux",
"Windows": "win",
"Darwin": "mac"
}.get(uname[0])
if platform is None:
raise ValueError(
"Unable to construct a valid Firefox package name for current platform"
)
if dest is None:
# os.getcwd() doesn't include the venv path
dest = os.path.join(os.getcwd(), "_venv")
dest = os.path.join(dest, "browsers")
filename = FactoryScraper("daily",
branch="mozilla-central",
destination=dest).download()
try:
mozinstall.install(filename, dest)
except mozinstall.mozinstall.InstallError as e:
if platform == "mac" and os.path.exists(
os.path.join(dest, "Firefox Nightly.app")):
# mozinstall will fail if nightly is already installed in the venv because
# mac installation uses shutil.copy_tree
mozinstall.uninstall(os.path.join(dest, "Firefox Nightly.app"))
mozinstall.install(filename, dest)
else:
raise
os.remove(filename)
return self.find_binary_path(dest)
def test_unknown_options_direct_scraper(self):
filename = 'download_test.txt'
test_url = urljoin(self.wdir, filename)
scraper = FactoryScraper(
scraper_type='direct',
url=test_url,
destination=self.temp_dir,
logger=self.logger,
platform='win32',
branch='mozilla-central',
build_id='20131001030204',
revision='8fcac92cfcad',
date='2013-10-01',
debug_build=True,
version='23.0.1',
)
self.assertEqual(scraper.url, test_url)
self.assertEqual(scraper.filename,
os.path.join(self.temp_dir, filename))
def query_file_url(self, properties, property_overrides=None):
"""Query for the specified build by using mozdownload.
This method uses the properties as received via Mozilla Pulse to query
the build via mozdownload. Use the property overrides to customize the
query, e.g. different build or test package files.
"""
property_overrides = property_overrides or {}
if property_overrides.get('build_type'):
build_type = property_overrides['build_type']
else:
build_type = 'candidate' if 'release-' in properties[
'tree'] else 'daily'
kwargs = {
# General arguments for all types of builds
'locale': properties['locale'],
'platform': self.get_platform_identifier(properties['platform']),
'retry_attempts': 5,
'retry_delay': 30,
# Arguments for daily builds
'branch': properties.get('branch'),
'build_id': properties.get('buildid'),
# Arguments for candidate builds
'build_number': properties.get('build_number'),
'version': properties.get('version'),
}
# Update arguments with given overrides
kwargs.update(property_overrides)
self.logger.debug('Retrieve url for a {} file: {}'.format(
build_type, kwargs))
scraper = FactoryScraper(build_type, **kwargs)
return scraper.url
#!/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from mozdownload import FactoryScraper
import mozinstall
scraper = FactoryScraper('daily',
branch=os.environ['MOZ_BRANCH'],
locale=os.environ['LOCALE'])
installer_path = scraper.download()
install_path = mozinstall.install(installer_path, 'application')
binary_path = mozinstall.get_binary(install_path, 'firefox')
with open('binary.txt', 'w') as f:
f.write(binary_path)
def test_release_without_version(httpd, tmpdir):
"""Test that missing mandatory options for release builds raise an exception"""
with pytest.raises(ValueError):
FactoryScraper('release',
destination=str(tmpdir),
base_url=httpd.get_url())
+ bcolors.ENDC
)
print("")
print(
"_____________________________________________________________________________________"
)
print("")
print(bcolors.OKBLUE + "Testing Mozdownload" + bcolors.ENDC)
print("")
try:
from mozdownload import FactoryScraper
scraper = FactoryScraper("daily")
print(scraper.url)
print(scraper.filename)
except (IOError, OSError) as e:
print(bcolors.FAIL + "ERROR" + bcolors.ENDC)
print(e)
except:
print("Unexpected error:", sys.exc_info()[0])
raise
else:
print("")
print("Version: " + mozdownload.__version__)
print("")
print(
bcolors.BOLD + bcolors.OKGREEN + "Mozdownload working correctly." + bcolors.ENDC
)
def get_test_candidate(self):
"""Download and extract a build candidate.
Build may either refer to a Firefox release identifier, package, or build directory.
:param:
build: str with firefox build
:return:
Installation path for the Firefox App
"""
location = ''
candidate_app = ''
if self.args.firefox == 'local':
if Settings.is_mac():
location = '/Applications/Firefox.app/Contents/'
candidate_app = os.path.join(location, 'MacOS', 'firefox')
elif Settings.is_windows():
location = 'C:\\Program Files (x86)\\Mozilla Firefox'
if not os.path.exists(location):
location = 'C:\\Program Files\\Mozilla Firefox'
candidate_app = os.path.join(location, 'firefox.exe')
elif Settings.is_linux():
location = '/usr/lib/firefox'
candidate_app = os.path.join(location, 'firefox')
else:
logger.critical('Platform not supported')
self.finish(code=5)
if not os.path.isdir(location):
logger.critical(
'Firefox not found. Please download if from https://www.mozilla.org/en-US/firefox/new/'
)
self.finish(code=5)
return candidate_app
else:
try:
locale = 'ja-JP-mac' if self.args.locale == 'ja' and Settings.is_mac(
) else self.args.locale
type, scraper_details = get_scraper_details(
self.args.firefox, Settings.CHANNELS,
os.path.join(IrisCore.get_working_dir(), 'cache'), locale)
scraper = FactoryScraper(type, **scraper_details)
firefox_dmg = scraper.download()
install_folder = install(src=firefox_dmg,
dest=IrisCore.get_current_run_dir())
binary = get_binary(install_folder, 'Firefox')
channel = get_firefox_channel(binary)
latest_type, latest_scraper_details = get_latest_scraper_details(
channel)
latest_path = FactoryScraper(latest_type,
**latest_scraper_details).filename
self.latest_version = get_version_from_path(latest_path)
logger.info('Latest available version for %s channel is: %s' %
(channel, self.latest_version))
return binary
except errors.NotFoundError:
logger.critical(
'Specified build (%s) has not been found. Closing Iris ...'
% self.args.firefox)
self.finish(5)
print(bcolors.FAIL + "ERROR" + bcolors.ENDC)
print(e)
else:
print(bcolors.BOLD + bcolors.OKGREEN + "All libraries imports succeeded." + bcolors.ENDC)
print("")
print("_____________________________________________________________________________________")
print("")
print(bcolors.OKBLUE + "Testing Mozdownload" + bcolors.ENDC)
print("")
try:
from mozdownload import FactoryScraper
scraper = FactoryScraper('daily')
print(scraper.url)
print(scraper.filename)
except (IOError, OSError) as e:
print(bcolors.FAIL + "ERROR" + bcolors.ENDC)
print(e)
except:
print("Unexpected error:", sys.exc_info()[0])
raise
else:
print("")
print("Version: " + mozdownload.__version__)
print("")
print(bcolors.BOLD + bcolors.OKGREEN + "Mozdownload working correctly." + bcolors.ENDC)
