def get_test_packages_url(properties):
"""Return the URL of the test packages JSON file.
In case of localized daily builds we can query the en-US build to get
the URL, but for candidate builds we need the tinderbox build
of the first parent changeset which was not checked-in by the release
automation process (necessary until bug 1242035 is not fixed).
"""
overrides = {
'locale': 'en-US',
'extension': 'test_packages.json',
'build_type': 'tinderbox',
'retry_attempts': 0,
}
platform_map = {
'linux': {'build_platform': 'linux32'},
'linux64': {'build_platform': 'linux64'},
'mac': {'build_os': 'mac', 'build_architecture': 'x86_64'},
'win32': {'build_os': 'win', 'build_architecture': 'x86'},
'win64': {'build_os': 'win', 'build_architecture': 'x86_64'},
}
revision = properties['revision'][:12]
client = TreeherderClient(host='treeherder.mozilla.org', protocol='https')
resultsets = client.get_resultsets(properties['branch'],
tochange=revision,
count=50)
# Retrieve the option hashes to filter for opt builds
option_hash = None
for key, values in client.get_option_collection_hash().iteritems():
for value in values:
if value['name'] == 'opt':
option_hash = key
break
if option_hash:
break
# Set filters to speed-up querying jobs
kwargs = {
'job_type_name': 'Build',
'exclusion_profile': False,
'option_collection_hash': option_hash,
'result': 'success',
}
kwargs.update(platform_map[properties['platform']])
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = client.get_jobs(properties['branch'], **kwargs)
if len(jobs):
revision = resultset['revision']
break
overrides['revision'] = revision
# For update tests we need the test package of the target build. That allows
# us to add fallback code in case major parts of the ui are changing in Firefox.
if properties.get('target_buildid'):
overrides['build_id'] = properties['target_buildid']
# The test package json file has a prefix with bug 1239808 fixed. Older builds need
# a fallback to a prefix-less filename.
try:
url = query_file_url(properties, property_overrides=overrides)
except download_errors.NotFoundError:
overrides.pop('extension')
build_url = query_file_url(properties, property_overrides=overrides)
url = '{}/test_packages.json'.format(build_url[:build_url.rfind('/')])
return url
def get_test_packages_url(self, properties):
"""Return the URL of the test packages JSON file.
In case of localized daily builds we can query the en-US build to get
the URL, but for candidate builds we need the tinderbox build
of the first parent changeset which was not checked-in by the release
automation process (necessary until bug 1242035 is not fixed).
"""
if properties.get('test_packages_url'):
url = properties['test_packages_url']
else:
overrides = {
'locale': 'en-US',
'extension': 'test_packages.json',
}
# Use Treeherder to query for the next revision which has Tinderbox builds
# available. We can use this revision to retrieve the test-packages URL.
if properties['tree'].startswith('release-'):
platform_map = {
'linux': {
'build_platform': 'linux32'
},
'linux64': {
'build_platform': 'linux64'
},
'macosx': {
'build_os': 'mac',
'build_architecture': 'x86_64'
},
'macosx64': {
'build_os': 'mac',
'build_architecture': 'x86_64'
},
'win32': {
'build_os': 'win',
'build_architecture': 'x86'
},
'win64': {
'build_os': 'win',
'build_architecture': 'x86_64'
},
}
self.logger.info(
'Querying tinderbox revision for {} build...'.format(
properties['tree']))
revision = properties['revision'][:12]
client = TreeherderClient(
server_url='https://treeherder.mozilla.org')
resultsets = client.get_resultsets(properties['branch'],
tochange=revision,
count=50)
# Retrieve the option hashes to filter for opt builds
option_hash = None
for key, values in client.get_option_collection_hash(
).iteritems():
for value in values:
if value['name'] == 'opt':
option_hash = key
break
if option_hash:
break
# Set filters to speed-up querying jobs
kwargs = {
'job_type_name': 'Build',
'exclusion_profile': False,
'option_collection_hash': option_hash,
'result': 'success',
}
kwargs.update(platform_map[properties['platform']])
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = client.get_jobs(properties['branch'], **kwargs)
if len(jobs):
revision = resultset['revision']
break
self.logger.info(
'Found revision for tinderbox build: {}'.format(revision))
overrides['build_type'] = 'tinderbox'
overrides['revision'] = revision
# For update tests we need the test package of the target build. That allows
# us to add fallback code in case major parts of the ui are changing in Firefox.
if properties.get('target_buildid'):
overrides['build_id'] = properties['target_buildid']
# The test package json file has a prefix with bug 1239808 fixed. Older builds need
# a fallback to a prefix-less filename.
try:
self.logger.info('Querying test packages URL...')
url = self.query_file_url(properties,
property_overrides=overrides)
except download_errors.NotFoundError:
self.logger.info(
'URL not found. Querying not-prefixed test packages URL...'
)
extension = overrides.pop('extension')
build_url = self.query_file_url(properties,
property_overrides=overrides)
url = '{}/{}'.format(build_url[:build_url.rfind('/')],
extension)
r = requests.head(url)
if r.status_code != 200:
url = None
self.logger.info('Found test package URL at: {}'.format(url))
return url
class Treeherder(object):
"""Wrapper class for TreeherderClient to ease the use of its API."""
def __init__(self,
application,
branch,
platform,
server_url=TREEHERDER_URL):
"""Create a new instance of the Treeherder class.
:param application: The name of the application to download.
:param branch: Name of the branch.
:param platform: Platform of the application.
:param server_url: The URL of the Treeherder instance to access.
"""
self.logger = logging.getLogger(__name__)
self.client = TreeherderClient(server_url=server_url)
self.application = application
self.branch = branch
self.platform = platform
def get_treeherder_platform(self, platform):
"""Return the internal Treeherder platform identifier.
:param platform: Platform of the application.
"""
try:
return PLATFORM_MAP[platform]
except KeyError:
raise NotSupportedError(
'Platform "{}" is not supported.'.format(platform))
def query_builds_by_revision(self,
revision,
job_type_name='Build',
debug_build=False):
"""Retrieve build folders for a given revision with the help of Treeherder.
:param revision: Revision of the build to download.
:param job_type_name: Name of the job to look for. For builds it should be
'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`.
:param debug_build: Download a debug build.
"""
builds = set()
try:
self.logger.info(
'Querying {url} for list of builds for revision: {revision}'.
format(url=self.client.server_url, revision=revision))
# Retrieve the option hash to filter for type of build (opt, and debug for now)
option_hash = None
for key, values in self.client.get_option_collection_hash(
).iteritems():
for value in values:
if value['name'] == ('debug' if debug_build else 'opt'):
option_hash = key
break
if option_hash:
break
resultsets = self.client.get_resultsets(self.branch,
revision=revision)
# Set filters to speed-up querying jobs
kwargs = {
'option_collection_hash': option_hash,
'job_type_name': job_type_name,
'exclusion_profile': False,
}
kwargs.update(self.get_treeherder_platform(self.platform))
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = self.client.get_jobs(self.branch, **kwargs)
for job in jobs:
log_urls = self.client.get_job_log_url(self.branch,
job_id=job['id'])
for log_url in log_urls:
if self.application in log_url['url']:
self.logger.debug('Found build folder: {}'.format(
log_url['url']))
builds.update([log_url['url']])
except Exception:
self.logger.exception(
'Failure occurred when querying Treeherder for builds')
return list(builds)
class Treeherder(object):
"""Wrapper class for TreeherderClient to ease the use of its API."""
def __init__(self, application, branch, platform, server_url=TREEHERDER_URL):
"""Create a new instance of the Treeherder class.
:param application: The name of the application to download.
:param branch: Name of the branch.
:param platform: Platform of the application.
:param server_url: The URL of the Treeherder instance to access.
"""
self.logger = logging.getLogger(__name__)
self.client = TreeherderClient(server_url=server_url)
self.application = application
self.branch = branch
self.platform = platform
def get_treeherder_platform(self, platform):
"""Return the internal Treeherder platform identifier.
:param platform: Platform of the application.
"""
try:
return PLATFORM_MAP[platform]
except KeyError:
raise NotSupportedError('Platform "{}" is not supported.'.format(platform))
def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=False):
"""Retrieve build folders for a given revision with the help of Treeherder.
:param revision: Revision of the build to download.
:param job_type_name: Name of the job to look for. For builds it should be
'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`.
:param debug_build: Download a debug build.
"""
builds = set()
try:
self.logger.info('Querying {url} for list of builds for revision: {revision}'.format(
url=self.client.server_url, revision=revision))
# Retrieve the option hash to filter for type of build (opt, and debug for now)
option_hash = None
for key, values in self.client.get_option_collection_hash().iteritems():
for value in values:
if value['name'] == ('debug' if debug_build else 'opt'):
option_hash = key
break
if option_hash:
break
resultsets = self.client.get_pushes(self.branch, revision=revision)
# Set filters to speed-up querying jobs
kwargs = {
'option_collection_hash': option_hash,
'job_type_name': job_type_name,
'exclusion_profile': False,
}
kwargs.update(self.get_treeherder_platform(self.platform))
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = self.client.get_jobs(self.branch, **kwargs)
for job in jobs:
log_urls = self.client.get_job_log_url(self.branch, job_id=job['id'])
for log_url in log_urls:
if self.application in log_url['url']:
self.logger.debug('Found build folder: {}'.format(log_url['url']))
builds.update([log_url['url']])
except Exception:
self.logger.exception('Failure occurred when querying Treeherder for builds')
return list(builds)
def get_test_packages_url(self, properties):
"""Return the URL of the test packages JSON file.
In case of localized daily builds we can query the en-US build to get
the URL, but for candidate builds we need the tinderbox build
of the first parent changeset which was not checked-in by the release
automation process (necessary until bug 1242035 is not fixed).
"""
if properties.get('test_packages_url'):
url = properties['test_packages_url']
else:
overrides = {
'locale': 'en-US',
'extension': 'test_packages.json',
}
# Use Treeherder to query for the next revision which has Tinderbox builds
# available. We can use this revision to retrieve the test-packages URL.
if properties['tree'].startswith('release-'):
platform_map = {
'linux': {'build_platform': 'linux32'},
'linux64': {'build_platform': 'linux64'},
'macosx': {'build_os': 'mac', 'build_architecture': 'x86_64'},
'macosx64': {'build_os': 'mac', 'build_architecture': 'x86_64'},
'win32': {'build_os': 'win', 'build_architecture': 'x86'},
'win64': {'build_os': 'win', 'build_architecture': 'x86_64'},
}
self.logger.info('Querying tinderbox revision for {} build...'.format(
properties['tree']))
revision = properties['revision'][:12]
client = TreeherderClient(server_url='https://treeherder.mozilla.org')
resultsets = client.get_resultsets(properties['branch'],
tochange=revision,
count=50)
# Retrieve the option hashes to filter for opt builds
option_hash = None
for key, values in client.get_option_collection_hash().iteritems():
for value in values:
if value['name'] == 'opt':
option_hash = key
break
if option_hash:
break
# Set filters to speed-up querying jobs
kwargs = {
'job_type_name': 'Build',
'exclusion_profile': False,
'option_collection_hash': option_hash,
'result': 'success',
}
kwargs.update(platform_map[properties['platform']])
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = client.get_jobs(properties['branch'], **kwargs)
if len(jobs):
revision = resultset['revision']
break
self.logger.info('Found revision for tinderbox build: {}'.format(revision))
overrides['build_type'] = 'tinderbox'
overrides['revision'] = revision
# For update tests we need the test package of the target build. That allows
# us to add fallback code in case major parts of the ui are changing in Firefox.
if properties.get('target_buildid'):
overrides['build_id'] = properties['target_buildid']
# The test package json file has a prefix with bug 1239808 fixed. Older builds need
# a fallback to a prefix-less filename.
try:
self.logger.info('Querying test packages URL...')
url = self.query_file_url(properties, property_overrides=overrides)
except download_errors.NotFoundError:
self.logger.info('URL not found. Querying not-prefixed test packages URL...')
extension = overrides.pop('extension')
build_url = self.query_file_url(properties, property_overrides=overrides)
url = '{}/{}'.format(build_url[:build_url.rfind('/')], extension)
r = requests.head(url)
if r.status_code != 200:
url = None
self.logger.info('Found test package URL at: {}'.format(url))
return url
def get_test_packages_url(properties):
"""Return the URL of the test packages JSON file.
In case of localized daily builds we can query the en-US build to get
the URL, but for candidate builds we need the tinderbox build
of the first parent changeset which was not checked-in by the release
automation process (necessary until bug 1242035 is not fixed).
"""
overrides = {
'locale': 'en-US',
'extension': 'test_packages.json',
'build_type': 'tinderbox',
'retry_attempts': 0,
}
platform_map = {
'linux': {
'build_platform': 'linux32'
},
'linux64': {
'build_platform': 'linux64'
},
'mac': {
'build_os': 'mac',
'build_architecture': 'x86_64'
},
'win32': {
'build_os': 'win',
'build_architecture': 'x86'
},
'win64': {
'build_os': 'win',
'build_architecture': 'x86_64'
},
}
revision = properties['revision'][:12]
client = TreeherderClient(host='treeherder.mozilla.org', protocol='https')
resultsets = client.get_resultsets(properties['branch'],
tochange=revision,
count=50)
# Retrieve the option hashes to filter for opt builds
option_hash = None
for key, values in client.get_option_collection_hash().iteritems():
for value in values:
if value['name'] == 'opt':
option_hash = key
break
if option_hash:
break
# Set filters to speed-up querying jobs
kwargs = {
'job_type_name': 'Build',
'exclusion_profile': False,
'option_collection_hash': option_hash,
'result': 'success',
}
kwargs.update(platform_map[properties['platform']])
for resultset in resultsets:
kwargs.update({'result_set_id': resultset['id']})
jobs = client.get_jobs(properties['branch'], **kwargs)
if len(jobs):
revision = resultset['revision']
break
overrides['revision'] = revision
# For update tests we need the test package of the target build. That allows
# us to add fallback code in case major parts of the ui are changing in Firefox.
if properties.get('target_buildid'):
overrides['build_id'] = properties['target_buildid']
# The test package json file has a prefix with bug 1239808 fixed. Older builds need
# a fallback to a prefix-less filename.
try:
url = query_file_url(properties, property_overrides=overrides)
except download_errors.NotFoundError:
overrides.pop('extension')
build_url = query_file_url(properties, property_overrides=overrides)
url = '{}/test_packages.json'.format(build_url[:build_url.rfind('/')])
return url
class Treeherder(object):
"""Wrapper class for TreeherderClient to ease the use of its API."""
def __init__(self, application, branch, platform, host=TREEHERDER_HOST, protocol="https"):
"""Create a new instance of the Treeherder class.
:param application: The name of the application to download.
:param branch: Name of the branch.
:param platform: Platform of the application.
:param host: The Treeherder host to make use of.
:param protocol: The protocol for the Treeherder host.
"""
self.logger = logging.getLogger(__name__)
self.client = TreeherderClient(host=host, protocol=protocol)
self.application = application
self.branch = branch
self.platform = platform
def get_treeherder_platform(self, platform):
"""Return the internal Treeherder platform identifier.
:param platform: Platform of the application.
"""
try:
return PLATFORM_MAP[platform]
except KeyError:
raise NotSupportedError('Platform "{}" is not supported.'.format(platform))
def query_builds_by_revision(self, revision, job_type_name="Build", debug_build=False):
"""Retrieve build folders for a given revision with the help of Treeherder.
:param revision: Revision of the build to download.
:param job_type_name: Name of the job to look for. For builds it should be
'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`.
:param debug_build: Download a debug build.
"""
builds = set()
try:
self.logger.info(
"Querying {host} for list of builds for revision: {revision}".format(
host=self.client.host, revision=revision
)
)
# Retrieve the option hash to filter for type of build (opt, and debug for now)
option_hash = None
for key, values in self.client.get_option_collection_hash().iteritems():
for value in values:
if value["name"] == ("debug" if debug_build else "opt"):
option_hash = key
break
if option_hash:
break
resultsets = self.client.get_resultsets(self.branch, revision=revision)
# Set filters to speed-up querying jobs
kwargs = {"option_collection_hash": option_hash, "job_type_name": job_type_name, "exclusion_profile": False}
kwargs.update(self.get_treeherder_platform(self.platform))
for resultset in resultsets:
kwargs.update({"result_set_id": resultset["id"]})
jobs = self.client.get_jobs(self.branch, **kwargs)
for job in jobs:
log_urls = self.client.get_job_log_url(self.branch, job_id=job["id"])
for log_url in log_urls:
if self.application in log_url["url"]:
self.logger.debug("Found build folder: {}".format(log_url["url"]))
builds.update([log_url["url"]])
except Exception:
self.logger.exception("Failure occurred when querying Treeherder for builds")
return list(builds)