def get_test_packages_url(properties): """Return the URL of the test packages JSON file. In case of localized daily builds we can query the en-US build to get the URL, but for candidate builds we need the tinderbox build of the first parent changeset which was not checked-in by the release automation process (necessary until bug 1242035 is not fixed). """ overrides = { 'locale': 'en-US', 'extension': 'test_packages.json', 'build_type': 'tinderbox', 'retry_attempts': 0, } platform_map = { 'linux': {'build_platform': 'linux32'}, 'linux64': {'build_platform': 'linux64'}, 'mac': {'build_os': 'mac', 'build_architecture': 'x86_64'}, 'win32': {'build_os': 'win', 'build_architecture': 'x86'}, 'win64': {'build_os': 'win', 'build_architecture': 'x86_64'}, } revision = properties['revision'][:12] client = TreeherderClient(host='treeherder.mozilla.org', protocol='https') resultsets = client.get_resultsets(properties['branch'], tochange=revision, count=50) # Retrieve the option hashes to filter for opt builds option_hash = None for key, values in client.get_option_collection_hash().iteritems(): for value in values: if value['name'] == 'opt': option_hash = key break if option_hash: break # Set filters to speed-up querying jobs kwargs = { 'job_type_name': 'Build', 'exclusion_profile': False, 'option_collection_hash': option_hash, 'result': 'success', } kwargs.update(platform_map[properties['platform']]) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = client.get_jobs(properties['branch'], **kwargs) if len(jobs): revision = resultset['revision'] break overrides['revision'] = revision # For update tests we need the test package of the target build. That allows # us to add fallback code in case major parts of the ui are changing in Firefox. if properties.get('target_buildid'): overrides['build_id'] = properties['target_buildid'] # The test package json file has a prefix with bug 1239808 fixed. Older builds need # a fallback to a prefix-less filename. try: url = query_file_url(properties, property_overrides=overrides) except download_errors.NotFoundError: overrides.pop('extension') build_url = query_file_url(properties, property_overrides=overrides) url = '{}/test_packages.json'.format(build_url[:build_url.rfind('/')]) return url
def get_test_packages_url(self, properties): """Return the URL of the test packages JSON file. In case of localized daily builds we can query the en-US build to get the URL, but for candidate builds we need the tinderbox build of the first parent changeset which was not checked-in by the release automation process (necessary until bug 1242035 is not fixed). """ if properties.get('test_packages_url'): url = properties['test_packages_url'] else: overrides = { 'locale': 'en-US', 'extension': 'test_packages.json', } # Use Treeherder to query for the next revision which has Tinderbox builds # available. We can use this revision to retrieve the test-packages URL. if properties['tree'].startswith('release-'): platform_map = { 'linux': { 'build_platform': 'linux32' }, 'linux64': { 'build_platform': 'linux64' }, 'macosx': { 'build_os': 'mac', 'build_architecture': 'x86_64' }, 'macosx64': { 'build_os': 'mac', 'build_architecture': 'x86_64' }, 'win32': { 'build_os': 'win', 'build_architecture': 'x86' }, 'win64': { 'build_os': 'win', 'build_architecture': 'x86_64' }, } self.logger.info( 'Querying tinderbox revision for {} build...'.format( properties['tree'])) revision = properties['revision'][:12] client = TreeherderClient( server_url='https://treeherder.mozilla.org') resultsets = client.get_resultsets(properties['branch'], tochange=revision, count=50) # Retrieve the option hashes to filter for opt builds option_hash = None for key, values in client.get_option_collection_hash( ).iteritems(): for value in values: if value['name'] == 'opt': option_hash = key break if option_hash: break # Set filters to speed-up querying jobs kwargs = { 'job_type_name': 'Build', 'exclusion_profile': False, 'option_collection_hash': option_hash, 'result': 'success', } kwargs.update(platform_map[properties['platform']]) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = client.get_jobs(properties['branch'], **kwargs) if len(jobs): revision = resultset['revision'] break self.logger.info( 'Found revision for tinderbox build: {}'.format(revision)) overrides['build_type'] = 'tinderbox' overrides['revision'] = revision # For update tests we need the test package of the target build. That allows # us to add fallback code in case major parts of the ui are changing in Firefox. if properties.get('target_buildid'): overrides['build_id'] = properties['target_buildid'] # The test package json file has a prefix with bug 1239808 fixed. Older builds need # a fallback to a prefix-less filename. try: self.logger.info('Querying test packages URL...') url = self.query_file_url(properties, property_overrides=overrides) except download_errors.NotFoundError: self.logger.info( 'URL not found. Querying not-prefixed test packages URL...' ) extension = overrides.pop('extension') build_url = self.query_file_url(properties, property_overrides=overrides) url = '{}/{}'.format(build_url[:build_url.rfind('/')], extension) r = requests.head(url) if r.status_code != 200: url = None self.logger.info('Found test package URL at: {}'.format(url)) return url
class Treeherder(object): """Wrapper class for TreeherderClient to ease the use of its API.""" def __init__(self, application, branch, platform, server_url=TREEHERDER_URL): """Create a new instance of the Treeherder class. :param application: The name of the application to download. :param branch: Name of the branch. :param platform: Platform of the application. :param server_url: The URL of the Treeherder instance to access. """ self.logger = logging.getLogger(__name__) self.client = TreeherderClient(server_url=server_url) self.application = application self.branch = branch self.platform = platform def get_treeherder_platform(self, platform): """Return the internal Treeherder platform identifier. :param platform: Platform of the application. """ try: return PLATFORM_MAP[platform] except KeyError: raise NotSupportedError( 'Platform "{}" is not supported.'.format(platform)) def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=False): """Retrieve build folders for a given revision with the help of Treeherder. :param revision: Revision of the build to download. :param job_type_name: Name of the job to look for. For builds it should be 'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`. :param debug_build: Download a debug build. """ builds = set() try: self.logger.info( 'Querying {url} for list of builds for revision: {revision}'. format(url=self.client.server_url, revision=revision)) # Retrieve the option hash to filter for type of build (opt, and debug for now) option_hash = None for key, values in self.client.get_option_collection_hash( ).iteritems(): for value in values: if value['name'] == ('debug' if debug_build else 'opt'): option_hash = key break if option_hash: break resultsets = self.client.get_resultsets(self.branch, revision=revision) # Set filters to speed-up querying jobs kwargs = { 'option_collection_hash': option_hash, 'job_type_name': job_type_name, 'exclusion_profile': False, } kwargs.update(self.get_treeherder_platform(self.platform)) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = self.client.get_jobs(self.branch, **kwargs) for job in jobs: log_urls = self.client.get_job_log_url(self.branch, job_id=job['id']) for log_url in log_urls: if self.application in log_url['url']: self.logger.debug('Found build folder: {}'.format( log_url['url'])) builds.update([log_url['url']]) except Exception: self.logger.exception( 'Failure occurred when querying Treeherder for builds') return list(builds)
class Treeherder(object): """Wrapper class for TreeherderClient to ease the use of its API.""" def __init__(self, application, branch, platform, server_url=TREEHERDER_URL): """Create a new instance of the Treeherder class. :param application: The name of the application to download. :param branch: Name of the branch. :param platform: Platform of the application. :param server_url: The URL of the Treeherder instance to access. """ self.logger = logging.getLogger(__name__) self.client = TreeherderClient(server_url=server_url) self.application = application self.branch = branch self.platform = platform def get_treeherder_platform(self, platform): """Return the internal Treeherder platform identifier. :param platform: Platform of the application. """ try: return PLATFORM_MAP[platform] except KeyError: raise NotSupportedError('Platform "{}" is not supported.'.format(platform)) def query_builds_by_revision(self, revision, job_type_name='Build', debug_build=False): """Retrieve build folders for a given revision with the help of Treeherder. :param revision: Revision of the build to download. :param job_type_name: Name of the job to look for. For builds it should be 'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`. :param debug_build: Download a debug build. """ builds = set() try: self.logger.info('Querying {url} for list of builds for revision: {revision}'.format( url=self.client.server_url, revision=revision)) # Retrieve the option hash to filter for type of build (opt, and debug for now) option_hash = None for key, values in self.client.get_option_collection_hash().iteritems(): for value in values: if value['name'] == ('debug' if debug_build else 'opt'): option_hash = key break if option_hash: break resultsets = self.client.get_pushes(self.branch, revision=revision) # Set filters to speed-up querying jobs kwargs = { 'option_collection_hash': option_hash, 'job_type_name': job_type_name, 'exclusion_profile': False, } kwargs.update(self.get_treeherder_platform(self.platform)) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = self.client.get_jobs(self.branch, **kwargs) for job in jobs: log_urls = self.client.get_job_log_url(self.branch, job_id=job['id']) for log_url in log_urls: if self.application in log_url['url']: self.logger.debug('Found build folder: {}'.format(log_url['url'])) builds.update([log_url['url']]) except Exception: self.logger.exception('Failure occurred when querying Treeherder for builds') return list(builds)
def get_test_packages_url(self, properties): """Return the URL of the test packages JSON file. In case of localized daily builds we can query the en-US build to get the URL, but for candidate builds we need the tinderbox build of the first parent changeset which was not checked-in by the release automation process (necessary until bug 1242035 is not fixed). """ if properties.get('test_packages_url'): url = properties['test_packages_url'] else: overrides = { 'locale': 'en-US', 'extension': 'test_packages.json', } # Use Treeherder to query for the next revision which has Tinderbox builds # available. We can use this revision to retrieve the test-packages URL. if properties['tree'].startswith('release-'): platform_map = { 'linux': {'build_platform': 'linux32'}, 'linux64': {'build_platform': 'linux64'}, 'macosx': {'build_os': 'mac', 'build_architecture': 'x86_64'}, 'macosx64': {'build_os': 'mac', 'build_architecture': 'x86_64'}, 'win32': {'build_os': 'win', 'build_architecture': 'x86'}, 'win64': {'build_os': 'win', 'build_architecture': 'x86_64'}, } self.logger.info('Querying tinderbox revision for {} build...'.format( properties['tree'])) revision = properties['revision'][:12] client = TreeherderClient(server_url='https://treeherder.mozilla.org') resultsets = client.get_resultsets(properties['branch'], tochange=revision, count=50) # Retrieve the option hashes to filter for opt builds option_hash = None for key, values in client.get_option_collection_hash().iteritems(): for value in values: if value['name'] == 'opt': option_hash = key break if option_hash: break # Set filters to speed-up querying jobs kwargs = { 'job_type_name': 'Build', 'exclusion_profile': False, 'option_collection_hash': option_hash, 'result': 'success', } kwargs.update(platform_map[properties['platform']]) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = client.get_jobs(properties['branch'], **kwargs) if len(jobs): revision = resultset['revision'] break self.logger.info('Found revision for tinderbox build: {}'.format(revision)) overrides['build_type'] = 'tinderbox' overrides['revision'] = revision # For update tests we need the test package of the target build. That allows # us to add fallback code in case major parts of the ui are changing in Firefox. if properties.get('target_buildid'): overrides['build_id'] = properties['target_buildid'] # The test package json file has a prefix with bug 1239808 fixed. Older builds need # a fallback to a prefix-less filename. try: self.logger.info('Querying test packages URL...') url = self.query_file_url(properties, property_overrides=overrides) except download_errors.NotFoundError: self.logger.info('URL not found. Querying not-prefixed test packages URL...') extension = overrides.pop('extension') build_url = self.query_file_url(properties, property_overrides=overrides) url = '{}/{}'.format(build_url[:build_url.rfind('/')], extension) r = requests.head(url) if r.status_code != 200: url = None self.logger.info('Found test package URL at: {}'.format(url)) return url
def get_test_packages_url(properties): """Return the URL of the test packages JSON file. In case of localized daily builds we can query the en-US build to get the URL, but for candidate builds we need the tinderbox build of the first parent changeset which was not checked-in by the release automation process (necessary until bug 1242035 is not fixed). """ overrides = { 'locale': 'en-US', 'extension': 'test_packages.json', 'build_type': 'tinderbox', 'retry_attempts': 0, } platform_map = { 'linux': { 'build_platform': 'linux32' }, 'linux64': { 'build_platform': 'linux64' }, 'mac': { 'build_os': 'mac', 'build_architecture': 'x86_64' }, 'win32': { 'build_os': 'win', 'build_architecture': 'x86' }, 'win64': { 'build_os': 'win', 'build_architecture': 'x86_64' }, } revision = properties['revision'][:12] client = TreeherderClient(host='treeherder.mozilla.org', protocol='https') resultsets = client.get_resultsets(properties['branch'], tochange=revision, count=50) # Retrieve the option hashes to filter for opt builds option_hash = None for key, values in client.get_option_collection_hash().iteritems(): for value in values: if value['name'] == 'opt': option_hash = key break if option_hash: break # Set filters to speed-up querying jobs kwargs = { 'job_type_name': 'Build', 'exclusion_profile': False, 'option_collection_hash': option_hash, 'result': 'success', } kwargs.update(platform_map[properties['platform']]) for resultset in resultsets: kwargs.update({'result_set_id': resultset['id']}) jobs = client.get_jobs(properties['branch'], **kwargs) if len(jobs): revision = resultset['revision'] break overrides['revision'] = revision # For update tests we need the test package of the target build. That allows # us to add fallback code in case major parts of the ui are changing in Firefox. if properties.get('target_buildid'): overrides['build_id'] = properties['target_buildid'] # The test package json file has a prefix with bug 1239808 fixed. Older builds need # a fallback to a prefix-less filename. try: url = query_file_url(properties, property_overrides=overrides) except download_errors.NotFoundError: overrides.pop('extension') build_url = query_file_url(properties, property_overrides=overrides) url = '{}/test_packages.json'.format(build_url[:build_url.rfind('/')]) return url
class Treeherder(object): """Wrapper class for TreeherderClient to ease the use of its API.""" def __init__(self, application, branch, platform, host=TREEHERDER_HOST, protocol="https"): """Create a new instance of the Treeherder class. :param application: The name of the application to download. :param branch: Name of the branch. :param platform: Platform of the application. :param host: The Treeherder host to make use of. :param protocol: The protocol for the Treeherder host. """ self.logger = logging.getLogger(__name__) self.client = TreeherderClient(host=host, protocol=protocol) self.application = application self.branch = branch self.platform = platform def get_treeherder_platform(self, platform): """Return the internal Treeherder platform identifier. :param platform: Platform of the application. """ try: return PLATFORM_MAP[platform] except KeyError: raise NotSupportedError('Platform "{}" is not supported.'.format(platform)) def query_builds_by_revision(self, revision, job_type_name="Build", debug_build=False): """Retrieve build folders for a given revision with the help of Treeherder. :param revision: Revision of the build to download. :param job_type_name: Name of the job to look for. For builds it should be 'Build', 'Nightly', and 'L10n Nightly'. Defaults to `Build`. :param debug_build: Download a debug build. """ builds = set() try: self.logger.info( "Querying {host} for list of builds for revision: {revision}".format( host=self.client.host, revision=revision ) ) # Retrieve the option hash to filter for type of build (opt, and debug for now) option_hash = None for key, values in self.client.get_option_collection_hash().iteritems(): for value in values: if value["name"] == ("debug" if debug_build else "opt"): option_hash = key break if option_hash: break resultsets = self.client.get_resultsets(self.branch, revision=revision) # Set filters to speed-up querying jobs kwargs = {"option_collection_hash": option_hash, "job_type_name": job_type_name, "exclusion_profile": False} kwargs.update(self.get_treeherder_platform(self.platform)) for resultset in resultsets: kwargs.update({"result_set_id": resultset["id"]}) jobs = self.client.get_jobs(self.branch, **kwargs) for job in jobs: log_urls = self.client.get_job_log_url(self.branch, job_id=job["id"]) for log_url in log_urls: if self.application in log_url["url"]: self.logger.debug("Found build folder: {}".format(log_url["url"])) builds.update([log_url["url"]]) except Exception: self.logger.exception("Failure occurred when querying Treeherder for builds") return list(builds)