async def product_details(product, version): if get_version_channel(product, version) is Channel.NIGHTLY: versions = await ongoing_versions(product) status = build_version_id( versions["nightly"]) >= build_version_id(version) message = "Last nightly version is {}".format(versions["nightly"]) url = "https://product-details.mozilla.org/1.0/{}_versions.json".format( product) return build_task_response(status, url, message) async with get_session() as session: url = 'https://product-details.mozilla.org/1.0/firefox.json' async with session.get(url) as resp: if resp.status != 200: msg = 'Product Details info not available (HTTP {})'.format( resp.status) raise TaskError(msg, url=url) body = await resp.json() status = 'firefox-{}'.format(version) in body['releases'] exists_message = "We found product-details information about version {}" missing_message = "We did not find product-details information about version {}" return build_task_response(status, url, exists_message.format(version), missing_message.format(version))
async def bouncer(product, version): """Make sure bouncer redirects to the expected version (or a later one).""" channel = get_version_channel(product, version) channel_value = channel.value if product == 'thunderbird' and channel == Channel.NIGHTLY: # NIGHTLY is a valid channel for Thunderbird, but it does not use the bouncer return build_task_response(True, "", "No nightly bouncer for Thunderbird.") if product == 'devedition': channel_value = "DEVEDITION" product_channel = 'firefox-{}'.format(channel_value.lower()) else: # product is 'firefox' or 'thunderbird' if channel == Channel.RELEASE: product_channel = product else: product_channel = '{}-{}'.format(product, channel_value.lower()) url = 'https://download.mozilla.org?product={}-latest-ssl&os=linux64&lang=en-US'.format( product_channel) async with get_session() as session: async with session.get(url, allow_redirects=False) as resp: if resp.status == 302: url = resp.headers['Location'] else: msg = 'Bouncer is down ({}).'.format(resp.status) raise TaskError(msg, url=url) filename = os.path.basename(url) last_release = get_version_from_filename(filename) status = build_version_id(last_release) >= build_version_id(version) message = "Bouncer for {} redirects to version {}".format(channel_value, last_release) return build_task_response(status, url, message)
async def bouncer(product, version): """Fetch bedrock download page to grab the bouncer download link and then make sure it redirects to the expected version.""" channel = get_version_channel(product, version) channel_value = channel.value if channel is Channel.ESR: bedrock_url = "https://www.mozilla.org/en-US/{}/organizations/all/".format( product) elif channel is Channel.RELEASE: bedrock_url = 'https://www.mozilla.org/en-US/{}/all/'.format(product) else: bedrock_url = 'https://www.mozilla.org/fr/{}/channel/desktop/'.format( product) if product == 'devedition': bedrock_url = 'https://www.mozilla.org/en-US/firefox/developer/' channel_value = "DEVEDITION" async with get_session() as session: async with session.get(bedrock_url) as resp: if resp.status != 200: msg = 'Download page not available ({})'.format(resp.status) raise TaskError(msg, url=bedrock_url) body = await resp.text() d = pq(body) if product == 'devedition': link_path = "#intro-download > .download-list > .os_linux64 > a" elif channel is Channel.NIGHTLY: link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a" elif channel in (Channel.BETA, Channel.AURORA): link_path = "#desktop-beta-download > .download-list > .os_linux64 > a" else: # channel in (Channel.RELEASE, Channel.ESR): link_path = "#fr > .linux64 > a" url = d(link_path).attr('href') if url is not None: async with session.get(url, allow_redirects=False) as resp: if resp.status == 302: url = resp.headers['Location'] else: msg = 'Bouncer is down ({}).'.format(resp.status) raise TaskError(msg, url=url) else: msg = 'No links found.'.format(resp.status) raise TaskError(msg, url=bedrock_url) filename = os.path.basename(url) last_release = get_version_from_filename(filename) status = build_version_id(last_release) >= build_version_id( version) message = "Bouncer for {} redirects to version {}".format( channel_value, last_release) return build_task_response(status, url, message)
async def download_links(product, version): channel = get_version_channel(product, version) url = get_downloads_url(product, channel) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = 'Download page not available ({})'.format(resp.status) raise TaskError(msg) body = await resp.text() d = pq(body) if product == 'thunderbird': if channel is Channel.NIGHTLY: link_path = ".download-link.btn-nightly" url = d(link_path).attr('href') filename = os.path.basename(url) last_release = get_version_from_filename(filename) else: last_release = d("#all-downloads").attr( 'data-thunderbird-version') else: if channel in (Channel.NIGHTLY, Channel.BETA, Channel.AURORA): if product == 'devedition': link_path = "#intro-download > .download-list > .os_linux64 > a" elif channel is Channel.NIGHTLY: link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a" else: # channel is Channel.BETA: link_path = "#desktop-beta-download > .download-list > .os_linux64 > a" url = d(link_path).attr('href') async with session.get(url, allow_redirects=False) as resp: url = resp.headers['Location'] filename = os.path.basename(url) last_release = get_version_from_filename(filename) elif channel is Channel.ESR: version = re.sub('esr$', '', version) last_release = d("html").attr('data-esr-versions') else: # Does the content contains the version number? last_release = d("html").attr('data-latest-firefox') status = build_version_id(last_release) >= build_version_id( version) message = ( "The download links for release have been published for version {}" .format(last_release)) return build_task_response(status, url, message)
async def security_advisories(product, version): channel = get_version_channel(product, version) url = 'https://www.mozilla.org/en-US/security/known-vulnerabilities/{}/'.format( product) # Security advisories are always present for BETA and NIGHTLY # because we don't publish any. if channel in (Channel.BETA, Channel.NIGHTLY): return build_task_response( status=Status.MISSING, link=url, message="Security advisories are never published for {} releases". format(channel.value.lower())) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = 'Security advisories page not available ({})'.format( resp.status) raise TaskError(msg) # Does the content contains the version number? body = await resp.text() d = pq(body) if product in ['firefox', 'devedition']: security_product = 'firefox' if channel is Channel.ESR: version = re.sub('esr$', '', version) last_release = d("html").attr('data-esr-versions') else: last_release = d("html").attr('data-latest-firefox') elif product == 'thunderbird': security_product = product last_release = d("html").attr('data-esr-versions') status = build_version_id(last_release) >= build_version_id( version) message = ("Security advisories for release were " "updated up to version {}".format(last_release)) version_title = "#{}{}".format(security_product, version.split('.')[0]) if status and not d(version_title): status = Status.INCOMPLETE message += " but nothing was published for {} yet.".format( version_title) return build_task_response(status, url, message)
async def get_releases(product): RELEASE_CHANNEL = { 'devedition': 'aurora', 'firefox': 'release', } query = { "aggs": { "by_version": { "terms": { "field": "target.version", "size": 1000, "order": { "_term": "desc" } } } }, "query": { "bool": { "filter": [{ "term": { "source.product": product } }, { "term": { "target.channel": RELEASE_CHANNEL[product] } }] } }, "size": 0 } async with get_session() as session: url = '{}/buckets/build-hub/collections/releases/search' url = url.format(BUILDHUB_SERVER) async with session.post(url, data=json.dumps(query)) as response: if response.status != 200: message = "Buildhub is not available ({})".format( response.status) url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format( product) raise TaskError(message, url=url) data = await response.json() versions = sorted([ r['key'] for r in data['aggregations']['by_version']['buckets'] if strip_candidate_info(r['key']) == r['key'] ], key=lambda version: build_version_id(version)) if not versions: message = "Couldn't find any version matching." url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format( product) raise TaskError(message, url=url) return versions
async def balrog_rules(product, version): channel = get_version_channel(product, version) if channel is Channel.NIGHTLY: # In that case the rule doesn't change, so we grab the build IDs. # There are case were Nightly is deactivated, in that case # the mapping is not nightly-latest anymore url = 'https://aus-api.mozilla.org/api/v1/rules/firefox-nightly' async with get_session() as session: async with session.get(url) as resp: rule = await resp.json() status = rule[ 'mapping'] == 'Firefox-mozilla-central-nightly-latest' build_ids, appVersions = await get_release_info(rule['mapping'] ) last_build_id = max(build_ids.values()) date = last_build_id[:8] old_build_id = [ bid for bid in build_ids.values() if not bid.startswith(date) ] if rule['mapping'] != 'Firefox-mozilla-central-nightly-latest': status = Status.MISSING message = ( 'Balrog rule is configured for {} ({}) instead of ' '"Firefox-mozilla-central-nightly-latest"') message = message.format( rule['mapping'], ', '.join(sorted(set(build_ids.values())))) elif old_build_id: platforms = [ k for k, v in build_ids.items() if v in old_build_id ] status = Status.INCOMPLETE message = ( "Balrog rule is configured for {} ({}) platform {} with build ID {}" " seem outdated.") message = message.format( rule['mapping'], ', '.join(sorted(set(build_ids.values()))), ', '.join(platforms), ', '.join(sorted(set(old_build_id)))) else: status = Status.EXISTS message = ( 'Balrog rule is configured for the latest Nightly {} build ({}) ' 'with an update rate of {}%') message = message.format( ', '.join(sorted(appVersions)), ', '.join(sorted(set(build_ids.values()))), rule['backgroundRate']) return build_task_response(status, url, message) elif channel in (Channel.BETA, Channel.AURORA): rule_name = 'devedition' if product == 'devedition' else 'firefox-beta' url = 'https://aus-api.mozilla.org/api/v1/rules/{}'.format(rule_name) elif channel is Channel.ESR: version = re.sub('esr$', '', version) url = 'https://aus-api.mozilla.org/api/v1/rules/esr{}'.format( version.split('.')[0]) else: url = 'https://aus-api.mozilla.org/api/v1/rules/firefox-release' async with get_session() as session: async with session.get(url) as resp: rule = await resp.json() build_ids, appVersions = await get_release_info(rule['mapping']) status = build_version_id( appVersions.pop()) >= build_version_id(version) exists_message = ( 'Balrog rule has been updated for {} ({}) with an update rate of {}%' ).format(rule['mapping'], ', '.join(sorted(set(build_ids.values()))), rule['backgroundRate']) missing_message = 'Balrog rule is set for {} ({}) which is lower than {}'.format( rule['mapping'], ', '.join(sorted(set(build_ids.values()))), version) return build_task_response(status, url, exists_message, missing_message)
def test_parse_nightly_filename(arg, output): assert build_version_id(arg) == output