async def get_locales(product, version): channel = get_version_channel(product, version) locales_path = product_locales_path(product) tag_product = product.upper() tag = "{}_{}_RELEASE".format(tag_product, version.replace('.', '_')) if channel is Channel.NIGHTLY: repo_name = get_channel_repo(product, channel, version) url = "https://hg.mozilla.org/{}/raw-file/tip/{}/locales/all-locales".format( repo_name, locales_path) elif channel in (Channel.RELEASE, Channel.BETA, Channel.AURORA): repo_name = get_channel_repo(product, channel, version) url = ("https://hg.mozilla.org/releases/{}/raw-file/{}/" "{}/locales/shipped-locales").format(repo_name, tag, locales_path) elif channel is Channel.CANDIDATE: # Not supported for Thunderbird if 'rc' in version: version, build = version.split('rc') else: version, build = version.split('build') # Build revision URL url = ( 'https://archive.mozilla.org/pub/{}/candidates/{}-candidates/build{}' '/linux-x86_64/en-US/firefox-{}.txt') url = url.format(product, version, build, version) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = '{} not available (HTTP {})'.format(url, resp.status) raise TaskError(msg, url=url) body = await resp.text() buildID, rev_url = body.strip().split('\n') url = '{}/browser/locales/shipped-locales'.format( rev_url.replace('rev', 'raw-file')) else: # Not supported for Thunderbird (Channel.ESR) major, _ = version.split('.', 1) branch = "mozilla-esr{}".format(major) url = ("https://hg.mozilla.org/releases/{}/raw-file/{}/" "browser/locales/shipped-locales").format(branch, tag) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = '{} not available (HTTP {})'.format(url, resp.status) raise TaskError(msg, url=url) hg_locales = [] body = await resp.text() for line in body.split('\n'): try: locale, _ = line.split(' ', 1) except ValueError: locale = line # We ignore here ja-JP-mac since because it is ja for the mac platform. # And we want them to be considered as the same locale. if locale and locale != 'ja-JP-mac': hg_locales.append(locale) return hg_locales
async def get_releases(product): RELEASE_CHANNEL = { 'devedition': 'aurora', 'firefox': 'release', } query = { "aggs": { "by_version": { "terms": { "field": "target.version", "size": 1000, "order": { "_term": "desc" } } } }, "query": { "bool": { "filter": [{ "term": { "source.product": product } }, { "term": { "target.channel": RELEASE_CHANNEL[product] } }] } }, "size": 0 } async with get_session() as session: url = '{}/buckets/build-hub/collections/releases/search' url = url.format(BUILDHUB_SERVER) async with session.post(url, data=json.dumps(query)) as response: if response.status != 200: message = "Buildhub is not available ({})".format( response.status) url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format( product) raise TaskError(message, url=url) data = await response.json() versions = sorted([ r['key'] for r in data['aggregations']['by_version']['buckets'] if strip_candidate_info(r['key']) == r['key'] ], key=lambda version: build_version_id(version)) if not versions: message = "Couldn't find any version matching." url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format( product) raise TaskError(message, url=url) return versions
async def bouncer(product, version): """Fetch bedrock download page to grab the bouncer download link and then make sure it redirects to the expected version.""" channel = get_version_channel(product, version) channel_value = channel.value if channel is Channel.ESR: bedrock_url = "https://www.mozilla.org/en-US/{}/organizations/all/".format( product) elif channel is Channel.RELEASE: bedrock_url = 'https://www.mozilla.org/en-US/{}/all/'.format(product) else: bedrock_url = 'https://www.mozilla.org/fr/{}/channel/desktop/'.format( product) if product == 'devedition': bedrock_url = 'https://www.mozilla.org/en-US/firefox/developer/' channel_value = "DEVEDITION" async with get_session() as session: async with session.get(bedrock_url) as resp: if resp.status != 200: msg = 'Download page not available ({})'.format(resp.status) raise TaskError(msg, url=bedrock_url) body = await resp.text() d = pq(body) if product == 'devedition': link_path = "#intro-download > .download-list > .os_linux64 > a" elif channel is Channel.NIGHTLY: link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a" elif channel in (Channel.BETA, Channel.AURORA): link_path = "#desktop-beta-download > .download-list > .os_linux64 > a" else: # channel in (Channel.RELEASE, Channel.ESR): link_path = "#fr > .linux64 > a" url = d(link_path).attr('href') if url is not None: async with session.get(url, allow_redirects=False) as resp: if resp.status == 302: url = resp.headers['Location'] else: msg = 'Bouncer is down ({}).'.format(resp.status) raise TaskError(msg, url=url) else: msg = 'No links found.'.format(resp.status) raise TaskError(msg, url=bedrock_url) filename = os.path.basename(url) last_release = get_version_from_filename(filename) status = build_version_id(last_release) >= build_version_id( version) message = "Bouncer for {} redirects to version {}".format( channel_value, last_release) return build_task_response(status, url, message)
async def put_query(session, query_title, version_name, query, *, query_id=None, run=True): # Update query with the last build_id if query_id: url = "{}/api/queries/{}".format(TELEMETRY_SERVER, query_id) else: url = "{}/api/queries".format(TELEMETRY_SERVER) payload = { "name": query_title, "schedule": 50000, # Twice a day "schedule_until": (date.today() + timedelta(days=7)).strftime('%Y-%m-%dT%H:%M:%S'), "is_draft": True, "query": query, "data_source_id": ATHENA_DATASOURCE_ID, "options": { "parameters": [] } } async with session.post(url, json=payload) as resp: if resp.status != 200: message = "Unable to create the new query for {} (HTTP {})" raise TaskError(message.format(version_name, resp.status), url=url) body = await resp.json() query_id = body["id"] if run: # Query for results url = "{}/api/query_results".format(TELEMETRY_SERVER) payload = { "data_source_id": ATHENA_DATASOURCE_ID, "query": query, "max_age": 0, "query_id": query_id } async with session.post(url, json=payload) as resp: if resp.status != 200: message = "Unable to execute the query n°{} for {} (HTTP {})" raise TaskError(message.format(query_id, version_name, resp.status), url=url) return query_id
async def product_details(product, version): if get_version_channel(product, version) is Channel.NIGHTLY: versions = await ongoing_versions(product) status = build_version_id( versions["nightly"]) >= build_version_id(version) message = "Last nightly version is {}".format(versions["nightly"]) url = "https://product-details.mozilla.org/1.0/{}_versions.json".format( product) return build_task_response(status, url, message) async with get_session() as session: url = 'https://product-details.mozilla.org/1.0/firefox.json' async with session.get(url) as resp: if resp.status != 200: msg = 'Product Details info not available (HTTP {})'.format( resp.status) raise TaskError(msg, url=url) body = await resp.json() status = 'firefox-{}'.format(version) in body['releases'] exists_message = "We found product-details information about version {}" missing_message = "We did not find product-details information about version {}" return build_task_response(status, url, exists_message.format(version), missing_message.format(version))
async def bouncer(product, version): """Make sure bouncer redirects to the expected version (or a later one).""" channel = get_version_channel(product, version) channel_value = channel.value if product == 'thunderbird' and channel == Channel.NIGHTLY: # NIGHTLY is a valid channel for Thunderbird, but it does not use the bouncer return build_task_response(True, "", "No nightly bouncer for Thunderbird.") if product == 'devedition': channel_value = "DEVEDITION" product_channel = 'firefox-{}'.format(channel_value.lower()) else: # product is 'firefox' or 'thunderbird' if channel == Channel.RELEASE: product_channel = product else: product_channel = '{}-{}'.format(product, channel_value.lower()) url = 'https://download.mozilla.org?product={}-latest-ssl&os=linux64&lang=en-US'.format( product_channel) async with get_session() as session: async with session.get(url, allow_redirects=False) as resp: if resp.status == 302: url = resp.headers['Location'] else: msg = 'Bouncer is down ({}).'.format(resp.status) raise TaskError(msg, url=url) filename = os.path.basename(url) last_release = get_version_from_filename(filename) status = build_version_id(last_release) >= build_version_id(version) message = "Bouncer for {} redirects to version {}".format(channel_value, last_release) return build_task_response(status, url, message)
async def get_release_info(release_mapping): release_url = 'https://aus-api.mozilla.org/api/v1/releases/{}'.format( release_mapping) async with get_session() as session: async with session.get(release_url) as resp: body = await resp.json() platforms = body['platforms'] built_platforms = [ x for x in platforms.keys() if 'locales' in platforms[x] ] if not built_platforms: raise TaskError( 'No platform with locales were found in {}'.format( sorted(platforms.keys())), url=release_url) build_ids = {} appVersions = set() for platform in built_platforms: platform_info = platforms[platform]['locales']["de"] build_ids[platform] = platform_info['buildID'] appVersions.add(platform_info['displayVersion'].replace( ' Beta ', 'b')) return build_ids, appVersions
async def ongoing_versions(product): async with get_session() as session: url = 'https://product-details.mozilla.org/1.0/{}'.format( details_versions_url(product)) async with session.get(url) as resp: if resp.status != 200: msg = 'Product Details info not available (HTTP {})'.format( resp.status) raise TaskError(msg, url=url) body = await resp.json() return details_ongoing_versions(product, body)
async def get_build_ids_for_version(product, version, *, size=10): channel = get_version_channel(product, strip_candidate_info(version)) channel_value = channel.value.lower() if product == "devedition": channel_value = "aurora" query = { "aggs": { "by_version": { "terms": { "field": "build.id", "size": size, "order": { "_term": "desc" } } } }, "query": { "bool": { "filter": [{ "term": { "target.channel": channel_value } }, { "term": { "source.product": product } }, { "term": { "target.version": version } }] } }, "size": 0 } async with get_session() as session: url = '{}/buckets/build-hub/collections/releases/search' url = url.format(BUILDHUB_SERVER) async with session.post(url, data=json.dumps(query)) as response: data = await response.json() build_ids = [ r['key'] for r in data['aggregations']['by_version']['buckets'] ] if not build_ids: message = "Couldn't find any build matching." raise TaskError(message, url=get_buildhub_url(product, version, channel)) return build_ids
async def get_platform_locale(url, platform): async with get_session() as session: url = '{}/{}/'.format(url.rstrip('/'), platform) async with session.get(url, headers=JSON_HEADERS) as resp: if resp.status != 200: msg = 'Archive CDN not available; failing to get {} (HTTP {})'.format( url, resp.status) raise TaskError(msg, url=url) body = await resp.json() return sorted([ p.strip('/') for p in body['prefixes'] if not p.startswith('xpi') ])
async def get_query_info_from_title(session, query_title): query_params = urlencode({"include_drafts": "true", "q": query_title}) query_url = "{}/api/queries/search?{}".format(TELEMETRY_SERVER, query_params) async with session.get(query_url) as resp: body = await resp.json() if body: if 'message' in body: raise TaskError("STMO: {}".format(body['message'])) body = [ query for query in body if not query['name'].startswith('Copy of') ] return body[0] if len(body) > 0 else None
async def ongoing_versions(product): async with get_session() as session: url = 'https://product-details.mozilla.org/1.0/firefox_versions.json' async with session.get(url) as resp: if resp.status != 200: msg = 'Product Details info not available (HTTP {})'.format( resp.status) raise TaskError(msg, url=url) body = await resp.json() return { "esr": body["FIREFOX_ESR"], "release": body["LATEST_FIREFOX_VERSION"], "beta": body["LATEST_FIREFOX_DEVEL_VERSION"], "nightly": body["FIREFOX_NIGHTLY"], "devedition": body["FIREFOX_DEVEDITION"], }
async def security_advisories(product, version): channel = get_version_channel(product, version) url = 'https://www.mozilla.org/en-US/security/known-vulnerabilities/{}/'.format( product) # Security advisories are always present for BETA and NIGHTLY # because we don't publish any. if channel in (Channel.BETA, Channel.NIGHTLY): return build_task_response( status=Status.MISSING, link=url, message="Security advisories are never published for {} releases". format(channel.value.lower())) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = 'Security advisories page not available ({})'.format( resp.status) raise TaskError(msg) # Does the content contains the version number? body = await resp.text() d = pq(body) if product in ['firefox', 'devedition']: security_product = 'firefox' if channel is Channel.ESR: version = re.sub('esr$', '', version) last_release = d("html").attr('data-esr-versions') else: last_release = d("html").attr('data-latest-firefox') elif product == 'thunderbird': security_product = product last_release = d("html").attr('data-esr-versions') status = build_version_id(last_release) >= build_version_id( version) message = ("Security advisories for release were " "updated up to version {}".format(last_release)) version_title = "#{}{}".format(security_product, version.split('.')[0]) if status and not d(version_title): status = Status.INCOMPLETE message += " but nothing was published for {} yet.".format( version_title) return build_task_response(status, url, message)
async def download_links(product, version): channel = get_version_channel(product, version) url = get_downloads_url(product, channel) async with get_session() as session: async with session.get(url) as resp: if resp.status != 200: msg = 'Download page not available ({})'.format(resp.status) raise TaskError(msg) body = await resp.text() d = pq(body) if product == 'thunderbird': if channel is Channel.NIGHTLY: link_path = ".download-link.btn-nightly" url = d(link_path).attr('href') filename = os.path.basename(url) last_release = get_version_from_filename(filename) else: last_release = d("#all-downloads").attr( 'data-thunderbird-version') else: if channel in (Channel.NIGHTLY, Channel.BETA, Channel.AURORA): if product == 'devedition': link_path = "#intro-download > .download-list > .os_linux64 > a" elif channel is Channel.NIGHTLY: link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a" else: # channel is Channel.BETA: link_path = "#desktop-beta-download > .download-list > .os_linux64 > a" url = d(link_path).attr('href') async with session.get(url, allow_redirects=False) as resp: url = resp.headers['Location'] filename = os.path.basename(url) last_release = get_version_from_filename(filename) elif channel is Channel.ESR: version = re.sub('esr$', '', version) last_release = d("html").attr('data-esr-versions') else: # Does the content contains the version number? last_release = d("html").attr('data-latest-firefox') status = build_version_id(last_release) >= build_version_id( version) message = ( "The download links for release have been published for version {}" .format(last_release)) return build_task_response(status, url, message)
async def archives(product, version): async with get_session() as session: channel = get_version_channel(product, version) url = build_version_url(product, version) if channel is Channel.NIGHTLY: message = "No archive found at {}".format(url) async with session.get(url, headers=JSON_HEADERS) as resp: if resp.status != 200: success = False else: body = await resp.json() files = sorted([ r["name"] for r in body["files"] if r["name"].lower().startswith(product) and not r["name"].endswith('mar') ], reverse=True) success, message = await check_nightly_releases_files( url, files, product, version) return build_task_response(success, url, message) else: url = build_version_url(product, version) async with session.get(url, headers=JSON_HEADERS) as resp: if resp.status >= 500: msg = 'Archive CDN not available (HTTP {})'.format( resp.status) raise TaskError(msg, url=url) success = resp.status < 400 message = ("No archive found for this version number at {}". format(url)) if success: success, message = await check_releases_files( url, product, version) return build_task_response(success, url, message)
async def error_task(product, version): raise TaskError('Error message', url='http://www.perdu.com/')
async def error_task(product, version): raise TaskError('Error message')