async def product_details(product, version):
if get_version_channel(product, version) is Channel.NIGHTLY:
versions = await ongoing_versions(product)
status = build_version_id(
versions["nightly"]) >= build_version_id(version)
message = "Last nightly version is {}".format(versions["nightly"])
url = "https://product-details.mozilla.org/1.0/{}_versions.json".format(
product)
return build_task_response(status, url, message)
async with get_session() as session:
url = 'https://product-details.mozilla.org/1.0/firefox.json'
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Product Details info not available (HTTP {})'.format(
resp.status)
raise TaskError(msg, url=url)
body = await resp.json()
status = 'firefox-{}'.format(version) in body['releases']
exists_message = "We found product-details information about version {}"
missing_message = "We did not find product-details information about version {}"
return build_task_response(status, url,
exists_message.format(version),
missing_message.format(version))
async def bouncer(product, version):
"""Make sure bouncer redirects to the expected version (or a later one)."""
channel = get_version_channel(product, version)
channel_value = channel.value
if product == 'thunderbird' and channel == Channel.NIGHTLY:
# NIGHTLY is a valid channel for Thunderbird, but it does not use the bouncer
return build_task_response(True, "", "No nightly bouncer for Thunderbird.")
if product == 'devedition':
channel_value = "DEVEDITION"
product_channel = 'firefox-{}'.format(channel_value.lower())
else: # product is 'firefox' or 'thunderbird'
if channel == Channel.RELEASE:
product_channel = product
else:
product_channel = '{}-{}'.format(product, channel_value.lower())
url = 'https://download.mozilla.org?product={}-latest-ssl&os=linux64&lang=en-US'.format(
product_channel)
async with get_session() as session:
async with session.get(url, allow_redirects=False) as resp:
if resp.status == 302:
url = resp.headers['Location']
else:
msg = 'Bouncer is down ({}).'.format(resp.status)
raise TaskError(msg, url=url)
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
status = build_version_id(last_release) >= build_version_id(version)
message = "Bouncer for {} redirects to version {}".format(channel_value, last_release)
return build_task_response(status, url, message)
async def bouncer(product, version):
"""Fetch bedrock download page to grab the bouncer download link and then make sure
it redirects to the expected version."""
channel = get_version_channel(product, version)
channel_value = channel.value
if channel is Channel.ESR:
bedrock_url = "https://www.mozilla.org/en-US/{}/organizations/all/".format(
product)
elif channel is Channel.RELEASE:
bedrock_url = 'https://www.mozilla.org/en-US/{}/all/'.format(product)
else:
bedrock_url = 'https://www.mozilla.org/fr/{}/channel/desktop/'.format(
product)
if product == 'devedition':
bedrock_url = 'https://www.mozilla.org/en-US/firefox/developer/'
channel_value = "DEVEDITION"
async with get_session() as session:
async with session.get(bedrock_url) as resp:
if resp.status != 200:
msg = 'Download page not available ({})'.format(resp.status)
raise TaskError(msg, url=bedrock_url)
body = await resp.text()
d = pq(body)
if product == 'devedition':
link_path = "#intro-download > .download-list > .os_linux64 > a"
elif channel is Channel.NIGHTLY:
link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a"
elif channel in (Channel.BETA, Channel.AURORA):
link_path = "#desktop-beta-download > .download-list > .os_linux64 > a"
else: # channel in (Channel.RELEASE, Channel.ESR):
link_path = "#fr > .linux64 > a"
url = d(link_path).attr('href')
if url is not None:
async with session.get(url, allow_redirects=False) as resp:
if resp.status == 302:
url = resp.headers['Location']
else:
msg = 'Bouncer is down ({}).'.format(resp.status)
raise TaskError(msg, url=url)
else:
msg = 'No links found.'.format(resp.status)
raise TaskError(msg, url=bedrock_url)
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
status = build_version_id(last_release) >= build_version_id(
version)
message = "Bouncer for {} redirects to version {}".format(
channel_value, last_release)
return build_task_response(status, url, message)
async def download_links(product, version):
channel = get_version_channel(product, version)
url = get_downloads_url(product, channel)
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Download page not available ({})'.format(resp.status)
raise TaskError(msg)
body = await resp.text()
d = pq(body)
if product == 'thunderbird':
if channel is Channel.NIGHTLY:
link_path = ".download-link.btn-nightly"
url = d(link_path).attr('href')
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
else:
last_release = d("#all-downloads").attr(
'data-thunderbird-version')
else:
if channel in (Channel.NIGHTLY, Channel.BETA, Channel.AURORA):
if product == 'devedition':
link_path = "#intro-download > .download-list > .os_linux64 > a"
elif channel is Channel.NIGHTLY:
link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a"
else: # channel is Channel.BETA:
link_path = "#desktop-beta-download > .download-list > .os_linux64 > a"
url = d(link_path).attr('href')
async with session.get(url, allow_redirects=False) as resp:
url = resp.headers['Location']
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
elif channel is Channel.ESR:
version = re.sub('esr$', '', version)
last_release = d("html").attr('data-esr-versions')
else:
# Does the content contains the version number?
last_release = d("html").attr('data-latest-firefox')
status = build_version_id(last_release) >= build_version_id(
version)
message = (
"The download links for release have been published for version {}"
.format(last_release))
return build_task_response(status, url, message)
async def security_advisories(product, version):
channel = get_version_channel(product, version)
url = 'https://www.mozilla.org/en-US/security/known-vulnerabilities/{}/'.format(
product)
# Security advisories are always present for BETA and NIGHTLY
# because we don't publish any.
if channel in (Channel.BETA, Channel.NIGHTLY):
return build_task_response(
status=Status.MISSING,
link=url,
message="Security advisories are never published for {} releases".
format(channel.value.lower()))
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Security advisories page not available ({})'.format(
resp.status)
raise TaskError(msg)
# Does the content contains the version number?
body = await resp.text()
d = pq(body)
if product in ['firefox', 'devedition']:
security_product = 'firefox'
if channel is Channel.ESR:
version = re.sub('esr$', '', version)
last_release = d("html").attr('data-esr-versions')
else:
last_release = d("html").attr('data-latest-firefox')
elif product == 'thunderbird':
security_product = product
last_release = d("html").attr('data-esr-versions')
status = build_version_id(last_release) >= build_version_id(
version)
message = ("Security advisories for release were "
"updated up to version {}".format(last_release))
version_title = "#{}{}".format(security_product,
version.split('.')[0])
if status and not d(version_title):
status = Status.INCOMPLETE
message += " but nothing was published for {} yet.".format(
version_title)
return build_task_response(status, url, message)
async def get_releases(product):
RELEASE_CHANNEL = {
'devedition': 'aurora',
'firefox': 'release',
}
query = {
"aggs": {
"by_version": {
"terms": {
"field": "target.version",
"size": 1000,
"order": {
"_term": "desc"
}
}
}
},
"query": {
"bool": {
"filter": [{
"term": {
"source.product": product
}
}, {
"term": {
"target.channel": RELEASE_CHANNEL[product]
}
}]
}
},
"size": 0
}
async with get_session() as session:
url = '{}/buckets/build-hub/collections/releases/search'
url = url.format(BUILDHUB_SERVER)
async with session.post(url, data=json.dumps(query)) as response:
if response.status != 200:
message = "Buildhub is not available ({})".format(
response.status)
url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format(
product)
raise TaskError(message, url=url)
data = await response.json()
versions = sorted([
r['key'] for r in data['aggregations']['by_version']['buckets']
if strip_candidate_info(r['key']) == r['key']
],
key=lambda version: build_version_id(version))
if not versions:
message = "Couldn't find any version matching."
url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format(
product)
raise TaskError(message, url=url)
return versions
async def balrog_rules(product, version):
channel = get_version_channel(product, version)
if channel is Channel.NIGHTLY:
# In that case the rule doesn't change, so we grab the build IDs.
# There are case were Nightly is deactivated, in that case
# the mapping is not nightly-latest anymore
url = 'https://aus-api.mozilla.org/api/v1/rules/firefox-nightly'
async with get_session() as session:
async with session.get(url) as resp:
rule = await resp.json()
status = rule[
'mapping'] == 'Firefox-mozilla-central-nightly-latest'
build_ids, appVersions = await get_release_info(rule['mapping']
)
last_build_id = max(build_ids.values())
date = last_build_id[:8]
old_build_id = [
bid for bid in build_ids.values()
if not bid.startswith(date)
]
if rule['mapping'] != 'Firefox-mozilla-central-nightly-latest':
status = Status.MISSING
message = (
'Balrog rule is configured for {} ({}) instead of '
'"Firefox-mozilla-central-nightly-latest"')
message = message.format(
rule['mapping'],
', '.join(sorted(set(build_ids.values()))))
elif old_build_id:
platforms = [
k for k, v in build_ids.items() if v in old_build_id
]
status = Status.INCOMPLETE
message = (
"Balrog rule is configured for {} ({}) platform {} with build ID {}"
" seem outdated.")
message = message.format(
rule['mapping'],
', '.join(sorted(set(build_ids.values()))),
', '.join(platforms),
', '.join(sorted(set(old_build_id))))
else:
status = Status.EXISTS
message = (
'Balrog rule is configured for the latest Nightly {} build ({}) '
'with an update rate of {}%')
message = message.format(
', '.join(sorted(appVersions)),
', '.join(sorted(set(build_ids.values()))),
rule['backgroundRate'])
return build_task_response(status, url, message)
elif channel in (Channel.BETA, Channel.AURORA):
rule_name = 'devedition' if product == 'devedition' else 'firefox-beta'
url = 'https://aus-api.mozilla.org/api/v1/rules/{}'.format(rule_name)
elif channel is Channel.ESR:
version = re.sub('esr$', '', version)
url = 'https://aus-api.mozilla.org/api/v1/rules/esr{}'.format(
version.split('.')[0])
else:
url = 'https://aus-api.mozilla.org/api/v1/rules/firefox-release'
async with get_session() as session:
async with session.get(url) as resp:
rule = await resp.json()
build_ids, appVersions = await get_release_info(rule['mapping'])
status = build_version_id(
appVersions.pop()) >= build_version_id(version)
exists_message = (
'Balrog rule has been updated for {} ({}) with an update rate of {}%'
).format(rule['mapping'],
', '.join(sorted(set(build_ids.values()))),
rule['backgroundRate'])
missing_message = 'Balrog rule is set for {} ({}) which is lower than {}'.format(
rule['mapping'], ', '.join(sorted(set(build_ids.values()))),
version)
return build_task_response(status, url, exists_message,
missing_message)
def test_parse_nightly_filename(arg, output):
assert build_version_id(arg) == output