async def get_locales(product, version):
channel = get_version_channel(product, version)
locales_path = product_locales_path(product)
tag_product = product.upper()
tag = "{}_{}_RELEASE".format(tag_product, version.replace('.', '_'))
if channel is Channel.NIGHTLY:
repo_name = get_channel_repo(product, channel, version)
url = "https://hg.mozilla.org/{}/raw-file/tip/{}/locales/all-locales".format(
repo_name, locales_path)
elif channel in (Channel.RELEASE, Channel.BETA, Channel.AURORA):
repo_name = get_channel_repo(product, channel, version)
url = ("https://hg.mozilla.org/releases/{}/raw-file/{}/"
"{}/locales/shipped-locales").format(repo_name, tag,
locales_path)
elif channel is Channel.CANDIDATE:
# Not supported for Thunderbird
if 'rc' in version:
version, build = version.split('rc')
else:
version, build = version.split('build')
# Build revision URL
url = (
'https://archive.mozilla.org/pub/{}/candidates/{}-candidates/build{}'
'/linux-x86_64/en-US/firefox-{}.txt')
url = url.format(product, version, build, version)
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = '{} not available (HTTP {})'.format(url, resp.status)
raise TaskError(msg, url=url)
body = await resp.text()
buildID, rev_url = body.strip().split('\n')
url = '{}/browser/locales/shipped-locales'.format(
rev_url.replace('rev', 'raw-file'))
else:
# Not supported for Thunderbird (Channel.ESR)
major, _ = version.split('.', 1)
branch = "mozilla-esr{}".format(major)
url = ("https://hg.mozilla.org/releases/{}/raw-file/{}/"
"browser/locales/shipped-locales").format(branch, tag)
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = '{} not available (HTTP {})'.format(url, resp.status)
raise TaskError(msg, url=url)
hg_locales = []
body = await resp.text()
for line in body.split('\n'):
try:
locale, _ = line.split(' ', 1)
except ValueError:
locale = line
# We ignore here ja-JP-mac since because it is ja for the mac platform.
# And we want them to be considered as the same locale.
if locale and locale != 'ja-JP-mac':
hg_locales.append(locale)
return hg_locales
async def get_releases(product):
RELEASE_CHANNEL = {
'devedition': 'aurora',
'firefox': 'release',
}
query = {
"aggs": {
"by_version": {
"terms": {
"field": "target.version",
"size": 1000,
"order": {
"_term": "desc"
}
}
}
},
"query": {
"bool": {
"filter": [{
"term": {
"source.product": product
}
}, {
"term": {
"target.channel": RELEASE_CHANNEL[product]
}
}]
}
},
"size": 0
}
async with get_session() as session:
url = '{}/buckets/build-hub/collections/releases/search'
url = url.format(BUILDHUB_SERVER)
async with session.post(url, data=json.dumps(query)) as response:
if response.status != 200:
message = "Buildhub is not available ({})".format(
response.status)
url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format(
product)
raise TaskError(message, url=url)
data = await response.json()
versions = sorted([
r['key'] for r in data['aggregations']['by_version']['buckets']
if strip_candidate_info(r['key']) == r['key']
],
key=lambda version: build_version_id(version))
if not versions:
message = "Couldn't find any version matching."
url = "https://mozilla-services.github.io/buildhub/?products[0]={}".format(
product)
raise TaskError(message, url=url)
return versions
async def bouncer(product, version):
"""Fetch bedrock download page to grab the bouncer download link and then make sure
it redirects to the expected version."""
channel = get_version_channel(product, version)
channel_value = channel.value
if channel is Channel.ESR:
bedrock_url = "https://www.mozilla.org/en-US/{}/organizations/all/".format(
product)
elif channel is Channel.RELEASE:
bedrock_url = 'https://www.mozilla.org/en-US/{}/all/'.format(product)
else:
bedrock_url = 'https://www.mozilla.org/fr/{}/channel/desktop/'.format(
product)
if product == 'devedition':
bedrock_url = 'https://www.mozilla.org/en-US/firefox/developer/'
channel_value = "DEVEDITION"
async with get_session() as session:
async with session.get(bedrock_url) as resp:
if resp.status != 200:
msg = 'Download page not available ({})'.format(resp.status)
raise TaskError(msg, url=bedrock_url)
body = await resp.text()
d = pq(body)
if product == 'devedition':
link_path = "#intro-download > .download-list > .os_linux64 > a"
elif channel is Channel.NIGHTLY:
link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a"
elif channel in (Channel.BETA, Channel.AURORA):
link_path = "#desktop-beta-download > .download-list > .os_linux64 > a"
else: # channel in (Channel.RELEASE, Channel.ESR):
link_path = "#fr > .linux64 > a"
url = d(link_path).attr('href')
if url is not None:
async with session.get(url, allow_redirects=False) as resp:
if resp.status == 302:
url = resp.headers['Location']
else:
msg = 'Bouncer is down ({}).'.format(resp.status)
raise TaskError(msg, url=url)
else:
msg = 'No links found.'.format(resp.status)
raise TaskError(msg, url=bedrock_url)
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
status = build_version_id(last_release) >= build_version_id(
version)
message = "Bouncer for {} redirects to version {}".format(
channel_value, last_release)
return build_task_response(status, url, message)
async def put_query(session,
query_title,
version_name,
query,
*,
query_id=None,
run=True):
# Update query with the last build_id
if query_id:
url = "{}/api/queries/{}".format(TELEMETRY_SERVER, query_id)
else:
url = "{}/api/queries".format(TELEMETRY_SERVER)
payload = {
"name":
query_title,
"schedule":
50000, # Twice a day
"schedule_until":
(date.today() + timedelta(days=7)).strftime('%Y-%m-%dT%H:%M:%S'),
"is_draft":
True,
"query":
query,
"data_source_id":
ATHENA_DATASOURCE_ID,
"options": {
"parameters": []
}
}
async with session.post(url, json=payload) as resp:
if resp.status != 200:
message = "Unable to create the new query for {} (HTTP {})"
raise TaskError(message.format(version_name, resp.status), url=url)
body = await resp.json()
query_id = body["id"]
if run:
# Query for results
url = "{}/api/query_results".format(TELEMETRY_SERVER)
payload = {
"data_source_id": ATHENA_DATASOURCE_ID,
"query": query,
"max_age": 0,
"query_id": query_id
}
async with session.post(url, json=payload) as resp:
if resp.status != 200:
message = "Unable to execute the query n°{} for {} (HTTP {})"
raise TaskError(message.format(query_id, version_name,
resp.status),
url=url)
return query_id
async def product_details(product, version):
if get_version_channel(product, version) is Channel.NIGHTLY:
versions = await ongoing_versions(product)
status = build_version_id(
versions["nightly"]) >= build_version_id(version)
message = "Last nightly version is {}".format(versions["nightly"])
url = "https://product-details.mozilla.org/1.0/{}_versions.json".format(
product)
return build_task_response(status, url, message)
async with get_session() as session:
url = 'https://product-details.mozilla.org/1.0/firefox.json'
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Product Details info not available (HTTP {})'.format(
resp.status)
raise TaskError(msg, url=url)
body = await resp.json()
status = 'firefox-{}'.format(version) in body['releases']
exists_message = "We found product-details information about version {}"
missing_message = "We did not find product-details information about version {}"
return build_task_response(status, url,
exists_message.format(version),
missing_message.format(version))
async def bouncer(product, version):
"""Make sure bouncer redirects to the expected version (or a later one)."""
channel = get_version_channel(product, version)
channel_value = channel.value
if product == 'thunderbird' and channel == Channel.NIGHTLY:
# NIGHTLY is a valid channel for Thunderbird, but it does not use the bouncer
return build_task_response(True, "", "No nightly bouncer for Thunderbird.")
if product == 'devedition':
channel_value = "DEVEDITION"
product_channel = 'firefox-{}'.format(channel_value.lower())
else: # product is 'firefox' or 'thunderbird'
if channel == Channel.RELEASE:
product_channel = product
else:
product_channel = '{}-{}'.format(product, channel_value.lower())
url = 'https://download.mozilla.org?product={}-latest-ssl&os=linux64&lang=en-US'.format(
product_channel)
async with get_session() as session:
async with session.get(url, allow_redirects=False) as resp:
if resp.status == 302:
url = resp.headers['Location']
else:
msg = 'Bouncer is down ({}).'.format(resp.status)
raise TaskError(msg, url=url)
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
status = build_version_id(last_release) >= build_version_id(version)
message = "Bouncer for {} redirects to version {}".format(channel_value, last_release)
return build_task_response(status, url, message)
async def get_release_info(release_mapping):
release_url = 'https://aus-api.mozilla.org/api/v1/releases/{}'.format(
release_mapping)
async with get_session() as session:
async with session.get(release_url) as resp:
body = await resp.json()
platforms = body['platforms']
built_platforms = [
x for x in platforms.keys() if 'locales' in platforms[x]
]
if not built_platforms:
raise TaskError(
'No platform with locales were found in {}'.format(
sorted(platforms.keys())),
url=release_url)
build_ids = {}
appVersions = set()
for platform in built_platforms:
platform_info = platforms[platform]['locales']["de"]
build_ids[platform] = platform_info['buildID']
appVersions.add(platform_info['displayVersion'].replace(
' Beta ', 'b'))
return build_ids, appVersions
async def ongoing_versions(product):
async with get_session() as session:
url = 'https://product-details.mozilla.org/1.0/{}'.format(
details_versions_url(product))
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Product Details info not available (HTTP {})'.format(
resp.status)
raise TaskError(msg, url=url)
body = await resp.json()
return details_ongoing_versions(product, body)
async def get_build_ids_for_version(product, version, *, size=10):
channel = get_version_channel(product, strip_candidate_info(version))
channel_value = channel.value.lower()
if product == "devedition":
channel_value = "aurora"
query = {
"aggs": {
"by_version": {
"terms": {
"field": "build.id",
"size": size,
"order": {
"_term": "desc"
}
}
}
},
"query": {
"bool": {
"filter": [{
"term": {
"target.channel": channel_value
}
}, {
"term": {
"source.product": product
}
}, {
"term": {
"target.version": version
}
}]
}
},
"size": 0
}
async with get_session() as session:
url = '{}/buckets/build-hub/collections/releases/search'
url = url.format(BUILDHUB_SERVER)
async with session.post(url, data=json.dumps(query)) as response:
data = await response.json()
build_ids = [
r['key'] for r in data['aggregations']['by_version']['buckets']
]
if not build_ids:
message = "Couldn't find any build matching."
raise TaskError(message,
url=get_buildhub_url(product, version, channel))
return build_ids
async def get_platform_locale(url, platform):
async with get_session() as session:
url = '{}/{}/'.format(url.rstrip('/'), platform)
async with session.get(url, headers=JSON_HEADERS) as resp:
if resp.status != 200:
msg = 'Archive CDN not available; failing to get {} (HTTP {})'.format(
url, resp.status)
raise TaskError(msg, url=url)
body = await resp.json()
return sorted([
p.strip('/') for p in body['prefixes']
if not p.startswith('xpi')
])
async def get_query_info_from_title(session, query_title):
query_params = urlencode({"include_drafts": "true", "q": query_title})
query_url = "{}/api/queries/search?{}".format(TELEMETRY_SERVER,
query_params)
async with session.get(query_url) as resp:
body = await resp.json()
if body:
if 'message' in body:
raise TaskError("STMO: {}".format(body['message']))
body = [
query for query in body
if not query['name'].startswith('Copy of')
]
return body[0] if len(body) > 0 else None
async def ongoing_versions(product):
async with get_session() as session:
url = 'https://product-details.mozilla.org/1.0/firefox_versions.json'
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Product Details info not available (HTTP {})'.format(
resp.status)
raise TaskError(msg, url=url)
body = await resp.json()
return {
"esr": body["FIREFOX_ESR"],
"release": body["LATEST_FIREFOX_VERSION"],
"beta": body["LATEST_FIREFOX_DEVEL_VERSION"],
"nightly": body["FIREFOX_NIGHTLY"],
"devedition": body["FIREFOX_DEVEDITION"],
}
async def security_advisories(product, version):
channel = get_version_channel(product, version)
url = 'https://www.mozilla.org/en-US/security/known-vulnerabilities/{}/'.format(
product)
# Security advisories are always present for BETA and NIGHTLY
# because we don't publish any.
if channel in (Channel.BETA, Channel.NIGHTLY):
return build_task_response(
status=Status.MISSING,
link=url,
message="Security advisories are never published for {} releases".
format(channel.value.lower()))
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Security advisories page not available ({})'.format(
resp.status)
raise TaskError(msg)
# Does the content contains the version number?
body = await resp.text()
d = pq(body)
if product in ['firefox', 'devedition']:
security_product = 'firefox'
if channel is Channel.ESR:
version = re.sub('esr$', '', version)
last_release = d("html").attr('data-esr-versions')
else:
last_release = d("html").attr('data-latest-firefox')
elif product == 'thunderbird':
security_product = product
last_release = d("html").attr('data-esr-versions')
status = build_version_id(last_release) >= build_version_id(
version)
message = ("Security advisories for release were "
"updated up to version {}".format(last_release))
version_title = "#{}{}".format(security_product,
version.split('.')[0])
if status and not d(version_title):
status = Status.INCOMPLETE
message += " but nothing was published for {} yet.".format(
version_title)
return build_task_response(status, url, message)
async def download_links(product, version):
channel = get_version_channel(product, version)
url = get_downloads_url(product, channel)
async with get_session() as session:
async with session.get(url) as resp:
if resp.status != 200:
msg = 'Download page not available ({})'.format(resp.status)
raise TaskError(msg)
body = await resp.text()
d = pq(body)
if product == 'thunderbird':
if channel is Channel.NIGHTLY:
link_path = ".download-link.btn-nightly"
url = d(link_path).attr('href')
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
else:
last_release = d("#all-downloads").attr(
'data-thunderbird-version')
else:
if channel in (Channel.NIGHTLY, Channel.BETA, Channel.AURORA):
if product == 'devedition':
link_path = "#intro-download > .download-list > .os_linux64 > a"
elif channel is Channel.NIGHTLY:
link_path = "#desktop-nightly-download > .download-list > .os_linux64 > a"
else: # channel is Channel.BETA:
link_path = "#desktop-beta-download > .download-list > .os_linux64 > a"
url = d(link_path).attr('href')
async with session.get(url, allow_redirects=False) as resp:
url = resp.headers['Location']
filename = os.path.basename(url)
last_release = get_version_from_filename(filename)
elif channel is Channel.ESR:
version = re.sub('esr$', '', version)
last_release = d("html").attr('data-esr-versions')
else:
# Does the content contains the version number?
last_release = d("html").attr('data-latest-firefox')
status = build_version_id(last_release) >= build_version_id(
version)
message = (
"The download links for release have been published for version {}"
.format(last_release))
return build_task_response(status, url, message)
async def archives(product, version):
async with get_session() as session:
channel = get_version_channel(product, version)
url = build_version_url(product, version)
if channel is Channel.NIGHTLY:
message = "No archive found at {}".format(url)
async with session.get(url, headers=JSON_HEADERS) as resp:
if resp.status != 200:
success = False
else:
body = await resp.json()
files = sorted([
r["name"] for r in body["files"]
if r["name"].lower().startswith(product)
and not r["name"].endswith('mar')
],
reverse=True)
success, message = await check_nightly_releases_files(
url, files, product, version)
return build_task_response(success, url, message)
else:
url = build_version_url(product, version)
async with session.get(url, headers=JSON_HEADERS) as resp:
if resp.status >= 500:
msg = 'Archive CDN not available (HTTP {})'.format(
resp.status)
raise TaskError(msg, url=url)
success = resp.status < 400
message = ("No archive found for this version number at {}".
format(url))
if success:
success, message = await check_releases_files(
url, product, version)
return build_task_response(success, url, message)
async def error_task(product, version):
raise TaskError('Error message', url='http://www.perdu.com/')
async def error_task(product, version):
raise TaskError('Error message')
