async def scan_candidates(session, product):
# For each version take the latest build.
global _candidates_build_folder
if product == 'mobile':
product = 'fennec'
if product in _candidates_build_folder:
return
logger.info(
"Scan '{}' candidates to get their latest build folder...".format(
product))
candidates_url = archive_url(product, candidate='/')
candidates_folders, _ = await fetch_listing(session, candidates_url)
for chunk in chunked(candidates_folders, NB_PARALLEL_REQUESTS):
futures = []
versions = []
for folder in chunk:
if '-candidates' not in folder:
continue
version = folder.replace('-candidates/', '')
versions.append(version)
builds_url = archive_url(product, version, candidate='/')
future = fetch_listing(session, builds_url)
futures.append(future)
listings = await asyncio.gather(*futures)
for version, (build_folders, _) in zip(versions, listings):
latest_build_folder = sorted(
build_folders,
key=lambda x: re.sub("[^0-9]", "", x).zfill(3))[-1]
_candidates_build_folder[product][version] = latest_build_folder
async def fetch_release_metadata(session, record):
"""The `candidates` folder contains build info about recent
released versions.
"""
global _candidates_build_folder
product = record['source']['product']
version = record['target']['version']
platform = record['target']['platform']
locale = 'en-US'
try:
latest_build_folder = _candidates_build_folder[product][version]
except KeyError:
# Version is not listed in candidates. Give up.
return None
build_number = int(latest_build_folder.strip('/')[-1]) # build3 -> 3
# Metadata for EME-free and sha1 repacks are the same as original release.
platform = re.sub('-(eme-free|sha1)', '', platform, flags=re.I)
url = archive_url(product,
version,
platform,
locale,
candidate='/' + latest_build_folder)
# We already have the metadata for this platform and version.
if url in _release_metadata:
return _release_metadata[url]
try:
_, files = await fetch_listing(session, url)
except ValueError:
# Some partial update don't have metadata. eg. /47.0.1-candidates/
_release_metadata[url] = None
return None
for f in files:
filename = f['name']
if is_release_build_metadata(product, version, filename):
try:
metadata = await fetch_json(session, url + filename)
metadata['buildnumber'] = build_number
_release_metadata[url] = metadata
return metadata
except aiohttp.ClientError as e:
# Sometimes, some XML comes out \o/ (see #259)
pass
# Version exists in candidates but has no metadata!
_release_metadata[url] = None # Don't try it anymore.
raise ValueError('Missing metadata for candidate {}'.format(url))
def test_archive_url(info, expected_record):
assert archive_url(*info) == expected_record