async def scan_candidates(session, product): # For each version take the latest build. global _candidates_build_folder if product == 'mobile': product = 'fennec' if product in _candidates_build_folder: return logger.info( "Scan '{}' candidates to get their latest build folder...".format( product)) candidates_url = archive_url(product, candidate='/') candidates_folders, _ = await fetch_listing(session, candidates_url) for chunk in chunked(candidates_folders, NB_PARALLEL_REQUESTS): futures = [] versions = [] for folder in chunk: if '-candidates' not in folder: continue version = folder.replace('-candidates/', '') versions.append(version) builds_url = archive_url(product, version, candidate='/') future = fetch_listing(session, builds_url) futures.append(future) listings = await asyncio.gather(*futures) for version, (build_folders, _) in zip(versions, listings): latest_build_folder = sorted( build_folders, key=lambda x: re.sub("[^0-9]", "", x).zfill(3))[-1] _candidates_build_folder[product][version] = latest_build_folder
async def fetch_release_metadata(session, record): """The `candidates` folder contains build info about recent released versions. """ global _candidates_build_folder product = record['source']['product'] version = record['target']['version'] platform = record['target']['platform'] locale = 'en-US' try: latest_build_folder = _candidates_build_folder[product][version] except KeyError: # Version is not listed in candidates. Give up. return None build_number = int(latest_build_folder.strip('/')[-1]) # build3 -> 3 # Metadata for EME-free and sha1 repacks are the same as original release. platform = re.sub('-(eme-free|sha1)', '', platform, flags=re.I) url = archive_url(product, version, platform, locale, candidate='/' + latest_build_folder) # We already have the metadata for this platform and version. if url in _release_metadata: return _release_metadata[url] try: _, files = await fetch_listing(session, url) except ValueError: # Some partial update don't have metadata. eg. /47.0.1-candidates/ _release_metadata[url] = None return None for f in files: filename = f['name'] if is_release_build_metadata(product, version, filename): try: metadata = await fetch_json(session, url + filename) metadata['buildnumber'] = build_number _release_metadata[url] = metadata return metadata except aiohttp.ClientError as e: # Sometimes, some XML comes out \o/ (see #259) pass # Version exists in candidates but has no metadata! _release_metadata[url] = None # Don't try it anymore. raise ValueError('Missing metadata for candidate {}'.format(url))
def test_archive_url(info, expected_record): assert archive_url(*info) == expected_record