async def scan_candidates(session, product): # For each version take the latest build. global _candidates_build_folder if product == 'mobile': product = 'fennec' if product in _candidates_build_folder: return logger.info( "Scan '{}' candidates to get their latest build folder...".format( product)) candidates_url = archive_url(product, candidate='/') candidates_folders, _ = await fetch_listing(session, candidates_url) for chunk in chunked(candidates_folders, NB_PARALLEL_REQUESTS): futures = [] versions = [] for folder in chunk: if '-candidates' not in folder: continue version = folder.replace('-candidates/', '') versions.append(version) builds_url = archive_url(product, version, candidate='/') future = fetch_listing(session, builds_url) futures.append(future) listings = await asyncio.gather(*futures) for version, (build_folders, _) in zip(versions, listings): latest_build_folder = sorted( build_folders, key=lambda x: re.sub("[^0-9]", "", x).zfill(3))[-1] _candidates_build_folder[product][version] = latest_build_folder
async def async_gen(): _csv_input = ( 'net-mozaws-delivery-firefox,pub/firefox/releases/51.0/win64/fy-NL/' 'Firefox Setup 51.0.exe,67842,2017-06-11T12:20:10.2Z,' 'f1aa742ef0973db098947bd6d875f193\n' 'net-mozaws-delivery-firefox,pub/firefox/nightly/2017/06/' '2017-06-16-03-02-07-mozilla-central-l10n/firefox-56.0a1.ach.win32.' 'installer.exe,45678,2017-06-16T03:02:07.0Z,' 'f1aa742ef0973db098947bd6d875f193\n' 'net-mozaws-delivery-firefox,pub/firefox/nightly/2017/06/' '2017-06-16-03-02-07-mozilla-central-l10n/firefox-56.0a1.ach.win32.' 'zip,45678,2017-06-16T03:02:07.0Z,' 'f1aa742ef0973db098947bd6d875f193\n') for line in utils.chunked(_csv_input, 32): yield bytes(line, 'utf-8')
def test_chunked(iterable, size, chunks): assert list(chunked(iterable, size)) == chunks