async def scan_candidates(session, product):
# For each version take the latest build.
global _candidates_build_folder
if product == 'mobile':
product = 'fennec'
if product in _candidates_build_folder:
return
logger.info(
"Scan '{}' candidates to get their latest build folder...".format(
product))
candidates_url = archive_url(product, candidate='/')
candidates_folders, _ = await fetch_listing(session, candidates_url)
for chunk in chunked(candidates_folders, NB_PARALLEL_REQUESTS):
futures = []
versions = []
for folder in chunk:
if '-candidates' not in folder:
continue
version = folder.replace('-candidates/', '')
versions.append(version)
builds_url = archive_url(product, version, candidate='/')
future = fetch_listing(session, builds_url)
futures.append(future)
listings = await asyncio.gather(*futures)
for version, (build_folders, _) in zip(versions, listings):
latest_build_folder = sorted(
build_folders,
key=lambda x: re.sub("[^0-9]", "", x).zfill(3))[-1]
_candidates_build_folder[product][version] = latest_build_folder
async def async_gen():
_csv_input = (
'net-mozaws-delivery-firefox,pub/firefox/releases/51.0/win64/fy-NL/'
'Firefox Setup 51.0.exe,67842,2017-06-11T12:20:10.2Z,'
'f1aa742ef0973db098947bd6d875f193\n'
'net-mozaws-delivery-firefox,pub/firefox/nightly/2017/06/'
'2017-06-16-03-02-07-mozilla-central-l10n/firefox-56.0a1.ach.win32.'
'installer.exe,45678,2017-06-16T03:02:07.0Z,'
'f1aa742ef0973db098947bd6d875f193\n'
'net-mozaws-delivery-firefox,pub/firefox/nightly/2017/06/'
'2017-06-16-03-02-07-mozilla-central-l10n/firefox-56.0a1.ach.win32.'
'zip,45678,2017-06-16T03:02:07.0Z,'
'f1aa742ef0973db098947bd6d875f193\n')
for line in utils.chunked(_csv_input, 32):
yield bytes(line, 'utf-8')
def test_chunked(iterable, size, chunks):
assert list(chunked(iterable, size)) == chunks
