async def fetch_nightly_metadata(session, record): """A JSON file containing build info is published along the nightly build archive. """ global _nightly_metadata url = record['download']['url'] # Make sure the nightly_url is turned into a en-US one. nightly_url = localize_nightly_url(url) if nightly_url in _nightly_metadata: return _nightly_metadata[nightly_url] extensions = '|'.join(FILE_EXTENSIONS) try: metadata_url = re.sub('\.({})$'.format(extensions), '.json', nightly_url) metadata = await fetch_json(session, metadata_url) _nightly_metadata[nightly_url] = metadata return metadata except aiohttp.ClientError: # Very old nightly metadata is published as .txt files. try: # e.g. https://archive.mozilla.org/pub/firefox/nightly/2011/05/ # 2011-05-05-03-mozilla-central/firefox-6.0a1.en-US.mac.txt old_metadata_url = re.sub('\.({})$'.format(extensions), '.txt', nightly_url) async with session.get(old_metadata_url) as response: old_metadata = await response.text() m = re.search('^(\d+)\n(http.+)/rev/(.+)$', old_metadata) if m: metadata = { 'buildid': m.group(1), 'moz_source_repo': m.group(2), 'moz_source_stamp': m.group(3), } _nightly_metadata[nightly_url] = metadata return metadata # e.g. https://archive.mozilla.org/pub/firefox/nightly/2010/07/2010-07-04-05 # -mozilla-central/firefox-4.0b2pre.en-US.win64-x86_64.txt m = re.search('^(\d+) (.+)$', old_metadata) if m: metadata = { 'buildid': m.group(1), 'moz_source_stamp': m.group(2), 'moz_source_repo': 'http://hg.mozilla.org/mozilla-central', } _nightly_metadata[nightly_url] = metadata return metadata except aiohttp.ClientError as e: pass logger.warning("Could not fetch metadata for '%s' from '%s'" % (record['id'], metadata_url)) _nightly_metadata[url] = None # Don't try it anymore. return None
def test_localize_nightly_url(localized_url, american_url): assert localize_nightly_url(localized_url) == american_url