async def main(loop, inventory): """ Trigger to populate kinto with the last inventories. """ server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1') bucket = os.getenv('BUCKET', 'build-hub') collection = os.getenv('COLLECTION', 'releases') kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, bucket=bucket, collection=collection, retry=NB_RETRY_REQUEST) # Create bucket/collection and schemas. if INITIALIZE_SERVER: await initialize_kinto(loop, kinto_client, bucket, collection) # Download CSVs, deduce records and push to Kinto. session = aiobotocore.get_session(loop=loop) boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED) async with session.create_client('s3', region_name=REGION_NAME, config=boto_config) as client: keys_stream = list_manifest_entries(loop, client, inventory) csv_stream = download_csv(loop, client, keys_stream) records_stream = csv_to_records(loop, csv_stream, skip_incomplete=True) await to_kinto(loop, records_stream, kinto_client, skip_existing=True)
async def test_csv_to_records_keep_incomplete(self): output = inventory_to_records.csv_to_records(self.loop, self.stdin, skip_incomplete=False) records = [] async for r in output: records.append(r) assert len(records) == 2 assert records[1] == { 'data': { 'id': 'firefox_nightly_2017-06-16-03-02-07_56-0a1_win32_ach', 'download': { 'date': '2017-06-16T03:02:07Z', 'mimetype': 'application/zip', 'size': 45678, 'url': ('https://archive.mozilla.org/pub/firefox/nightly/' '2017/06/2017-06-16-03-02-07-mozilla-central-l10n/' 'firefox-56.0a1.ach.win32.zip') }, 'source': { 'product': 'firefox' }, 'target': { 'channel': 'nightly', 'locale': 'ach', 'platform': 'win32', 'os': 'win', 'version': '56.0a1' } } }
async def test_csv_to_records_continues_on_error(self): with mock.patch('buildhub.utils.guess_mimetype', side_effect=(ValueError, 'application/zip')): output = inventory_to_records.csv_to_records(self.loop, self.stdin, skip_incomplete=False) records = [] async for r in output: records.append(r) assert len(records) == 1 # Instead of 2 like the test above.
async def main(loop, inventories=INVENTORIES): """ Trigger to populate kinto with the last inventories. """ server_url = config('SERVER_URL', default='http://localhost:8888/v1') bucket = config('BUCKET', default='build-hub') collection = config('COLLECTION', default='releases') kinto_auth = tuple(config('AUTH', default='user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, bucket=bucket, collection=collection, retry=NB_RETRY_REQUEST) # Create bucket/collection and schemas. if INITIALIZE_SERVER: await initialize_kinto(loop, kinto_client, bucket, collection) min_last_modified = None # Convert the simple env var integer to a datetime.datetime instance. if MIN_AGE_LAST_MODIFIED_HOURS: assert MIN_AGE_LAST_MODIFIED_HOURS > 0, MIN_AGE_LAST_MODIFIED_HOURS min_last_modified = datetime.datetime.utcnow() - datetime.timedelta( hours=MIN_AGE_LAST_MODIFIED_HOURS) # Make it timezone aware (to UTC) min_last_modified = min_last_modified.replace( tzinfo=datetime.timezone.utc) # Fetch all existing records as a big dict from kinto existing = fetch_existing(kinto_client) # Download CSVs, deduce records and push to Kinto. session = aiobotocore.get_session(loop=loop) boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED) async with session.create_client('s3', region_name=REGION_NAME, config=boto_config) as client: for inventory in inventories: files_stream = list_manifest_entries(loop, client, inventory) csv_stream = download_csv(loop, client, files_stream) records_stream = csv_to_records( loop, csv_stream, skip_incomplete=True, min_last_modified=min_last_modified, ) await to_kinto_main(loop, records_stream, kinto_client, existing=existing, skip_existing=False)
async def test_csv_to_records(self): output = inventory_to_records.csv_to_records(self.loop, self.stdin) records = [] async for r in output: records.append(r) assert records == [{ 'data': { 'id': 'firefox_51-0_win64_fy-nl', 'build': { 'id': '20170118123726', 'date': '2017-01-18T12:37:26Z', 'number': 2, 'as': 'ml64.exe', 'cc': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/', 'src/vs2015u3/VC/bin/amd64/cl.EXE'], 'cxx': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/', 'src/vs2015u3/VC/bin/amd64/cl.EXE'], 'date': '2017-01-18T12:37:26Z', 'host': 'x86_64-pc-mingw32', 'ld': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/', 'src/vs2015u3/VC/bin/amd64/link.exe'], 'target': 'x86_64-pc-mingw32' }, 'download': { 'date': '2017-06-11T12:20:10Z', 'mimetype': 'application/msdos-windows', 'size': 67842, 'url': ('https://archive.mozilla.org/pub/firefox/releases/51.0/win64/' 'fy-NL/Firefox Setup 51.0.exe') }, 'source': { 'product': 'firefox', 'repository': 'https://hg.mozilla.org/releases/mozilla-release', 'revision': 'ea82b5e20cbbd103f8fa65f0df0386ee4135cc47', 'tree': 'releases/mozilla-release' }, 'target': { 'channel': 'release', 'locale': 'fy-NL', 'platform': 'win64', 'os': 'win', 'version': '51.0' } } }]
async def test_csv_to_records_ancient_entries_skipped(self): today = datetime.datetime.utcnow() recently = today - datetime.timedelta(hours=1) # Make it timezone aware (to UTC) recently = recently.replace(tzinfo=datetime.timezone.utc) output = inventory_to_records.csv_to_records( self.loop, self.stdin, min_last_modified=recently, cache_folder=self.cache_folder, ) records = [] async for r in output: records.append(r) assert len(records) == 0