async def main(loop, inventory):
"""
Trigger to populate kinto with the last inventories.
"""
server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1')
bucket = os.getenv('BUCKET', 'build-hub')
collection = os.getenv('COLLECTION', 'releases')
kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
bucket=bucket,
collection=collection,
retry=NB_RETRY_REQUEST)
# Create bucket/collection and schemas.
if INITIALIZE_SERVER:
await initialize_kinto(loop, kinto_client, bucket, collection)
# Download CSVs, deduce records and push to Kinto.
session = aiobotocore.get_session(loop=loop)
boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED)
async with session.create_client('s3',
region_name=REGION_NAME,
config=boto_config) as client:
keys_stream = list_manifest_entries(loop, client, inventory)
csv_stream = download_csv(loop, client, keys_stream)
records_stream = csv_to_records(loop, csv_stream, skip_incomplete=True)
await to_kinto(loop, records_stream, kinto_client, skip_existing=True)
async def test_csv_to_records_keep_incomplete(self):
output = inventory_to_records.csv_to_records(self.loop, self.stdin,
skip_incomplete=False)
records = []
async for r in output:
records.append(r)
assert len(records) == 2
assert records[1] == {
'data': {
'id': 'firefox_nightly_2017-06-16-03-02-07_56-0a1_win32_ach',
'download': {
'date': '2017-06-16T03:02:07Z',
'mimetype': 'application/zip',
'size': 45678,
'url': ('https://archive.mozilla.org/pub/firefox/nightly/'
'2017/06/2017-06-16-03-02-07-mozilla-central-l10n/'
'firefox-56.0a1.ach.win32.zip')
},
'source': {
'product': 'firefox'
},
'target': {
'channel': 'nightly',
'locale': 'ach',
'platform': 'win32',
'os': 'win',
'version': '56.0a1'
}
}
}
async def test_csv_to_records_continues_on_error(self):
with mock.patch('buildhub.utils.guess_mimetype',
side_effect=(ValueError, 'application/zip')):
output = inventory_to_records.csv_to_records(self.loop, self.stdin,
skip_incomplete=False)
records = []
async for r in output:
records.append(r)
assert len(records) == 1 # Instead of 2 like the test above.
async def main(loop, inventories=INVENTORIES):
"""
Trigger to populate kinto with the last inventories.
"""
server_url = config('SERVER_URL', default='http://localhost:8888/v1')
bucket = config('BUCKET', default='build-hub')
collection = config('COLLECTION', default='releases')
kinto_auth = tuple(config('AUTH', default='user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
bucket=bucket,
collection=collection,
retry=NB_RETRY_REQUEST)
# Create bucket/collection and schemas.
if INITIALIZE_SERVER:
await initialize_kinto(loop, kinto_client, bucket, collection)
min_last_modified = None
# Convert the simple env var integer to a datetime.datetime instance.
if MIN_AGE_LAST_MODIFIED_HOURS:
assert MIN_AGE_LAST_MODIFIED_HOURS > 0, MIN_AGE_LAST_MODIFIED_HOURS
min_last_modified = datetime.datetime.utcnow() - datetime.timedelta(
hours=MIN_AGE_LAST_MODIFIED_HOURS)
# Make it timezone aware (to UTC)
min_last_modified = min_last_modified.replace(
tzinfo=datetime.timezone.utc)
# Fetch all existing records as a big dict from kinto
existing = fetch_existing(kinto_client)
# Download CSVs, deduce records and push to Kinto.
session = aiobotocore.get_session(loop=loop)
boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED)
async with session.create_client('s3',
region_name=REGION_NAME,
config=boto_config) as client:
for inventory in inventories:
files_stream = list_manifest_entries(loop, client, inventory)
csv_stream = download_csv(loop, client, files_stream)
records_stream = csv_to_records(
loop,
csv_stream,
skip_incomplete=True,
min_last_modified=min_last_modified,
)
await to_kinto_main(loop,
records_stream,
kinto_client,
existing=existing,
skip_existing=False)
async def test_csv_to_records(self):
output = inventory_to_records.csv_to_records(self.loop, self.stdin)
records = []
async for r in output:
records.append(r)
assert records == [{
'data': {
'id': 'firefox_51-0_win64_fy-nl',
'build': {
'id': '20170118123726',
'date': '2017-01-18T12:37:26Z',
'number': 2,
'as': 'ml64.exe',
'cc': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/',
'src/vs2015u3/VC/bin/amd64/cl.EXE'],
'cxx': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/',
'src/vs2015u3/VC/bin/amd64/cl.EXE'],
'date': '2017-01-18T12:37:26Z',
'host': 'x86_64-pc-mingw32',
'ld': ['c:/builds/moz2_slave/m-rel-w64-00000000000000000000/build/',
'src/vs2015u3/VC/bin/amd64/link.exe'],
'target': 'x86_64-pc-mingw32'
},
'download': {
'date': '2017-06-11T12:20:10Z',
'mimetype': 'application/msdos-windows',
'size': 67842,
'url': ('https://archive.mozilla.org/pub/firefox/releases/51.0/win64/'
'fy-NL/Firefox Setup 51.0.exe')
},
'source': {
'product': 'firefox',
'repository': 'https://hg.mozilla.org/releases/mozilla-release',
'revision': 'ea82b5e20cbbd103f8fa65f0df0386ee4135cc47',
'tree': 'releases/mozilla-release'
},
'target': {
'channel': 'release',
'locale': 'fy-NL',
'platform': 'win64',
'os': 'win',
'version': '51.0'
}
}
}]
async def test_csv_to_records_ancient_entries_skipped(self):
today = datetime.datetime.utcnow()
recently = today - datetime.timedelta(hours=1)
# Make it timezone aware (to UTC)
recently = recently.replace(tzinfo=datetime.timezone.utc)
output = inventory_to_records.csv_to_records(
self.loop,
self.stdin,
min_last_modified=recently,
cache_folder=self.cache_folder,
)
records = []
async for r in output:
records.append(r)
assert len(records) == 0
