async def main(loop, inventory):
"""
Trigger to populate kinto with the last inventories.
"""
server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1')
bucket = os.getenv('BUCKET', 'build-hub')
collection = os.getenv('COLLECTION', 'releases')
kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
bucket=bucket,
collection=collection,
retry=NB_RETRY_REQUEST)
# Create bucket/collection and schemas.
if INITIALIZE_SERVER:
await initialize_kinto(loop, kinto_client, bucket, collection)
# Download CSVs, deduce records and push to Kinto.
session = aiobotocore.get_session(loop=loop)
boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED)
async with session.create_client('s3',
region_name=REGION_NAME,
config=boto_config) as client:
keys_stream = list_manifest_entries(loop, client, inventory)
csv_stream = download_csv(loop, client, keys_stream)
records_stream = csv_to_records(loop, csv_stream, skip_incomplete=True)
await to_kinto(loop, records_stream, kinto_client, skip_existing=True)
def get_main_records():
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
return client.get_records(bucket=settings.KINTO_BUCKET_MAIN,
collection=settings.KINTO_COLLECTION)
def __init__(self, collection, review=True):
self.collection = collection
self.kinto_http_client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
self.review = review
self.collection_data = None
def delete_rejected_record(record_id):
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
client.delete_record(id=record_id,
bucket=settings.KINTO_BUCKET,
collection=settings.KINTO_COLLECTION)
def __init__(self):
# Kinto is the underlying implementation of Remote Settings. The client
# is basically a tiny abstraction on top of the requests library.
self.client = (kinto_http.Client(
server_url=rs_settings.URL,
auth=(rs_settings.USERNAME, rs_settings.PASSWORD),
retry=rs_settings.RETRY_REQUESTS,
) if rs_settings.URL else None)
def has_pending_review():
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
collection = client.get_collection(id=settings.KINTO_COLLECTION,
bucket=settings.KINTO_BUCKET)
return collection["data"]["status"] == KINTO_REVIEW_STATUS
def setUp(self):
self.logger.info("Vérifications Kinto")
client = kinto_http.Client(server_url=KINTO_SERVER,
auth=(KINTO_ADMIN_LOGIN,
KINTO_ADMIN_PASSWORD))
try:
info = client.server_info()
except ConnectionError as err:
raise KintoImporterError(
f"Connection au serveur Kinto impossible: {err}. Vérifiez la documentation pour paramétrer l'accès."
)
if "schema" not in info["capabilities"]:
raise KintoImporterError(
"Le serveur Kinto ne supporte pas la validation par schéma.")
else:
self.logger.success("Validation de schéma activée.")
if self.truncate:
if self.usePrompt and not prompt(
self.logger,
"Confimer la suppression et recréation de la collection existante ?",
"non"):
raise KintoImporterError("Commande annulée.")
self.logger.warn("Suppression de la collection Kinto existante...")
client.delete_collection(id=KINTO_COLLECTION, bucket=KINTO_BUCKET)
self.logger.success("La collection précédente a été supprimée.")
try:
coll = client.get_collection(id=KINTO_COLLECTION,
bucket=KINTO_BUCKET)
self.logger.success("La collection existe.")
except KintoException as err:
self.logger.warn("La collection n'existe pas, création")
try:
coll = client.create_collection(id=KINTO_COLLECTION,
data={"schema": self.schema},
bucket=KINTO_BUCKET)
self.logger.success("La collection a été crée.")
except KintoException as err:
raise KintoImporterError(
f"Impossible de créer la collection {KINTO_BUCKET}/{KINTO_COLLECTION}: {err}"
)
if "schema" not in coll["data"]:
self.logger.warn(
"La collection ne possède pas schéma de validation JSON.")
self.logger.info(
f"Ajout du schéma à la collection {KINTO_COLLECTION}.")
try:
patch = BasicPatch(data={"schema": self.schema})
client.patch_collection(id=KINTO_COLLECTION,
bucket=KINTO_BUCKET,
changes=patch)
self.logger.info(
"Le schéma de validation JSON a été ajouté à la collection."
)
except (KintoException, TypeError, KeyError, ValueError) as err:
raise KintoImporterError(
f"Impossible d'ajouter le schéma de validation à la collection {KINTO_COLLECTION}: {err}"
)
return client
def get_rejected_collection_data():
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
collection = client.get_collection(id=settings.KINTO_COLLECTION,
bucket=settings.KINTO_BUCKET)
if collection["data"]["status"] == KINTO_REJECTED_STATUS:
return collection["data"]
def __init__(self):
# Kinto is the underlying implementation of Remote Settings. The client
# is basically a tiny abstraction on top of the requests library.
self.client = (kinto_http.Client(
server_url=settings.REMOTE_SETTINGS_URL,
auth=(settings.REMOTE_SETTINGS_USERNAME,
settings.REMOTE_SETTINGS_PASSWORD),
bucket=settings.REMOTE_SETTINGS_BUCKET_ID,
collection=settings.REMOTE_SETTINGS_COLLECTION_ID,
retry=settings.REMOTE_SETTINGS_RETRY_REQUESTS,
) if settings.REMOTE_SETTINGS_URL else None)
async def main(loop, inventories=INVENTORIES):
"""
Trigger to populate kinto with the last inventories.
"""
server_url = config('SERVER_URL', default='http://localhost:8888/v1')
bucket = config('BUCKET', default='build-hub')
collection = config('COLLECTION', default='releases')
kinto_auth = tuple(config('AUTH', default='user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
bucket=bucket,
collection=collection,
retry=NB_RETRY_REQUEST)
# Create bucket/collection and schemas.
if INITIALIZE_SERVER:
await initialize_kinto(loop, kinto_client, bucket, collection)
min_last_modified = None
# Convert the simple env var integer to a datetime.datetime instance.
if MIN_AGE_LAST_MODIFIED_HOURS:
assert MIN_AGE_LAST_MODIFIED_HOURS > 0, MIN_AGE_LAST_MODIFIED_HOURS
min_last_modified = datetime.datetime.utcnow() - datetime.timedelta(
hours=MIN_AGE_LAST_MODIFIED_HOURS)
# Make it timezone aware (to UTC)
min_last_modified = min_last_modified.replace(
tzinfo=datetime.timezone.utc)
# Fetch all existing records as a big dict from kinto
existing = fetch_existing(kinto_client)
# Download CSVs, deduce records and push to Kinto.
session = aiobotocore.get_session(loop=loop)
boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED)
async with session.create_client('s3',
region_name=REGION_NAME,
config=boto_config) as client:
for inventory in inventories:
files_stream = list_manifest_entries(loop, client, inventory)
csv_stream = download_csv(loop, client, files_stream)
records_stream = csv_to_records(
loop,
csv_stream,
skip_incomplete=True,
min_last_modified=min_last_modified,
)
await to_kinto_main(loop,
records_stream,
kinto_client,
existing=existing,
skip_existing=False)
def push_to_kinto(data):
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
client.create_record(
data=data,
collection=settings.KINTO_COLLECTION,
bucket=settings.KINTO_BUCKET,
if_not_exists=True,
)
client.patch_collection(
id=settings.KINTO_COLLECTION,
data={"status": KINTO_REVIEW_STATUS},
bucket=settings.KINTO_BUCKET,
)
def get_rejected_record():
client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
main_records = client.get_records(bucket=settings.KINTO_BUCKET_MAIN,
collection=settings.KINTO_COLLECTION)
workspace_records = client.get_records(
bucket=settings.KINTO_BUCKET, collection=settings.KINTO_COLLECTION)
main_record_ids = [record["id"] for record in main_records]
workspace_record_ids = [record["id"] for record in workspace_records]
return list(set(workspace_record_ids) - set(main_record_ids))
def __init__(self, *args, **kwargs):
kwargs.setdefault("retry", config.REQUESTS_MAX_RETRIES)
kwargs.setdefault("timeout", config.REQUESTS_TIMEOUT_SECONDS)
kwargs.setdefault("headers", {
"User-Agent": USER_AGENT,
**config.DEFAULT_REQUEST_HEADERS
})
auth = kwargs.get("auth")
if auth is not None:
_type = None
if " " in auth:
# eg, "Bearer ghruhgrwyhg"
_type, auth = auth.split(" ", 1)
auth = (tuple(auth.split(":", 1)) if ":" in auth else
kinto_http.BearerTokenAuth(auth, type=_type))
kwargs["auth"] = auth
self._client = kinto_http.Client(*args, **kwargs)
def __init__(self, collection):
self.collection = collection
self.kinto_http_client = kinto_http.Client(
server_url=settings.KINTO_HOST,
auth=(settings.KINTO_USER, settings.KINTO_PASS),
)
requests.put(
urllib.parse.urljoin(os.environ["KINTO_HOST"],
f"/accounts/{user}"),
json={
"data": {
"password": passw
}
},
).content, )
create_user(ADMIN_USER, ADMIN_PASS)
create_user(REVIEW_USER, REVIEW_PASS)
create_user(EXPERIMENTER_USER, EXPERIMENTER_PASS)
client = kinto_http.Client(server_url=os.environ["KINTO_HOST"],
auth=(ADMIN_USER, ADMIN_PASS))
print(f">>>> Creating kinto bucket: {os.environ['KINTO_BUCKET']}")
print(
client.create_bucket(
id=os.environ["KINTO_BUCKET"],
permissions={"read": ["system.Everyone"]},
if_not_exists=True,
))
print(">>>> Creating kinto group: editors")
print(
client.create_group(
id=f"{os.environ['KINTO_COLLECTION']}-editors",
bucket=os.environ["KINTO_BUCKET"],
data={"members": [f"account:{os.environ['KINTO_USER']}"]},
"""
Load OpenStreetMap data on Kinto, and take advantage of a Algolia geo-index.
"""
import requests
import kinto_http
server = "http://localhost:8888/v1"
bucket = "restaurants"
collection = "pizzerias"
auth = ("user", "pass")
# Create the destination bucket and collection.
client = kinto_http.Client(server_url=server, auth=auth)
client.create_bucket(id=bucket, if_not_exists=True)
# Define the ElasticSearch mapping in the collection metadata.
collection_metadata = {"algolia:settings": {"attributesToIndex": ["name"]}}
# Let anonymous users read the records (online map demo).
collection_permissions = {"read": ["system.Everyone"]}
client.create_collection(id=collection,
bucket=bucket,
data=collection_metadata,
permissions=collection_permissions,
if_not_exists=True)
# Fetch OpenStreetMap data.
# This is a GeoJSON export of the following query on from http://overpass-turbo.eu
# area[name="Italia"];(node["cuisine"="pizza"](area););out;
pizzerias_export = (
"https://gist.githubusercontent.com/leplatrem/887e61efc1a7dfc7a68bcdf170d1ced9"
"/raw/c0507d874ebe84923d51b2db7990af465eb9cf66/export.geoson")
def test_load_into_kinto(self):
lambda_s3_event.lambda_handler(self.event, None)
rid = 'firefox_54-0_win64_fr'
client = kinto_http.Client(server_url=server)
record = client.get_record(bucket=bid, collection=cid, id=rid)['data']
record.pop('last_modified')
assert record == {
'id': 'firefox_54-0_win64_fr',
'source': {
'repository':
('https://hg.mozilla.org/releases/mozilla-release'),
'revision': 'e832ed037a3c23004be73178e546d240e57b6ee1',
'product': 'firefox',
'tree': 'releases/mozilla-release'
},
'download': {
'mimetype': 'application/msdos-windows',
'url': 'https://archive.mozilla.org/pub/firefox/releases/'
'54.0/win64/fr/Firefox Setup 54.0.exe',
'size': 51001024,
'date': '2017-08-08T17:06:52Z'
},
'target': {
'locale': 'fr',
'platform': 'win64',
'os': 'win',
'version': '54.0',
'channel': 'release'
},
'build': {
'as':
'ml64.exe',
'cc': ('c:/builds/moz2_slave/m-rel-w64-00000000000000000000/'
'build/src/vs2015u3/VC/bin/amd64/cl.exe'),
'cxx': ('c:/builds/moz2_slave/m-rel-w64-00000000000000000000/'
'build/src/vs2015u3/VC/bin/amd64/cl.exe'),
'date':
'2017-06-08T10:58:25Z',
'host':
'x86_64-pc-mingw32',
'id':
'20170608105825',
'number':
3,
'target':
'x86_64-pc-mingw32'
}
}
rid = 'firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us'
record = client.get_record(bucket=bid, collection=cid, id=rid)['data']
record.pop('last_modified')
assert record == {
'build': {
'as':
'$(CC)',
'cc': ('/usr/bin/ccache '
'/builds/worker/workspace/build/src/gcc/bin/gcc -m32 '
'-march=pentium-m -std=gnu99'),
'cxx': ('/usr/bin/ccache '
'/builds/worker/workspace/build/src/gcc/bin/g++ -m32 '
'-march=pentium-m -std=gnu++11'),
'date':
'2017-10-29T22:01:12Z',
'host':
'i686-pc-linux-gnu',
'id':
'20171029220112',
'target':
'i686-pc-linux-gnu',
},
'download': {
'date':
'2017-10-29T17:06:52Z',
'mimetype':
'application/x-bzip2',
'size':
51001024,
'url':
('https://archive.mozilla.org/pub/firefox/nightly/2017/10/'
'2017-10-29-22-01-12-mozilla-central/firefox-58.0a1.'
'en-US.linux-i686.tar.bz2')
},
'id':
('firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us'),
'source': {
'product': 'firefox',
'repository': 'https://hg.mozilla.org/mozilla-central',
'revision': 'd3910b7628b8066d3f30d58b17b5824b05768854',
'tree': 'mozilla-central'
},
'target': {
'channel': 'nightly',
'locale': 'en-US',
'os': 'linux',
'platform': 'linux-i686',
'version': '58.0a1'
}
}
async def main(loop, event):
"""
Trigger when S3 event kicks in.
http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
"""
server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1')
bucket = os.getenv('BUCKET', 'build-hub')
collection = os.getenv('COLLECTION', 'releases')
kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
retry=NB_RETRY_REQUEST)
records = []
for record in event['Records']:
if record.get('EventSource') == 'aws:sns':
records.extend(json.loads(record['Sns']['Message'])['Records'])
else:
records.append(record)
async with aiohttp.ClientSession(loop=loop) as session:
for event_record in records:
records_to_create = []
# Use event time as archive publication.
event_time = datetime.datetime.strptime(event_record['eventTime'],
'%Y-%m-%dT%H:%M:%S.%fZ')
event_time = event_time.strftime(utils.DATETIME_FORMAT)
key = event_record['s3']['object']['key']
filesize = event_record['s3']['object']['size']
url = utils.ARCHIVE_URL + key
logger.debug("Event file {}".format(url))
try:
product = key.split('/')[1] # /pub/thunderbird/nightly/...
except IndexError:
continue # e.g. https://archive.mozilla.org/favicon.ico
if product not in utils.ALL_PRODUCTS:
logger.info('Skip product {}'.format(product))
continue
# Release / Nightly / RC archive.
if utils.is_build_url(product, url):
logger.info('Processing {} archive: {}'.format(product, key))
record = utils.record_from_url(url)
# Use S3 event infos for the archive.
record['download']['size'] = filesize
record['download']['date'] = event_time
# Fetch release metadata.
await scan_candidates(session, product)
logger.debug("Fetch record metadata")
metadata = await fetch_metadata(session, record)
# If JSON metadata not available, archive will be handled when JSON
# is delivered.
if metadata is None:
logger.info('JSON metadata not available {}'.format(
record['id']))
continue
# Merge obtained metadata.
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
# RC metadata
elif utils.is_rc_build_metadata(product, url):
logger.info('Processing {} RC metadata: {}'.format(
product, key))
# pub/firefox/candidates/55.0b12-candidates/build1/mac/en-US/
# firefox-55.0b12.json
logger.debug("Fetch new metadata")
metadata = await fetch_json(session, url)
metadata['buildnumber'] = int(
re.search('/build(\d+)/', url).group(1))
# Check if localized languages are here (including en-US archive).
l10n_parent_url = re.sub('en-US/.+$', '', url)
l10n_folders, _ = await fetch_listing(session, l10n_parent_url)
for locale in l10n_folders:
_, files = await fetch_listing(session,
l10n_parent_url + locale)
for f in files:
rc_url = l10n_parent_url + locale + f['name']
if utils.is_build_url(product, rc_url):
record = utils.record_from_url(rc_url)
record['download']['size'] = f['size']
record['download']['date'] = f['last_modified']
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
# Theorically release should never be there yet :)
# And repacks like EME-free/sha1 don't seem to be published in RC.
# Nightly metadata
# pub/firefox/nightly/2017/08/2017-08-08-11-40-32-mozilla-central/
# firefox-57.0a1.en-US.linux-i686.json
# -l10n/...
elif utils.is_nightly_build_metadata(product, url):
logger.info('Processing {} nightly metadata: {}'.format(
product, key))
logger.debug("Fetch new nightly metadata")
metadata = await fetch_json(session, url)
platform = metadata['moz_pkg_platform']
# Check if english version is here.
parent_url = re.sub('/[^/]+$', '/', url)
logger.debug("Fetch parent listing {}".format(parent_url))
_, files = await fetch_listing(session, parent_url)
for f in files:
if ('.' + platform + '.') not in f['name']:
# metadata are by platform.
continue
en_nightly_url = parent_url + f['name']
if utils.is_build_url(product, en_nightly_url):
record = utils.record_from_url(en_nightly_url)
record['download']['size'] = f['size']
record['download']['date'] = f['last_modified']
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
break # Only one file for english.
# Check also localized versions.
l10n_folder_url = re.sub('-mozilla-central([^/]*)/([^/]+)$',
'-mozilla-central\\1-l10n/', url)
logger.debug("Fetch l10n listing {}".format(l10n_folder_url))
try:
_, files = await fetch_listing(session, l10n_folder_url)
except ValueError:
files = [] # No -l10/ folder published yet.
for f in files:
if ('.' + platform +
'.') not in f['name'] and product != 'mobile':
# metadata are by platform.
# (mobile platforms are contained by folder)
continue
nightly_url = l10n_folder_url + f['name']
if utils.is_build_url(product, nightly_url):
record = utils.record_from_url(nightly_url)
record['download']['size'] = f['size']
record['download']['date'] = f['last_modified']
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
else:
logger.info('Ignored {}'.format(key))
logger.debug("{} records to create.".format(
len(records_to_create)))
for record in records_to_create:
# Check that fields values look OK.
utils.check_record(record)
# Push result to Kinto.
kinto_client.create_record(data=record,
bucket=bucket,
collection=collection,
if_not_exists=True)
logger.info('Created {}'.format(record['id']))
async def main(loop, event):
"""
Trigger when S3 event kicks in.
http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
"""
server_url = config('SERVER_URL', default='http://localhost:8888/v1')
bucket = config('BUCKET', default='build-hub')
collection = config('COLLECTION', default='releases')
kinto_auth = tuple(config('AUTH', 'user:pass').split(':'))
kinto_client = kinto_http.Client(server_url=server_url,
auth=kinto_auth,
retry=NB_RETRY_REQUEST)
records = []
for record in event['Records']:
if record.get('EventSource') == 'aws:sns':
records.extend(json.loads(record['Sns']['Message'])['Records'])
else:
records.append(record)
async with aiohttp.ClientSession(loop=loop) as session:
for event_record in records:
metrics.incr('s3_event_event')
records_to_create = []
# Use event time as archive publication.
event_time = ciso8601.parse_datetime(event_record['eventTime'])
event_time = event_time.strftime(utils.DATETIME_FORMAT)
key = event_record['s3']['object']['key']
filesize = event_record['s3']['object']['size']
url = utils.key_to_archive_url(key)
logger.debug("Event file {}".format(url))
try:
product = key.split('/')[1] # /pub/thunderbird/nightly/...
except IndexError:
continue # e.g. https://archive.mozilla.org/favicon.ico
if product not in utils.ALL_PRODUCTS:
logger.info('Skip product {}'.format(product))
continue
# Release / Nightly / RC archive.
if utils.is_build_url(product, url):
logger.info('Processing {} archive: {}'.format(product, key))
record = utils.record_from_url(url)
# Use S3 event infos for the archive.
record['download']['size'] = filesize
record['download']['date'] = event_time
# Fetch release metadata.
await scan_candidates(session, product)
logger.debug("Fetch record metadata")
# metadata = await fetch_metadata(session, record)
metadata = await fetch_metadata(session, record)
# If JSON metadata not available, archive will be
# handled when JSON is delivered.
if metadata is None:
logger.info(f"JSON metadata not available {record['id']}")
continue
# Merge obtained metadata.
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
# RC metadata
elif utils.is_rc_build_metadata(product, url):
logger.info(f'Processing {product} RC metadata: {key}')
# pub/firefox/candidates/55.0b12-candidates/build1/mac/en-US/
# firefox-55.0b12.json
logger.debug("Fetch new metadata")
# It has been known to happen that right after an S3 Event
# there's a slight delay to the metadata json file being
# available. If that's the case we want to retry in a couple
# of seconds to see if it's available on the next backoff
# attempt.
metadata = await fetch_json(session,
url,
retry_on_notfound=True)
metadata['buildnumber'] = int(
re.search('/build(\d+)/', url).group(1))
# We just received the metadata file. Lookup if the associated
# archives are here too.
archives = []
if 'multi' in url:
# For multi we just check the associated archive
# is here already.
parent_folder = re.sub('multi/.+$', 'multi/', url)
_, files = await fetch_listing(session,
parent_folder,
retry_on_notfound=True)
for f in files:
rc_url = parent_folder + f['name']
if utils.is_build_url(product, rc_url):
archives.append(
(rc_url, f['size'], f['last_modified']))
else:
# For en-US it's different, it applies to every
# localized archives.
# Check if they are here by listing the parent folder
# (including en-US archive).
l10n_parent_url = re.sub('en-US/.+$', '', url)
l10n_folders, _ = await fetch_listing(
session,
l10n_parent_url,
retry_on_notfound=True,
)
for locale in l10n_folders:
_, files = await fetch_listing(
session,
l10n_parent_url + locale,
retry_on_notfound=True,
)
for f in files:
rc_url = l10n_parent_url + locale + f['name']
if utils.is_build_url(product, rc_url):
archives.append((
rc_url,
f['size'],
f['last_modified'],
))
for rc_url, size, last_modified in archives:
record = utils.record_from_url(rc_url)
record['download']['size'] = size
record['download']['date'] = last_modified
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
# Theorically release should never be there yet :)
# And repacks like EME-free/sha1 don't seem to be
# published in RC.
# Nightly metadata
# pub/firefox/nightly/2017/08/2017-08-08-11-40-32-mozilla-central/
# firefox-57.0a1.en-US.linux-i686.json
# -l10n/...
elif utils.is_nightly_build_metadata(product, url):
logger.info(f'Processing {product} nightly metadata: {key}')
logger.debug("Fetch new nightly metadata")
# See comment above about the exceptional need of
# setting retry_on_notfound here.
metadata = await fetch_json(session,
url,
retry_on_notfound=True)
platform = metadata['moz_pkg_platform']
# Check if english version is here.
parent_url = re.sub('/[^/]+$', '/', url)
logger.debug("Fetch parent listing {}".format(parent_url))
_, files = await fetch_listing(session, parent_url)
for f in files:
if ('.' + platform + '.') not in f['name']:
# metadata are by platform.
continue
en_nightly_url = parent_url + f['name']
if utils.is_build_url(product, en_nightly_url):
record = utils.record_from_url(en_nightly_url)
record['download']['size'] = f['size']
record['download']['date'] = f['last_modified']
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
break # Only one file for english.
# Check also localized versions.
l10n_folder_url = re.sub('-mozilla-central([^/]*)/([^/]+)$',
'-mozilla-central\\1-l10n/', url)
logger.debug("Fetch l10n listing {}".format(l10n_folder_url))
try:
_, files = await fetch_listing(
session,
l10n_folder_url,
retry_on_notfound=True,
)
except ValueError:
files = [] # No -l10/ folder published yet.
for f in files:
if (('.' + platform + '.') not in f['name']
and product != 'mobile'):
# metadata are by platform.
# (mobile platforms are contained by folder)
continue
nightly_url = l10n_folder_url + f['name']
if utils.is_build_url(product, nightly_url):
record = utils.record_from_url(nightly_url)
record['download']['size'] = f['size']
record['download']['date'] = f['last_modified']
record = utils.merge_metadata(record, metadata)
records_to_create.append(record)
else:
logger.info('Ignored {}'.format(key))
logger.debug(f"{len(records_to_create)} records to create.")
with metrics.timer('s3_event_records_to_create'):
for record in records_to_create:
# Check that fields values look OK.
utils.check_record(record)
# Push result to Kinto.
kinto_client.create_record(data=record,
bucket=bucket,
collection=collection,
if_not_exists=True)
logger.info('Created {}'.format(record['id']))
metrics.incr('s3_event_record_created')
print(
requests.put(
urllib.parse.urljoin(KINTO_HOST, f"/accounts/{user}"),
json={
"data": {
"password": passw
}
},
).content, )
create_user(ADMIN_USER, ADMIN_PASS)
create_user(REVIEW_USER, REVIEW_PASS)
create_user(EXPERIMENTER_USER, EXPERIMENTER_PASS)
client = kinto_http.Client(server_url=KINTO_HOST,
auth=(ADMIN_USER, ADMIN_PASS))
print(f">>>> Creating kinto bucket: {KINTO_BUCKET}")
print(
client.create_bucket(
id=KINTO_BUCKET,
permissions={"read": ["system.Everyone"]},
if_not_exists=True,
))
for collection in [
KINTO_COLLECTION_NIMBUS_DESKTOP,
KINTO_COLLECTION_NIMBUS_MOBILE,
]:
print(">>>> Creating kinto group: editors")
print(
