async def main(loop, inventory): """ Trigger to populate kinto with the last inventories. """ server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1') bucket = os.getenv('BUCKET', 'build-hub') collection = os.getenv('COLLECTION', 'releases') kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, bucket=bucket, collection=collection, retry=NB_RETRY_REQUEST) # Create bucket/collection and schemas. if INITIALIZE_SERVER: await initialize_kinto(loop, kinto_client, bucket, collection) # Download CSVs, deduce records and push to Kinto. session = aiobotocore.get_session(loop=loop) boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED) async with session.create_client('s3', region_name=REGION_NAME, config=boto_config) as client: keys_stream = list_manifest_entries(loop, client, inventory) csv_stream = download_csv(loop, client, keys_stream) records_stream = csv_to_records(loop, csv_stream, skip_incomplete=True) await to_kinto(loop, records_stream, kinto_client, skip_existing=True)
def get_main_records(): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) return client.get_records(bucket=settings.KINTO_BUCKET_MAIN, collection=settings.KINTO_COLLECTION)
def __init__(self, collection, review=True): self.collection = collection self.kinto_http_client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) self.review = review self.collection_data = None
def delete_rejected_record(record_id): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) client.delete_record(id=record_id, bucket=settings.KINTO_BUCKET, collection=settings.KINTO_COLLECTION)
def __init__(self): # Kinto is the underlying implementation of Remote Settings. The client # is basically a tiny abstraction on top of the requests library. self.client = (kinto_http.Client( server_url=rs_settings.URL, auth=(rs_settings.USERNAME, rs_settings.PASSWORD), retry=rs_settings.RETRY_REQUESTS, ) if rs_settings.URL else None)
def has_pending_review(): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) collection = client.get_collection(id=settings.KINTO_COLLECTION, bucket=settings.KINTO_BUCKET) return collection["data"]["status"] == KINTO_REVIEW_STATUS
def setUp(self): self.logger.info("Vérifications Kinto") client = kinto_http.Client(server_url=KINTO_SERVER, auth=(KINTO_ADMIN_LOGIN, KINTO_ADMIN_PASSWORD)) try: info = client.server_info() except ConnectionError as err: raise KintoImporterError( f"Connection au serveur Kinto impossible: {err}. Vérifiez la documentation pour paramétrer l'accès." ) if "schema" not in info["capabilities"]: raise KintoImporterError( "Le serveur Kinto ne supporte pas la validation par schéma.") else: self.logger.success("Validation de schéma activée.") if self.truncate: if self.usePrompt and not prompt( self.logger, "Confimer la suppression et recréation de la collection existante ?", "non"): raise KintoImporterError("Commande annulée.") self.logger.warn("Suppression de la collection Kinto existante...") client.delete_collection(id=KINTO_COLLECTION, bucket=KINTO_BUCKET) self.logger.success("La collection précédente a été supprimée.") try: coll = client.get_collection(id=KINTO_COLLECTION, bucket=KINTO_BUCKET) self.logger.success("La collection existe.") except KintoException as err: self.logger.warn("La collection n'existe pas, création") try: coll = client.create_collection(id=KINTO_COLLECTION, data={"schema": self.schema}, bucket=KINTO_BUCKET) self.logger.success("La collection a été crée.") except KintoException as err: raise KintoImporterError( f"Impossible de créer la collection {KINTO_BUCKET}/{KINTO_COLLECTION}: {err}" ) if "schema" not in coll["data"]: self.logger.warn( "La collection ne possède pas schéma de validation JSON.") self.logger.info( f"Ajout du schéma à la collection {KINTO_COLLECTION}.") try: patch = BasicPatch(data={"schema": self.schema}) client.patch_collection(id=KINTO_COLLECTION, bucket=KINTO_BUCKET, changes=patch) self.logger.info( "Le schéma de validation JSON a été ajouté à la collection." ) except (KintoException, TypeError, KeyError, ValueError) as err: raise KintoImporterError( f"Impossible d'ajouter le schéma de validation à la collection {KINTO_COLLECTION}: {err}" ) return client
def get_rejected_collection_data(): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) collection = client.get_collection(id=settings.KINTO_COLLECTION, bucket=settings.KINTO_BUCKET) if collection["data"]["status"] == KINTO_REJECTED_STATUS: return collection["data"]
def __init__(self): # Kinto is the underlying implementation of Remote Settings. The client # is basically a tiny abstraction on top of the requests library. self.client = (kinto_http.Client( server_url=settings.REMOTE_SETTINGS_URL, auth=(settings.REMOTE_SETTINGS_USERNAME, settings.REMOTE_SETTINGS_PASSWORD), bucket=settings.REMOTE_SETTINGS_BUCKET_ID, collection=settings.REMOTE_SETTINGS_COLLECTION_ID, retry=settings.REMOTE_SETTINGS_RETRY_REQUESTS, ) if settings.REMOTE_SETTINGS_URL else None)
async def main(loop, inventories=INVENTORIES): """ Trigger to populate kinto with the last inventories. """ server_url = config('SERVER_URL', default='http://localhost:8888/v1') bucket = config('BUCKET', default='build-hub') collection = config('COLLECTION', default='releases') kinto_auth = tuple(config('AUTH', default='user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, bucket=bucket, collection=collection, retry=NB_RETRY_REQUEST) # Create bucket/collection and schemas. if INITIALIZE_SERVER: await initialize_kinto(loop, kinto_client, bucket, collection) min_last_modified = None # Convert the simple env var integer to a datetime.datetime instance. if MIN_AGE_LAST_MODIFIED_HOURS: assert MIN_AGE_LAST_MODIFIED_HOURS > 0, MIN_AGE_LAST_MODIFIED_HOURS min_last_modified = datetime.datetime.utcnow() - datetime.timedelta( hours=MIN_AGE_LAST_MODIFIED_HOURS) # Make it timezone aware (to UTC) min_last_modified = min_last_modified.replace( tzinfo=datetime.timezone.utc) # Fetch all existing records as a big dict from kinto existing = fetch_existing(kinto_client) # Download CSVs, deduce records and push to Kinto. session = aiobotocore.get_session(loop=loop) boto_config = botocore.config.Config(signature_version=botocore.UNSIGNED) async with session.create_client('s3', region_name=REGION_NAME, config=boto_config) as client: for inventory in inventories: files_stream = list_manifest_entries(loop, client, inventory) csv_stream = download_csv(loop, client, files_stream) records_stream = csv_to_records( loop, csv_stream, skip_incomplete=True, min_last_modified=min_last_modified, ) await to_kinto_main(loop, records_stream, kinto_client, existing=existing, skip_existing=False)
def push_to_kinto(data): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) client.create_record( data=data, collection=settings.KINTO_COLLECTION, bucket=settings.KINTO_BUCKET, if_not_exists=True, ) client.patch_collection( id=settings.KINTO_COLLECTION, data={"status": KINTO_REVIEW_STATUS}, bucket=settings.KINTO_BUCKET, )
def get_rejected_record(): client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), ) main_records = client.get_records(bucket=settings.KINTO_BUCKET_MAIN, collection=settings.KINTO_COLLECTION) workspace_records = client.get_records( bucket=settings.KINTO_BUCKET, collection=settings.KINTO_COLLECTION) main_record_ids = [record["id"] for record in main_records] workspace_record_ids = [record["id"] for record in workspace_records] return list(set(workspace_record_ids) - set(main_record_ids))
def __init__(self, *args, **kwargs): kwargs.setdefault("retry", config.REQUESTS_MAX_RETRIES) kwargs.setdefault("timeout", config.REQUESTS_TIMEOUT_SECONDS) kwargs.setdefault("headers", { "User-Agent": USER_AGENT, **config.DEFAULT_REQUEST_HEADERS }) auth = kwargs.get("auth") if auth is not None: _type = None if " " in auth: # eg, "Bearer ghruhgrwyhg" _type, auth = auth.split(" ", 1) auth = (tuple(auth.split(":", 1)) if ":" in auth else kinto_http.BearerTokenAuth(auth, type=_type)) kwargs["auth"] = auth self._client = kinto_http.Client(*args, **kwargs)
def __init__(self, collection): self.collection = collection self.kinto_http_client = kinto_http.Client( server_url=settings.KINTO_HOST, auth=(settings.KINTO_USER, settings.KINTO_PASS), )
requests.put( urllib.parse.urljoin(os.environ["KINTO_HOST"], f"/accounts/{user}"), json={ "data": { "password": passw } }, ).content, ) create_user(ADMIN_USER, ADMIN_PASS) create_user(REVIEW_USER, REVIEW_PASS) create_user(EXPERIMENTER_USER, EXPERIMENTER_PASS) client = kinto_http.Client(server_url=os.environ["KINTO_HOST"], auth=(ADMIN_USER, ADMIN_PASS)) print(f">>>> Creating kinto bucket: {os.environ['KINTO_BUCKET']}") print( client.create_bucket( id=os.environ["KINTO_BUCKET"], permissions={"read": ["system.Everyone"]}, if_not_exists=True, )) print(">>>> Creating kinto group: editors") print( client.create_group( id=f"{os.environ['KINTO_COLLECTION']}-editors", bucket=os.environ["KINTO_BUCKET"], data={"members": [f"account:{os.environ['KINTO_USER']}"]},
""" Load OpenStreetMap data on Kinto, and take advantage of a Algolia geo-index. """ import requests import kinto_http server = "http://localhost:8888/v1" bucket = "restaurants" collection = "pizzerias" auth = ("user", "pass") # Create the destination bucket and collection. client = kinto_http.Client(server_url=server, auth=auth) client.create_bucket(id=bucket, if_not_exists=True) # Define the ElasticSearch mapping in the collection metadata. collection_metadata = {"algolia:settings": {"attributesToIndex": ["name"]}} # Let anonymous users read the records (online map demo). collection_permissions = {"read": ["system.Everyone"]} client.create_collection(id=collection, bucket=bucket, data=collection_metadata, permissions=collection_permissions, if_not_exists=True) # Fetch OpenStreetMap data. # This is a GeoJSON export of the following query on from http://overpass-turbo.eu # area[name="Italia"];(node["cuisine"="pizza"](area););out; pizzerias_export = ( "https://gist.githubusercontent.com/leplatrem/887e61efc1a7dfc7a68bcdf170d1ced9" "/raw/c0507d874ebe84923d51b2db7990af465eb9cf66/export.geoson")
def test_load_into_kinto(self): lambda_s3_event.lambda_handler(self.event, None) rid = 'firefox_54-0_win64_fr' client = kinto_http.Client(server_url=server) record = client.get_record(bucket=bid, collection=cid, id=rid)['data'] record.pop('last_modified') assert record == { 'id': 'firefox_54-0_win64_fr', 'source': { 'repository': ('https://hg.mozilla.org/releases/mozilla-release'), 'revision': 'e832ed037a3c23004be73178e546d240e57b6ee1', 'product': 'firefox', 'tree': 'releases/mozilla-release' }, 'download': { 'mimetype': 'application/msdos-windows', 'url': 'https://archive.mozilla.org/pub/firefox/releases/' '54.0/win64/fr/Firefox Setup 54.0.exe', 'size': 51001024, 'date': '2017-08-08T17:06:52Z' }, 'target': { 'locale': 'fr', 'platform': 'win64', 'os': 'win', 'version': '54.0', 'channel': 'release' }, 'build': { 'as': 'ml64.exe', 'cc': ('c:/builds/moz2_slave/m-rel-w64-00000000000000000000/' 'build/src/vs2015u3/VC/bin/amd64/cl.exe'), 'cxx': ('c:/builds/moz2_slave/m-rel-w64-00000000000000000000/' 'build/src/vs2015u3/VC/bin/amd64/cl.exe'), 'date': '2017-06-08T10:58:25Z', 'host': 'x86_64-pc-mingw32', 'id': '20170608105825', 'number': 3, 'target': 'x86_64-pc-mingw32' } } rid = 'firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us' record = client.get_record(bucket=bid, collection=cid, id=rid)['data'] record.pop('last_modified') assert record == { 'build': { 'as': '$(CC)', 'cc': ('/usr/bin/ccache ' '/builds/worker/workspace/build/src/gcc/bin/gcc -m32 ' '-march=pentium-m -std=gnu99'), 'cxx': ('/usr/bin/ccache ' '/builds/worker/workspace/build/src/gcc/bin/g++ -m32 ' '-march=pentium-m -std=gnu++11'), 'date': '2017-10-29T22:01:12Z', 'host': 'i686-pc-linux-gnu', 'id': '20171029220112', 'target': 'i686-pc-linux-gnu', }, 'download': { 'date': '2017-10-29T17:06:52Z', 'mimetype': 'application/x-bzip2', 'size': 51001024, 'url': ('https://archive.mozilla.org/pub/firefox/nightly/2017/10/' '2017-10-29-22-01-12-mozilla-central/firefox-58.0a1.' 'en-US.linux-i686.tar.bz2') }, 'id': ('firefox_nightly_2017-10-29-22-01-12_58-0a1_linux-i686_en-us'), 'source': { 'product': 'firefox', 'repository': 'https://hg.mozilla.org/mozilla-central', 'revision': 'd3910b7628b8066d3f30d58b17b5824b05768854', 'tree': 'mozilla-central' }, 'target': { 'channel': 'nightly', 'locale': 'en-US', 'os': 'linux', 'platform': 'linux-i686', 'version': '58.0a1' } }
async def main(loop, event): """ Trigger when S3 event kicks in. http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html """ server_url = os.getenv('SERVER_URL', 'http://localhost:8888/v1') bucket = os.getenv('BUCKET', 'build-hub') collection = os.getenv('COLLECTION', 'releases') kinto_auth = tuple(os.getenv('AUTH', 'user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, retry=NB_RETRY_REQUEST) records = [] for record in event['Records']: if record.get('EventSource') == 'aws:sns': records.extend(json.loads(record['Sns']['Message'])['Records']) else: records.append(record) async with aiohttp.ClientSession(loop=loop) as session: for event_record in records: records_to_create = [] # Use event time as archive publication. event_time = datetime.datetime.strptime(event_record['eventTime'], '%Y-%m-%dT%H:%M:%S.%fZ') event_time = event_time.strftime(utils.DATETIME_FORMAT) key = event_record['s3']['object']['key'] filesize = event_record['s3']['object']['size'] url = utils.ARCHIVE_URL + key logger.debug("Event file {}".format(url)) try: product = key.split('/')[1] # /pub/thunderbird/nightly/... except IndexError: continue # e.g. https://archive.mozilla.org/favicon.ico if product not in utils.ALL_PRODUCTS: logger.info('Skip product {}'.format(product)) continue # Release / Nightly / RC archive. if utils.is_build_url(product, url): logger.info('Processing {} archive: {}'.format(product, key)) record = utils.record_from_url(url) # Use S3 event infos for the archive. record['download']['size'] = filesize record['download']['date'] = event_time # Fetch release metadata. await scan_candidates(session, product) logger.debug("Fetch record metadata") metadata = await fetch_metadata(session, record) # If JSON metadata not available, archive will be handled when JSON # is delivered. if metadata is None: logger.info('JSON metadata not available {}'.format( record['id'])) continue # Merge obtained metadata. record = utils.merge_metadata(record, metadata) records_to_create.append(record) # RC metadata elif utils.is_rc_build_metadata(product, url): logger.info('Processing {} RC metadata: {}'.format( product, key)) # pub/firefox/candidates/55.0b12-candidates/build1/mac/en-US/ # firefox-55.0b12.json logger.debug("Fetch new metadata") metadata = await fetch_json(session, url) metadata['buildnumber'] = int( re.search('/build(\d+)/', url).group(1)) # Check if localized languages are here (including en-US archive). l10n_parent_url = re.sub('en-US/.+$', '', url) l10n_folders, _ = await fetch_listing(session, l10n_parent_url) for locale in l10n_folders: _, files = await fetch_listing(session, l10n_parent_url + locale) for f in files: rc_url = l10n_parent_url + locale + f['name'] if utils.is_build_url(product, rc_url): record = utils.record_from_url(rc_url) record['download']['size'] = f['size'] record['download']['date'] = f['last_modified'] record = utils.merge_metadata(record, metadata) records_to_create.append(record) # Theorically release should never be there yet :) # And repacks like EME-free/sha1 don't seem to be published in RC. # Nightly metadata # pub/firefox/nightly/2017/08/2017-08-08-11-40-32-mozilla-central/ # firefox-57.0a1.en-US.linux-i686.json # -l10n/... elif utils.is_nightly_build_metadata(product, url): logger.info('Processing {} nightly metadata: {}'.format( product, key)) logger.debug("Fetch new nightly metadata") metadata = await fetch_json(session, url) platform = metadata['moz_pkg_platform'] # Check if english version is here. parent_url = re.sub('/[^/]+$', '/', url) logger.debug("Fetch parent listing {}".format(parent_url)) _, files = await fetch_listing(session, parent_url) for f in files: if ('.' + platform + '.') not in f['name']: # metadata are by platform. continue en_nightly_url = parent_url + f['name'] if utils.is_build_url(product, en_nightly_url): record = utils.record_from_url(en_nightly_url) record['download']['size'] = f['size'] record['download']['date'] = f['last_modified'] record = utils.merge_metadata(record, metadata) records_to_create.append(record) break # Only one file for english. # Check also localized versions. l10n_folder_url = re.sub('-mozilla-central([^/]*)/([^/]+)$', '-mozilla-central\\1-l10n/', url) logger.debug("Fetch l10n listing {}".format(l10n_folder_url)) try: _, files = await fetch_listing(session, l10n_folder_url) except ValueError: files = [] # No -l10/ folder published yet. for f in files: if ('.' + platform + '.') not in f['name'] and product != 'mobile': # metadata are by platform. # (mobile platforms are contained by folder) continue nightly_url = l10n_folder_url + f['name'] if utils.is_build_url(product, nightly_url): record = utils.record_from_url(nightly_url) record['download']['size'] = f['size'] record['download']['date'] = f['last_modified'] record = utils.merge_metadata(record, metadata) records_to_create.append(record) else: logger.info('Ignored {}'.format(key)) logger.debug("{} records to create.".format( len(records_to_create))) for record in records_to_create: # Check that fields values look OK. utils.check_record(record) # Push result to Kinto. kinto_client.create_record(data=record, bucket=bucket, collection=collection, if_not_exists=True) logger.info('Created {}'.format(record['id']))
async def main(loop, event): """ Trigger when S3 event kicks in. http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html """ server_url = config('SERVER_URL', default='http://localhost:8888/v1') bucket = config('BUCKET', default='build-hub') collection = config('COLLECTION', default='releases') kinto_auth = tuple(config('AUTH', 'user:pass').split(':')) kinto_client = kinto_http.Client(server_url=server_url, auth=kinto_auth, retry=NB_RETRY_REQUEST) records = [] for record in event['Records']: if record.get('EventSource') == 'aws:sns': records.extend(json.loads(record['Sns']['Message'])['Records']) else: records.append(record) async with aiohttp.ClientSession(loop=loop) as session: for event_record in records: metrics.incr('s3_event_event') records_to_create = [] # Use event time as archive publication. event_time = ciso8601.parse_datetime(event_record['eventTime']) event_time = event_time.strftime(utils.DATETIME_FORMAT) key = event_record['s3']['object']['key'] filesize = event_record['s3']['object']['size'] url = utils.key_to_archive_url(key) logger.debug("Event file {}".format(url)) try: product = key.split('/')[1] # /pub/thunderbird/nightly/... except IndexError: continue # e.g. https://archive.mozilla.org/favicon.ico if product not in utils.ALL_PRODUCTS: logger.info('Skip product {}'.format(product)) continue # Release / Nightly / RC archive. if utils.is_build_url(product, url): logger.info('Processing {} archive: {}'.format(product, key)) record = utils.record_from_url(url) # Use S3 event infos for the archive. record['download']['size'] = filesize record['download']['date'] = event_time # Fetch release metadata. await scan_candidates(session, product) logger.debug("Fetch record metadata") # metadata = await fetch_metadata(session, record) metadata = await fetch_metadata(session, record) # If JSON metadata not available, archive will be # handled when JSON is delivered. if metadata is None: logger.info(f"JSON metadata not available {record['id']}") continue # Merge obtained metadata. record = utils.merge_metadata(record, metadata) records_to_create.append(record) # RC metadata elif utils.is_rc_build_metadata(product, url): logger.info(f'Processing {product} RC metadata: {key}') # pub/firefox/candidates/55.0b12-candidates/build1/mac/en-US/ # firefox-55.0b12.json logger.debug("Fetch new metadata") # It has been known to happen that right after an S3 Event # there's a slight delay to the metadata json file being # available. If that's the case we want to retry in a couple # of seconds to see if it's available on the next backoff # attempt. metadata = await fetch_json(session, url, retry_on_notfound=True) metadata['buildnumber'] = int( re.search('/build(\d+)/', url).group(1)) # We just received the metadata file. Lookup if the associated # archives are here too. archives = [] if 'multi' in url: # For multi we just check the associated archive # is here already. parent_folder = re.sub('multi/.+$', 'multi/', url) _, files = await fetch_listing(session, parent_folder, retry_on_notfound=True) for f in files: rc_url = parent_folder + f['name'] if utils.is_build_url(product, rc_url): archives.append( (rc_url, f['size'], f['last_modified'])) else: # For en-US it's different, it applies to every # localized archives. # Check if they are here by listing the parent folder # (including en-US archive). l10n_parent_url = re.sub('en-US/.+$', '', url) l10n_folders, _ = await fetch_listing( session, l10n_parent_url, retry_on_notfound=True, ) for locale in l10n_folders: _, files = await fetch_listing( session, l10n_parent_url + locale, retry_on_notfound=True, ) for f in files: rc_url = l10n_parent_url + locale + f['name'] if utils.is_build_url(product, rc_url): archives.append(( rc_url, f['size'], f['last_modified'], )) for rc_url, size, last_modified in archives: record = utils.record_from_url(rc_url) record['download']['size'] = size record['download']['date'] = last_modified record = utils.merge_metadata(record, metadata) records_to_create.append(record) # Theorically release should never be there yet :) # And repacks like EME-free/sha1 don't seem to be # published in RC. # Nightly metadata # pub/firefox/nightly/2017/08/2017-08-08-11-40-32-mozilla-central/ # firefox-57.0a1.en-US.linux-i686.json # -l10n/... elif utils.is_nightly_build_metadata(product, url): logger.info(f'Processing {product} nightly metadata: {key}') logger.debug("Fetch new nightly metadata") # See comment above about the exceptional need of # setting retry_on_notfound here. metadata = await fetch_json(session, url, retry_on_notfound=True) platform = metadata['moz_pkg_platform'] # Check if english version is here. parent_url = re.sub('/[^/]+$', '/', url) logger.debug("Fetch parent listing {}".format(parent_url)) _, files = await fetch_listing(session, parent_url) for f in files: if ('.' + platform + '.') not in f['name']: # metadata are by platform. continue en_nightly_url = parent_url + f['name'] if utils.is_build_url(product, en_nightly_url): record = utils.record_from_url(en_nightly_url) record['download']['size'] = f['size'] record['download']['date'] = f['last_modified'] record = utils.merge_metadata(record, metadata) records_to_create.append(record) break # Only one file for english. # Check also localized versions. l10n_folder_url = re.sub('-mozilla-central([^/]*)/([^/]+)$', '-mozilla-central\\1-l10n/', url) logger.debug("Fetch l10n listing {}".format(l10n_folder_url)) try: _, files = await fetch_listing( session, l10n_folder_url, retry_on_notfound=True, ) except ValueError: files = [] # No -l10/ folder published yet. for f in files: if (('.' + platform + '.') not in f['name'] and product != 'mobile'): # metadata are by platform. # (mobile platforms are contained by folder) continue nightly_url = l10n_folder_url + f['name'] if utils.is_build_url(product, nightly_url): record = utils.record_from_url(nightly_url) record['download']['size'] = f['size'] record['download']['date'] = f['last_modified'] record = utils.merge_metadata(record, metadata) records_to_create.append(record) else: logger.info('Ignored {}'.format(key)) logger.debug(f"{len(records_to_create)} records to create.") with metrics.timer('s3_event_records_to_create'): for record in records_to_create: # Check that fields values look OK. utils.check_record(record) # Push result to Kinto. kinto_client.create_record(data=record, bucket=bucket, collection=collection, if_not_exists=True) logger.info('Created {}'.format(record['id'])) metrics.incr('s3_event_record_created')
print( requests.put( urllib.parse.urljoin(KINTO_HOST, f"/accounts/{user}"), json={ "data": { "password": passw } }, ).content, ) create_user(ADMIN_USER, ADMIN_PASS) create_user(REVIEW_USER, REVIEW_PASS) create_user(EXPERIMENTER_USER, EXPERIMENTER_PASS) client = kinto_http.Client(server_url=KINTO_HOST, auth=(ADMIN_USER, ADMIN_PASS)) print(f">>>> Creating kinto bucket: {KINTO_BUCKET}") print( client.create_bucket( id=KINTO_BUCKET, permissions={"read": ["system.Everyone"]}, if_not_exists=True, )) for collection in [ KINTO_COLLECTION_NIMBUS_DESKTOP, KINTO_COLLECTION_NIMBUS_MOBILE, ]: print(">>>> Creating kinto group: editors") print(