def main(args=None): parser = cli_utils.add_parser_options( description='Generate blocked item description files.', default_collection=None, default_bucket=constants.DESTINATION_BUCKET, default_server=constants.KINTO_SERVER, default_auth=constants.AUTH, include_collection=False) parser.add_argument('--addons-collection', help='Collection name for addon', type=str, default=constants.ADDONS_COLLECTION) parser.add_argument('--plugins-collection', help='Collection name for plugin', type=str, default=constants.PLUGINS_COLLECTION) parser.add_argument('-d', '--target-dir', help='Destination directory to write files in.', type=str, default=constants.TARGET_DIR) args = parser.parse_args(args=args) cli_utils.setup_logger(logger, args) kinto_client = cli_utils.create_client_from_args(args) collections = [ COLLECTION_FORMAT.format(bucket_id=args.bucket, collection_id=args.addons_collection), COLLECTION_FORMAT.format(bucket_id=args.bucket, collection_id=args.plugins_collection), ] generate(kinto_client, collections, args.target_dir, 'collection.tpl', 'record.tpl')
def test_get_auth_is_called_by_argparse(self): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates") args = parser.parse_args(['-a', 'user:password']) assert args.auth == ('user', 'password')
def test_add_parser_options_adds_arguments_on_existing_parser(self): parser = argparse.ArgumentParser(prog="importer") parser.add_argument('-t', '--type', help='File type', type=str, default='xml') parser = cli_utils.add_parser_options(parser) self.assert_option_strings(parser, ['-t', '--type'], *ALL_PARAMETERS) assert len(parser._actions) == 11
def main(): parser = cli_utils.add_parser_options( description='Upgrade the attachment structure', default_server=DEFAULT_SERVER, default_bucket='fennec', default_collection='experiments') args = parser.parse_args() client = cli_utils.create_client_from_args(args) push(client)
def test_add_parser_options_can_ignore_bucket_and_collection(self): parser = cli_utils.add_parser_options( include_bucket=False, include_collection=False) parameters = [ ['-h', '--help'], ['-s', '--server'], ['-a', '--auth'], ['-v', '--verbose'], ['-q', '--quiet'], ['-D', '--debug'], ] self.assert_option_strings(parser, *parameters) assert len(parser._actions) == 6
def test_create_client_from_args_build_a_client(self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth=('user', 'password') ) args = parser.parse_args([]) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url='https://firefox.settings.services.mozilla.com/', auth=('user', 'password'), bucket='blocklists', collection='certificates')
def test_create_client_from_args_default_bucket_and_collection_to_none( self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_auth=('user', 'password'), include_bucket=False, include_collection=False ) args = parser.parse_args([]) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url='https://firefox.settings.services.mozilla.com/', auth=('user', 'password'), bucket=None, collection=None)
def test_can_change_default_values(self): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth='user:password' ) args = vars(parser.parse_args([])) assert args == { 'server': 'https://firefox.settings.services.mozilla.com/', 'auth': 'user:password', 'bucket': 'blocklists', 'collection': 'certificates', 'verbosity': None }
def test_create_client_from_args_build_a_client(self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", ) args = parser.parse_args(['--auth', 'user:password', '--bucket', 'blocklists', '--collection', 'certificates', '--retry', '3']) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url='https://firefox.settings.services.mozilla.com/', auth=('user', 'password'), bucket='blocklists', collection='certificates', retry=3, retry_after=None)
def test_create_client_from_args_default_bucket_and_collection_to_none( self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_auth=('user', 'password'), include_bucket=False, include_collection=False) args = parser.parse_args([]) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url='https://firefox.settings.services.mozilla.com/', auth=('user', 'password'), bucket=None, collection=None, retry=0, retry_after=None)
def test_can_change_default_values(self): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth='user:password') args = vars(parser.parse_args([])) assert args == { 'server': 'https://firefox.settings.services.mozilla.com/', 'auth': 'user:password', 'bucket': 'blocklists', 'collection': 'certificates', 'retry': 0, 'retry_after': None, 'verbosity': None }
def test_create_client_from_default_args_build_a_client( self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth=('user', 'password')) args = parser.parse_args([]) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url='https://firefox.settings.services.mozilla.com/', auth=('user', 'password'), bucket='blocklists', collection='certificates', retry=0, retry_after=None)
def test_can_change_default_values(self): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth="user:password", ) args = vars(parser.parse_args([])) assert args == { "server": "https://firefox.settings.services.mozilla.com/", "auth": "user:password", "bucket": "blocklists", "collection": "certificates", "retry": 0, "retry_after": None, "verbosity": None, "ignore_batch_4xx": False, }
def test_create_client_from_default_args_build_a_client( self, mocked_client): parser = cli_utils.add_parser_options( default_server="https://firefox.settings.services.mozilla.com/", default_bucket="blocklists", default_collection="certificates", default_auth=("user", "password"), ) args = parser.parse_args([]) cli_utils.create_client_from_args(args) mocked_client.assert_called_with( server_url="https://firefox.settings.services.mozilla.com/", auth=("user", "password"), bucket="blocklists", collection="certificates", ignore_batch_4xx=False, retry=0, retry_after=None, )
def main(): parser = cli_utils.add_parser_options( description="Update the collections metadata", default_server=DEFAULT_SERVER, default_bucket=DEFAULT_BUCKET, ) args = parser.parse_args() client = cli_utils.create_client_from_args(args) print("Updating the `upcoming` collection") collections_metadata.update_upcoming(client) print("Updating the `videos` collection") collections_metadata.update_videos(client) print("Updating the `contacts` collection") collections_metadata.update_contacts(client) print("Updating the `thumbnails` collection") collections_metadata.update_thumbnails(client)
def main(args=None): parser = cli_utils.add_parser_options( description='Validate collection signature', default_server=DEFAULT_SERVER, default_bucket=DEST_BUCKET, default_collection=DEST_COLLECTION) args = parser.parse_args(args) client = cli_utils.create_client_from_args(args) # 1. Grab collection information dest_col = client.get_collection() # 2. Grab records records = list(client.get_records(_sort='-last_modified')) timestamp = client.get_records_timestamp() # 3. Serialize serialized = canonical_json(records, timestamp) # 4. Compute the hash computed_hash = compute_hash(serialized) # 5. Grab the signature signature = dest_col['data']['signature'] # 6. Grab the public key with open('pub', 'w') as f: f.write(signature['public_key']) # 7. Verify the signature matches the hash signer = ECDSASigner(public_key='pub') try: signer.verify(serialized, signature) print('Signature OK') except Exception: print('Signature KO. Computed hash: %s' % computed_hash) raise
def main(args=None): parser = cli_utils.add_parser_options( description='Import the blocklists from the addons server into Kinto.', default_collection=None, default_bucket=None, default_server=constants.KINTO_SERVER, default_auth=constants.AUTH, include_bucket=False, include_collection=False) parser.add_argument('-S', '--schema-file', help='JSON Schemas file', type=str, default=constants.SCHEMA_FILE) parser.add_argument('--no-schema', help='Should we handle schemas', action="store_true") parser.add_argument('--editor-auth', help='Credentials to be used for requesting a review', type=str, default=None) parser.add_argument('--reviewer-auth', help='Credentials to be used for validating the review', type=str, default=None) parser.add_argument('--certificates-bucket', help='Bucket name for certificates', type=str, default=constants.CERT_BUCKET) parser.add_argument('--certificates-collection', help='Collection name for certificates', type=str, default=constants.CERT_COLLECTION) parser.add_argument('--gfx-bucket', help='Bucket name for gfx', type=str, default=constants.GFX_BUCKET) parser.add_argument('--gfx-collection', help='Collection name for gfx', type=str, default=constants.GFX_COLLECTION) parser.add_argument('--addons-bucket', help='Bucket name for addons', type=str, default=constants.ADDONS_BUCKET) parser.add_argument('--addons-collection', help='Collection name for addon', type=str, default=constants.ADDONS_COLLECTION) parser.add_argument('--plugins-bucket', help='Bucket name for plugins', type=str, default=constants.PLUGINS_BUCKET) parser.add_argument('--plugins-collection', help='Collection name for plugin', type=str, default=constants.PLUGINS_COLLECTION) parser.add_argument('-C', '--certificates', help='Only import certificates', action='store_true') parser.add_argument('-G', '--gfx', help='Only import GFX drivers', action='store_true') parser.add_argument('-A', '--addons', help='Only import addons', action='store_true') parser.add_argument('-P', '--plugins', help='Only import plugins', action='store_true') # Addons Server selection parser.add_argument('--addons-server', help='The addons server to import from', type=str, default=constants.ADDONS_SERVER) args = parser.parse_args(args=args) cli_utils.setup_logger(logger, args) kinto_client = cli_utils.create_client_from_args(args) editor_client = None if args.editor_auth is not None: args.editor_auth = tuple(args.editor_auth.split(':', 1)) editor_client = Client(server_url=args.server, auth=args.editor_auth) reviewer_client = None if args.reviewer_auth is not None: args.reviewer_auth = tuple(args.reviewer_auth.split(':', 1)) reviewer_client = Client(server_url=args.server, auth=args.reviewer_auth) # If none of the different "collections" were passed as parameter, then we # want to import them all. import_all = not any([ args.certificates, args.gfx, args.addons, args.plugins]) # Check if the schema capability is activated body, headers = kinto_client.session.request('get', '/') if 'schema' not in body.get('capabilities', {}): logger.warn('\t --- Server schema validation disabled --- \n') logger.warn("The server at {} won't validate the records against " "the collection JSON schema. More information " "http://kinto.readthedocs.io/en/stable/api/1.x/" "collections.html?highlight=json%20validation" "#collection-json-schema\n".format(args.server)) # Load the schemas schemas = {} if not args.no_schema: with codecs.open(args.schema_file, 'r', encoding='utf-8') as f: schemas = json.load(f)['collections'] blocklists_url = urljoin(args.addons_server, '/blocked/blocklists.json') resp = requests.get(blocklists_url) resp.raise_for_status() blocklists = resp.json() for collection_type, records in six.iteritems(blocklists): collection_type = collection_type.replace('-', '') if hasattr(args, collection_type) and ( getattr(args, collection_type) or import_all): bucket = getattr(args, '%s_bucket' % collection_type) collection = getattr(args, '%s_collection' % collection_type) config = None if collection_type in schemas: config = schemas[collection_type]['config'] sync_records(amo_records=records, fields=FIELDS[collection_type], kinto_client=kinto_client, editor_client=editor_client, reviewer_client=reviewer_client, bucket=bucket, collection=collection, config=config, permissions=constants.COLLECTION_PERMISSIONS)
def main(args=None): parser = cli_utils.add_parser_options( description='Build a blocklists.xml file from Kinto blocklists.', default_collection=None, default_bucket=None, default_server=constants.KINTO_SERVER, default_auth=constants.AUTH, include_bucket=False, include_collection=False) parser.add_argument('--certificates-bucket', help='Bucket name for certificates', type=str, default=constants.CERT_BUCKET) parser.add_argument('--certificates-collection', help='Collection name for certificates', type=str, default=constants.CERT_COLLECTION) parser.add_argument('--gfx-bucket', help='Bucket name for gfx', type=str, default=constants.GFX_BUCKET) parser.add_argument('--gfx-collection', help='Collection name for gfx', type=str, default=constants.GFX_COLLECTION) parser.add_argument('--addons-bucket', help='Bucket name for addons', type=str, default=constants.ADDONS_BUCKET) parser.add_argument('--addons-collection', help='Collection name for addon', type=str, default=constants.ADDONS_COLLECTION) parser.add_argument('--plugins-bucket', help='Bucket name for plugins', type=str, default=constants.PLUGINS_BUCKET) parser.add_argument('--plugins-collection', help='Collection name for plugin', type=str, default=constants.PLUGINS_COLLECTION) parser.add_argument('--api-version', help='Targeted blocklists.xml APP id', type=int, default=3) parser.add_argument('--app', help='Targeted blocklists.xml APP id', type=str, default=constants.FIREFOX_APPID) parser.add_argument('--app-version', help='The targetted app version', type=str, default=None) # Choose where to write the file down. parser.add_argument('-o', '--out', help='Output XML file.', type=str, default=None) args = parser.parse_args(args=args) cli_utils.setup_logger(logger, args) if not args.out: out_fd = sys.stdout close_out_fd = False else: out_fd = open(args.out, 'w+') close_out_fd = True client = cli_utils.create_client_from_args(args) last_update = 0 # Retrieve the collection of records. try: addons_records = client.get_records(bucket=args.addons_bucket, collection=args.addons_collection, enabled=True, _sort="last_modified") except Exception: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.addons_bucket, collection=args.addons_collection, )) addons_records = [] if addons_records: last_update = addons_records[-1]['last_modified'] try: plugin_records = client.get_records(bucket=args.plugins_bucket, collection=args.plugins_collection, enabled=True, _sort="last_modified") except Exception: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.plugins_bucket, collection=args.plugins_collection, )) plugin_records = [] if plugin_records: last_update = max(last_update, plugin_records[-1]['last_modified']) try: gfx_records = client.get_records(bucket=args.gfx_bucket, collection=args.gfx_collection, enabled=True, _sort="last_modified") except Exception: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.gfx_bucket, collection=args.gfx_collection, )) gfx_records = [] if gfx_records: last_update = max(last_update, gfx_records[-1]['last_modified']) cert_records = None if should_include_certs(args.app, args.app_version): try: cert_records = client.get_records( bucket=args.certificates_bucket, collection=args.certificates_collection, enabled=True, _sort="last_modified") except Exception: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'. format( bucket=args.certificates_bucket, collection=args.certificates_collection, )) cert_records = [] if cert_records: last_update = max(last_update, cert_records[-1]['last_modified']) xml_tree = etree.Element( 'blocklist', xmlns="http://www.mozilla.org/2006/addons-blocklist", lastupdate='%s' % last_update) write_addons_items(xml_tree, addons_records, api_ver=args.api_version, app_id=args.app, app_ver=args.app_version) write_plugin_items(xml_tree, plugin_records, api_ver=args.api_version, app_id=args.app, app_ver=args.app_version) write_gfx_items(xml_tree, gfx_records, api_ver=args.api_version, app_id=args.app) write_cert_items(xml_tree, cert_records, api_ver=args.api_version, app_id=args.app, app_ver=args.app_version) doc = etree.ElementTree(xml_tree) out_fd.write( etree.tostring(doc, pretty_print=True, xml_declaration=True, encoding='UTF-8').decode('utf-8')) if close_out_fd: out_fd.close()
def main(args=None): parser = cli_utils.add_parser_options( description="Validate kinto-emailer setup", default_server=DEFAULT_SERVER, default_bucket=BUCKET, default_collection=COLLECTION) args = parser.parse_args(args) cli_utils.setup_logger(logger, args) client = cli_utils.create_client_from_args(args) # 1. Check capabilities capabilities = client.server_info()["capabilities"] if "emailer" not in capabilities: print("Server doesn't have support for email notifications: \x1b[1;31m KO \x1b[0m") return 2 print("Clear inbox") clear_inbox(MAILBOX) # 2. Configure collection metadata. print("Configure emailing in %r collection" % args.collection) setup_notifs_signing(client, RECIPIENT) # 3. Create a record. print("Create a dummy record") client.create_record(data={"script": "validate kinto-emailer setup"}) # 4. If signing is enabled for this collection, then see integration with kinto-signer. signed_resources = capabilities.get("signer", {"resources": []})["resources"] ids = [(r["source"]["bucket"], r["source"]["collection"]) for r in signed_resources] signing_enabled = (args.bucket, args.collection) in ids if signing_enabled: # 4. Request a review. client.patch_collection(data={"status": "to-review"}) # 5. Check that emails were received. print("Scan inbox...") first_try = time.time() while True: emails = fetch_emails(MAILBOX) subjects = [e["subject"] for e in emails] if SUBJECT_EMAIL_CREATION not in subjects: if time.time() - first_try < RETRY_TIMEOUT: time.sleep(3) sys.stdout.write('.') sys.stdout.flush() continue break if SUBJECT_EMAIL_CREATION in subjects: print("Email received:\x1b[1;32m OK \x1b[0m") if signing_enabled: if "requested review" in subjects: print("Review notification received:\x1b[1;32m OK \x1b[0m") else: print("Review notification not received:\x1b[1;31m KO \x1b[0m") else: print("Email not received:\x1b[1;31m KO \x1b[0m") return 0
def main(): parser = argparse.ArgumentParser( description="Wizard to setup Kinto with YAML") subparsers = parser.add_subparsers(title='subcommand', description='Load/Dump/Validate', dest='subcommand', help="Choose and run with --help") subparsers.required = True # load sub-command. subparser = subparsers.add_parser('load') subparser.set_defaults(which='load') cli_utils.add_parser_options(subparser) subparser.add_argument(dest='filepath', help='YAML file') subparser.add_argument( '--force', help='Load the file using the CLIENT_WINS conflict resolution strategy', action='store_true') subparser.add_argument('--dry-run', help="Do not apply write call to the server", action='store_true') subparser.add_argument('--delete-records', help='Delete records that are not in the file.', action='store_true') # dump sub-command. subparser = subparsers.add_parser('dump') subparser.set_defaults(which='dump') cli_utils.add_parser_options(subparser) subparser.add_argument( '--full', help='Full output (same as with both --data and --records options)', action='store_true') subparser.add_argument('--data', help='Export buckets, collections and groups data', action='store_true') subparser.add_argument('--records', help="Export collections' records", action='store_true') # validate sub-command. subparser = subparsers.add_parser('validate') subparser.set_defaults(which='validate') subparser.set_defaults(verbosity=logging.INFO) subparser.add_argument(dest='filepath', help='YAML file to validate') cli_utils.add_parser_options(subparser) # Parse CLI args. args = parser.parse_args() cli_utils.setup_logger(logger, args) kinto_logger = logging.getLogger('kinto_http') cli_utils.setup_logger(kinto_logger, args) if args.which == 'validate': logger.debug("Start validation...") logger.info("Load YAML file {!r}".format(args.filepath)) with open(args.filepath, 'r') as f: config = yaml.safe_load(f) logger.info("File loaded!") fine = validate_export(config) sys.exit(0 if fine else 1) logger.debug("Instantiate Kinto client.") client = cli_utils.create_client_from_args(args) thread_pool = ThreadPoolExecutor() event_loop = asyncio.get_event_loop() async_client = AsyncKintoClient(client, event_loop, thread_pool, dry_run=getattr(args, 'dry_run', False)) # Run chosen subcommand. if args.which == 'dump': if args.full: data = True records = True else: data = args.data records = args.records logger.debug( "Start introspection with %s%s%s..." % ("data" if data else "", " and " if data and records else "", "records" if records else "")) result = event_loop.run_until_complete( introspect_server(async_client, bucket=args.bucket, collection=args.collection, data=data, records=records)) yaml_result = yaml.safe_dump(result, default_flow_style=False) print(yaml_result, end=u'') elif args.which == 'load': logger.debug("Start initialization...") logger.info("Load YAML file {!r}".format(args.filepath)) with open(args.filepath, 'r') as f: config = yaml.safe_load(f) event_loop.run_until_complete( initialize_server(async_client, config, bucket=args.bucket, collection=args.collection, force=args.force, delete_missing_records=args.delete_records))
def test_add_parser_options_create_a_parser_if_needed(self): parser = cli_utils.add_parser_options() self.assert_option_strings(parser, *ALL_PARAMETERS) assert len(parser._actions) == 10
def main(args=None): parser = cli_utils.add_parser_options( description='Build a blocklists.xml file from Kinto blocklists.', default_collection=None, default_bucket=None, default_server=constants.KINTO_SERVER, default_auth=constants.AUTH, include_bucket=False, include_collection=False) parser.add_argument('--certificates-bucket', help='Bucket name for certificates', type=str, default=constants.CERT_BUCKET) parser.add_argument('--certificates-collection', help='Collection name for certificates', type=str, default=constants.CERT_COLLECTION) parser.add_argument('--gfx-bucket', help='Bucket name for gfx', type=str, default=constants.GFX_BUCKET) parser.add_argument('--gfx-collection', help='Collection name for gfx', type=str, default=constants.GFX_COLLECTION) parser.add_argument('--addons-bucket', help='Bucket name for addons', type=str, default=constants.ADDONS_BUCKET) parser.add_argument('--addons-collection', help='Collection name for addon', type=str, default=constants.ADDONS_COLLECTION) parser.add_argument('--plugins-bucket', help='Bucket name for plugins', type=str, default=constants.PLUGINS_BUCKET) parser.add_argument('--plugins-collection', help='Collection name for plugin', type=str, default=constants.PLUGINS_COLLECTION) parser.add_argument('--api-version', help='Targeted blocklists.xml APP id', type=int, default=3) parser.add_argument('--app', help='Targeted blocklists.xml APP id', type=str, default=constants.FIREFOX_APPID) parser.add_argument('--app-version', help='The targetted app version', type=str, default=None) # Choose where to write the file down. parser.add_argument('-o', '--out', help='Output XML file.', type=str, default=None) args = parser.parse_args(args=args) cli_utils.setup_logger(logger, args) if not args.out: out_fd = sys.stdout close_out_fd = False else: out_fd = open(args.out, 'w+') close_out_fd = True client = cli_utils.create_client_from_args(args) last_update = 0 # Retrieve the collection of records. try: addons_records = client.get_records( bucket=args.addons_bucket, collection=args.addons_collection, enabled=True, _sort="last_modified") except: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.addons_bucket, collection=args.addons_collection, ) ) addons_records = [] if addons_records: last_update = addons_records[-1]['last_modified'] try: plugin_records = client.get_records( bucket=args.plugins_bucket, collection=args.plugins_collection, enabled=True, _sort="last_modified") except: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.plugins_bucket, collection=args.plugins_collection, ) ) plugin_records = [] if plugin_records: last_update = max(last_update, plugin_records[-1]['last_modified']) try: gfx_records = client.get_records( bucket=args.gfx_bucket, collection=args.gfx_collection, enabled=True, _sort="last_modified") except: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.gfx_bucket, collection=args.gfx_collection, ) ) gfx_records = [] if gfx_records: last_update = max(last_update, gfx_records[-1]['last_modified']) try: cert_records = client.get_records( bucket=args.certificates_bucket, collection=args.certificates_collection, enabled=True, _sort="last_modified") except: logger.warn( 'Unable to fetch the ``{bucket}/{collection}`` records.'.format( bucket=args.certificates_bucket, collection=args.certificates_collection, ) ) cert_records = [] if cert_records: last_update = max(last_update, cert_records[-1]['last_modified']) xml_tree = etree.Element( 'blocklist', xmlns="http://www.mozilla.org/2006/addons-blocklist", lastupdate='%s' % last_update ) write_addons_items(xml_tree, addons_records, api_ver=args.api_version, app_id=args.app) write_plugin_items(xml_tree, plugin_records, api_ver=args.api_version, app_id=args.app, app_ver=args.app_version) write_gfx_items(xml_tree, gfx_records, api_ver=args.api_version, app_id=args.app) write_cert_items(xml_tree, cert_records, api_ver=args.api_version) doc = etree.ElementTree(xml_tree) out_fd.write(etree.tostring( doc, pretty_print=True, xml_declaration=True, encoding='UTF-8').decode('utf-8')) if close_out_fd: out_fd.close()
def test_add_parser_options_create_a_parser_if_needed(self): parser = cli_utils.add_parser_options() self.assert_option_strings(parser, *ALL_PARAMETERS) assert len(parser._actions) == len(ALL_PARAMETERS)
def main(args=None): parser = cli_utils.add_parser_options(description="Parse2Kinto importer", default_bucket=None, default_collection=None) parser.add_argument('--parse-server', help='Parse API server', type=str, default=PARSE_DEFAULT_SERVER) parser.add_argument('--parse-app', help='Your Parse APP ID', type=str) parser.add_argument('--parse-rest-key', help='Your Parse APP ID REST API KEY', type=str) parser.add_argument('--parse-class', help='The Parse APP class you want to import', type=str) args = parser.parse_args(args) cli_utils.setup_logger(logger, args) kinto_client = cli_utils.create_client_from_args(args) parse_client = parse.create_client_from_args(args) # Create bucket kinto_client.create_bucket(if_not_exists=True) # Create collection if doesn't exists yet # try: # kinto_client.create_collection(safe=True) # except KintoException as e: # if hasattr(e, 'response') and e.response.status_code == 412: # raise KintoException("The collection already exists. Please " # "delete it first to make a clean import.") kinto_client.create_collection(if_not_exists=True) # Count number of objects count = parse_client.get_number_of_records() pages = int(math.ceil(float(count) / RECORD_PER_PAGES)) widgets = [ 'Import: ', Percentage(), ' ', BouncingBar(), ' ', AdaptiveETA() ] print("Importing %d records from %s" % (count, args.parse_class)) num_processed = 0 with ProgressBar(widgets=widgets, max_value=count) as progress: # Get parse records progress.update(num_processed) # Manual batch management mode. p = kinto_client.batch() batch = p.__enter__() send_every = batch.session.batch_max_requests for page in range(pages): records = parse_client.get_records(page, RECORD_PER_PAGES) for record in records: batch.create_record(data=parse.convert_record(record), safe=True) num_processed += 1 if num_processed % send_every == 0: try: batch.session.send() except KintoException as e: if e.response['status'] != 412: raise finally: batch.session.reset() progress.update(num_processed) p.__exit__(None, None, None)