def main(args=None):
parser = cli_utils.add_parser_options(
description='Generate blocked item description files.',
default_collection=None,
default_bucket=constants.DESTINATION_BUCKET,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_collection=False)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-d', '--target-dir',
help='Destination directory to write files in.',
type=str, default=constants.TARGET_DIR)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
collections = [
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.addons_collection),
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.plugins_collection),
]
generate(kinto_client, collections, args.target_dir, 'collection.tpl', 'record.tpl')
def run():
loop = asyncio.get_event_loop()
stdin_generator = stream_as_generator(loop, sys.stdin)
records_generator = parse_json(stdin_generator)
parser = cli_utils.add_parser_options(
description='Read records from stdin as JSON and push them to Kinto',
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_retry=NB_RETRY_REQUEST,
default_collection=DEFAULT_COLLECTION)
parser.add_argument('--skip',
action='store_true',
help='Skip records that exist and are equal.')
cli_args = parser.parse_args()
cli_utils.setup_logger(logger, cli_args)
logger.info(
'Publish at {server}/buckets/{bucket}/collections/{collection}'.format(
**cli_args.__dict__))
client = cli_utils.create_client_from_args(cli_args)
main_coro = main(loop,
records_generator,
client,
skip_existing=cli_args.skip)
loop.run_until_complete(main_coro)
loop.close()
def main(args=None):
parser = cli_utils.add_parser_options(
description='Generate blocked item description files.',
default_collection=None,
default_bucket=constants.DESTINATION_BUCKET,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_collection=False)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-d', '--target-dir',
help='Destination directory to write files in.',
type=str, default=constants.TARGET_DIR)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
collections = [
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.addons_collection),
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.plugins_collection),
]
generate(kinto_client, collections, args.target_dir, 'collection.tpl', 'record.tpl')
def main():
parser = argparse.ArgumentParser(
description="Wizard to setup Kinto with YAML")
subparsers = parser.add_subparsers(title='subcommand',
description='Load/Dump',
dest='subcommand',
help="Choose and run with --help")
subparsers.required = True
# load sub-command.
subparser = subparsers.add_parser('load')
subparser.set_defaults(which='load')
cli_utils.add_parser_options(subparser)
subparser.add_argument(dest='filepath', help='YAML file')
subparser.add_argument(
'--force',
help='Load the file using the CLIENT_WINS conflict resolution strategy',
action='store_true')
# dump sub-command.
subparser = subparsers.add_parser('dump')
subparser.set_defaults(which='dump')
cli_utils.add_parser_options(subparser)
subparser.add_argument('--full', help='Full output', action='store_true')
# Parse CLI args.
args = parser.parse_args()
cli_utils.setup_logger(logger, args)
kinto_logger = logging.getLogger('kinto_http')
cli_utils.setup_logger(kinto_logger, args)
logger.debug("Instantiate Kinto client.")
client = cli_utils.create_client_from_args(args)
# Run chosen subcommand.
if args.which == 'dump':
logger.debug("Start %sintrospection..." %
("full " if args.full else ""))
result = introspect_server(client,
bucket=args.bucket,
collection=args.collection,
full=args.full)
yaml_result = yaml.safe_dump(result, default_flow_style=False)
print(yaml_result, end=u'')
elif args.which == 'load':
logger.debug("Start initialization...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
initialize_server(client,
config,
bucket=args.bucket,
collection=args.collection,
force=args.force)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(
server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection
)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def main():
# Setup parser
parser = argparse.ArgumentParser(
description='Migrate Kinto databases from specification files.'
)
parser.add_argument('-f', '--file',
help='File describing Kinto resources.')
cli_utils.add_parser_options(parser)
args = parser.parse_args()
# Setup logging
logger = logging.getLogger(__name__)
cli_utils.setup_logger(logger, args)
# Create client
client = cli_utils.create_client_from_args(args)
# Load models file
models = yaml.load(open(args.file or 'models.yml'))
# Run migration
migrate_model(client, models)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Import the blocklists from the addons server into Kinto.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
type=str, default=constants.SCHEMA_FILE)
parser.add_argument('--no-schema', help='Should we handle schemas',
action="store_true")
parser.add_argument('--editor-auth',
help='Credentials to be used for requesting a review',
type=str, default=None)
parser.add_argument('--reviewer-auth',
help='Credentials to be used for validating the review',
type=str, default=None)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str, default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str, default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-C', '--certificates',
help='Only import certificates',
action='store_true')
parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
action='store_true')
parser.add_argument('-A', '--addons', help='Only import addons',
action='store_true')
parser.add_argument('-P', '--plugins', help='Only import plugins',
action='store_true')
# Addons Server selection
parser.add_argument('--addons-server',
help='The addons server to import from',
type=str, default=constants.ADDONS_SERVER)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
editor_client = None
if args.editor_auth is not None:
args.editor_auth = tuple(args.editor_auth.split(':', 1))
editor_client = Client(server_url=args.server,
auth=args.editor_auth)
reviewer_client = None
if args.reviewer_auth is not None:
args.reviewer_auth = tuple(args.reviewer_auth.split(':', 1))
reviewer_client = Client(server_url=args.server,
auth=args.reviewer_auth)
# If none of the different "collections" were passed as parameter, then we
# want to import them all.
import_all = not any([
args.certificates,
args.gfx,
args.addons,
args.plugins])
# Check if the schema capability is activated
body, headers = kinto_client.session.request('get', '/')
if 'schema' not in body.get('capabilities', {}):
logger.warn('\t --- Server schema validation disabled --- \n')
logger.warn("The server at {} won't validate the records against "
"the collection JSON schema. More information "
"http://kinto.readthedocs.io/en/stable/api/1.x/"
"collections.html?highlight=json%20validation"
"#collection-json-schema\n".format(args.server))
# Load the schemas
schemas = {}
if not args.no_schema:
with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
schemas = json.load(f)['collections']
blocklists_url = urljoin(args.addons_server, '/blocked/blocklists.json')
resp = requests.get(blocklists_url)
resp.raise_for_status()
blocklists = resp.json()
for collection_type, records in six.iteritems(blocklists):
collection_type = collection_type.replace('-', '')
if hasattr(args, collection_type) and (
getattr(args, collection_type) or import_all):
bucket = getattr(args, '%s_bucket' % collection_type)
collection = getattr(args, '%s_collection' % collection_type)
config = None
if collection_type in schemas:
config = schemas[collection_type]['config']
sync_records(amo_records=records,
fields=FIELDS[collection_type],
kinto_client=kinto_client,
editor_client=editor_client,
reviewer_client=reviewer_client,
bucket=bucket,
collection=collection,
config=config,
permissions=constants.COLLECTION_PERMISSIONS)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Build a blocklists.xml file from Kinto blocklists.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str,
default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str,
default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket',
help='Bucket name for gfx',
type=str,
default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str,
default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket',
help='Bucket name for addons',
type=str,
default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str,
default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket',
help='Bucket name for plugins',
type=str,
default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str,
default=constants.PLUGINS_COLLECTION)
parser.add_argument('--api-version',
help='Targeted blocklists.xml APP id',
type=int,
default=3)
parser.add_argument('--app',
help='Targeted blocklists.xml APP id',
type=str,
default=constants.FIREFOX_APPID)
parser.add_argument('--app-version',
help='The targetted app version',
type=str,
default=None)
# Choose where to write the file down.
parser.add_argument('-o',
'--out',
help='Output XML file.',
type=str,
default=None)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
if not args.out:
out_fd = sys.stdout
close_out_fd = False
else:
out_fd = open(args.out, 'w+')
close_out_fd = True
client = cli_utils.create_client_from_args(args)
last_update = 0
# Retrieve the collection of records.
try:
addons_records = client.get_records(bucket=args.addons_bucket,
collection=args.addons_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.addons_bucket,
collection=args.addons_collection,
))
addons_records = []
if addons_records:
last_update = addons_records[-1]['last_modified']
try:
plugin_records = client.get_records(bucket=args.plugins_bucket,
collection=args.plugins_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
))
plugin_records = []
if plugin_records:
last_update = max(last_update, plugin_records[-1]['last_modified'])
try:
gfx_records = client.get_records(bucket=args.gfx_bucket,
collection=args.gfx_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
))
gfx_records = []
if gfx_records:
last_update = max(last_update, gfx_records[-1]['last_modified'])
cert_records = None
if should_include_certs(args.app, args.app_version):
try:
cert_records = client.get_records(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.
format(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
))
cert_records = []
if cert_records:
last_update = max(last_update, cert_records[-1]['last_modified'])
xml_tree = etree.Element(
'blocklist',
xmlns="http://www.mozilla.org/2006/addons-blocklist",
lastupdate='%s' % last_update)
write_addons_items(xml_tree,
addons_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_plugin_items(xml_tree,
plugin_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_gfx_items(xml_tree,
gfx_records,
api_ver=args.api_version,
app_id=args.app)
write_cert_items(xml_tree,
cert_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
doc = etree.ElementTree(xml_tree)
out_fd.write(
etree.tostring(doc,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8').decode('utf-8'))
if close_out_fd:
out_fd.close()
def main(args=None):
parser = cli_utils.add_parser_options(
description="Validate kinto-emailer setup",
default_server=DEFAULT_SERVER,
default_bucket=BUCKET,
default_collection=COLLECTION)
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
client = cli_utils.create_client_from_args(args)
# 1. Check capabilities
capabilities = client.server_info()["capabilities"]
if "emailer" not in capabilities:
print("Server doesn't have support for email notifications: \x1b[1;31m KO \x1b[0m")
return 2
print("Clear inbox")
clear_inbox(MAILBOX)
# 2. Configure collection metadata.
print("Configure emailing in %r collection" % args.collection)
setup_notifs_signing(client, RECIPIENT)
# 3. Create a record.
print("Create a dummy record")
client.create_record(data={"script": "validate kinto-emailer setup"})
# 4. If signing is enabled for this collection, then see integration with kinto-signer.
signed_resources = capabilities.get("signer", {"resources": []})["resources"]
ids = [(r["source"]["bucket"], r["source"]["collection"]) for r in signed_resources]
signing_enabled = (args.bucket, args.collection) in ids
if signing_enabled:
# 4. Request a review.
client.patch_collection(data={"status": "to-review"})
# 5. Check that emails were received.
print("Scan inbox...")
first_try = time.time()
while True:
emails = fetch_emails(MAILBOX)
subjects = [e["subject"] for e in emails]
if SUBJECT_EMAIL_CREATION not in subjects:
if time.time() - first_try < RETRY_TIMEOUT:
time.sleep(3)
sys.stdout.write('.')
sys.stdout.flush()
continue
break
if SUBJECT_EMAIL_CREATION in subjects:
print("Email received:\x1b[1;32m OK \x1b[0m")
if signing_enabled:
if "requested review" in subjects:
print("Review notification received:\x1b[1;32m OK \x1b[0m")
else:
print("Review notification not received:\x1b[1;31m KO \x1b[0m")
else:
print("Email not received:\x1b[1;31m KO \x1b[0m")
return 0
def main():
parser = argparse.ArgumentParser(
description="Wizard to setup Kinto with YAML")
subparsers = parser.add_subparsers(title='subcommand',
description='Load/Dump/Validate',
dest='subcommand',
help="Choose and run with --help")
subparsers.required = True
# load sub-command.
subparser = subparsers.add_parser('load')
subparser.set_defaults(which='load')
cli_utils.add_parser_options(subparser)
subparser.add_argument(dest='filepath', help='YAML file')
subparser.add_argument(
'--force',
help='Load the file using the CLIENT_WINS conflict resolution strategy',
action='store_true')
subparser.add_argument('--dry-run',
help="Do not apply write call to the server",
action='store_true')
subparser.add_argument('--delete-records',
help='Delete records that are not in the file.',
action='store_true')
# dump sub-command.
subparser = subparsers.add_parser('dump')
subparser.set_defaults(which='dump')
cli_utils.add_parser_options(subparser)
subparser.add_argument(
'--full',
help='Full output (same as with both --data and --records options)',
action='store_true')
subparser.add_argument('--data',
help='Export buckets, collections and groups data',
action='store_true')
subparser.add_argument('--records',
help="Export collections' records",
action='store_true')
# validate sub-command.
subparser = subparsers.add_parser('validate')
subparser.set_defaults(which='validate')
subparser.set_defaults(verbosity=logging.INFO)
subparser.add_argument(dest='filepath', help='YAML file to validate')
cli_utils.add_parser_options(subparser)
# Parse CLI args.
args = parser.parse_args()
cli_utils.setup_logger(logger, args)
kinto_logger = logging.getLogger('kinto_http')
cli_utils.setup_logger(kinto_logger, args)
if args.which == 'validate':
logger.debug("Start validation...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
logger.info("File loaded!")
fine = validate_export(config)
sys.exit(0 if fine else 1)
logger.debug("Instantiate Kinto client.")
client = cli_utils.create_client_from_args(args)
thread_pool = ThreadPoolExecutor()
event_loop = asyncio.get_event_loop()
async_client = AsyncKintoClient(client,
event_loop,
thread_pool,
dry_run=getattr(args, 'dry_run', False))
# Run chosen subcommand.
if args.which == 'dump':
if args.full:
data = True
records = True
else:
data = args.data
records = args.records
logger.debug(
"Start introspection with %s%s%s..." %
("data" if data else "", " and " if data and records else "",
"records" if records else ""))
result = event_loop.run_until_complete(
introspect_server(async_client,
bucket=args.bucket,
collection=args.collection,
data=data,
records=records))
yaml_result = yaml.safe_dump(result, default_flow_style=False)
print(yaml_result, end=u'')
elif args.which == 'load':
logger.debug("Start initialization...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
event_loop.run_until_complete(
initialize_server(async_client,
config,
bucket=args.bucket,
collection=args.collection,
force=args.force,
delete_missing_records=args.delete_records))
def main(args=None):
parser = cli_utils.add_parser_options(description="Parse2Kinto importer",
default_bucket=None,
default_collection=None)
parser.add_argument('--parse-server',
help='Parse API server',
type=str,
default=PARSE_DEFAULT_SERVER)
parser.add_argument('--parse-app', help='Your Parse APP ID', type=str)
parser.add_argument('--parse-rest-key',
help='Your Parse APP ID REST API KEY',
type=str)
parser.add_argument('--parse-class',
help='The Parse APP class you want to import',
type=str)
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
parse_client = parse.create_client_from_args(args)
# Create bucket
kinto_client.create_bucket(if_not_exists=True)
# Create collection if doesn't exists yet
# try:
# kinto_client.create_collection(safe=True)
# except KintoException as e:
# if hasattr(e, 'response') and e.response.status_code == 412:
# raise KintoException("The collection already exists. Please "
# "delete it first to make a clean import.")
kinto_client.create_collection(if_not_exists=True)
# Count number of objects
count = parse_client.get_number_of_records()
pages = int(math.ceil(float(count) / RECORD_PER_PAGES))
widgets = [
'Import: ',
Percentage(), ' ',
BouncingBar(), ' ',
AdaptiveETA()
]
print("Importing %d records from %s" % (count, args.parse_class))
num_processed = 0
with ProgressBar(widgets=widgets, max_value=count) as progress:
# Get parse records
progress.update(num_processed)
# Manual batch management mode.
p = kinto_client.batch()
batch = p.__enter__()
send_every = batch.session.batch_max_requests
for page in range(pages):
records = parse_client.get_records(page, RECORD_PER_PAGES)
for record in records:
batch.create_record(data=parse.convert_record(record),
safe=True)
num_processed += 1
if num_processed % send_every == 0:
try:
batch.session.send()
except KintoException as e:
if e.response['status'] != 412:
raise
finally:
batch.session.reset()
progress.update(num_processed)
p.__exit__(None, None, None)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Build a blocklists.xml file from Kinto blocklists.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str, default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str, default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('--api-version', help='Targeted blocklists.xml APP id',
type=int, default=3)
parser.add_argument('--app', help='Targeted blocklists.xml APP id',
type=str, default=constants.FIREFOX_APPID)
parser.add_argument('--app-version', help='The targetted app version',
type=str, default=None)
# Choose where to write the file down.
parser.add_argument('-o', '--out', help='Output XML file.',
type=str, default=None)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
if not args.out:
out_fd = sys.stdout
close_out_fd = False
else:
out_fd = open(args.out, 'w+')
close_out_fd = True
client = cli_utils.create_client_from_args(args)
last_update = 0
# Retrieve the collection of records.
try:
addons_records = client.get_records(
bucket=args.addons_bucket,
collection=args.addons_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.addons_bucket,
collection=args.addons_collection,
)
)
addons_records = []
if addons_records:
last_update = addons_records[-1]['last_modified']
try:
plugin_records = client.get_records(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
)
)
plugin_records = []
if plugin_records:
last_update = max(last_update, plugin_records[-1]['last_modified'])
try:
gfx_records = client.get_records(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
)
)
gfx_records = []
if gfx_records:
last_update = max(last_update, gfx_records[-1]['last_modified'])
try:
cert_records = client.get_records(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
)
)
cert_records = []
if cert_records:
last_update = max(last_update, cert_records[-1]['last_modified'])
xml_tree = etree.Element(
'blocklist',
xmlns="http://www.mozilla.org/2006/addons-blocklist",
lastupdate='%s' % last_update
)
write_addons_items(xml_tree, addons_records,
api_ver=args.api_version,
app_id=args.app)
write_plugin_items(xml_tree, plugin_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_gfx_items(xml_tree, gfx_records,
api_ver=args.api_version,
app_id=args.app)
write_cert_items(xml_tree, cert_records,
api_ver=args.api_version)
doc = etree.ElementTree(xml_tree)
out_fd.write(etree.tostring(
doc,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8').decode('utf-8'))
if close_out_fd:
out_fd.close()
