def test_create_client_from_args_build_a_client_and_ask_for_password(
self, mocked_client, mocked_getpass):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/")
mocked_getpass.return_value = "password"
args = parser.parse_args([
"--auth",
"user",
"--bucket",
"blocklists",
"--collection",
"certificates",
"--retry",
"3",
])
cli_utils.create_client_from_args(args)
mocked_getpass.assert_called_with("Please enter a password for user: "******"https://firefox.settings.services.mozilla.com/",
auth=("user", "password"),
bucket="blocklists",
collection="certificates",
ignore_batch_4xx=False,
retry=3,
retry_after=None,
)
def test_create_client_from_args_with_basic_bearer_token(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/")
args = parser.parse_args(["--auth", "Bearer Token.Dotted"])
cli_utils.create_client_from_args(args)
assert isinstance(mocked_client.call_args[1]["auth"], BearerTokenAuth)
assert mocked_client.call_args[1]["auth"].type == "Bearer"
assert mocked_client.call_args[1]["auth"].token == "Token.Dotted"
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password'))
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def main():
parser = cli_utils.add_parser_options(
description="Move the keywords from the `upcoming` records to their description",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
print("Fetching records")
videos = client.get_records()
print("Found", len(videos), "videos")
move_keywords_to_description(videos)
print("Updating records")
update_videos(client, videos)
def main():
parser = cli_utils.add_parser_options(description="Upload files to Kinto",
default_server=DEFAULT_SERVER)
parser.add_argument(
"--force",
dest="force",
action="store_true",
help="Force upload even if the hash matches",
)
parser.add_argument("files", metavar="FILE", action="store", nargs="+")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
existing = client.get_records()
to_upload = files_to_upload(existing, args.files, force=args.force)
upload_files(client, to_upload)
def main():
parser = cli_utils.add_parser_options(
description="Change the videos records 'keywords as a single string' to an array of strings",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
print("Fetching records")
videos = client.get_records()
videos_with_string_keywords = [
video for video in videos if type(video["keywords"]) == str
]
print(
"Found",
len(videos_with_string_keywords),
"videos which need their keywords changed to an array",
)
change_keywords_to_array(videos_with_string_keywords)
print("Updating records")
update_videos(client, videos_with_string_keywords)
def run():
loop = asyncio.get_event_loop()
stdin_generator = stream_as_generator(loop, sys.stdin)
records_generator = parse_json(stdin_generator)
parser = cli_utils.add_parser_options(
description='Read records from stdin as JSON and push them to Kinto',
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_retry=NB_RETRY_REQUEST,
default_collection=DEFAULT_COLLECTION)
parser.add_argument('--skip',
action='store_true',
help='Skip records that exist and are equal.')
cli_args = parser.parse_args()
cli_utils.setup_logger(logger, cli_args)
logger.info(
'Publish at {server}/buckets/{bucket}/collections/{collection}'.format(
**cli_args.__dict__))
client = cli_utils.create_client_from_args(cli_args)
main_coro = main(loop,
records_generator,
client,
skip_existing=cli_args.skip)
loop.run_until_complete(main_coro)
loop.close()
def main(args=None):
parser = cli_utils.add_parser_options(
description='Generate blocked item description files.',
default_collection=None,
default_bucket=constants.DESTINATION_BUCKET,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_collection=False)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-d', '--target-dir',
help='Destination directory to write files in.',
type=str, default=constants.TARGET_DIR)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
collections = [
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.addons_collection),
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.plugins_collection),
]
generate(kinto_client, collections, args.target_dir, 'collection.tpl', 'record.tpl')
def main():
parser = cli_utils.add_parser_options(
description=
"Move the thumbnails to attachments in the `thumbnails` collection",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
collections_metadata.update_thumbnails(client)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
print("Fetching records")
videos = client.get_records()
filtered_videos = [
video for video in videos if video["thumbnail"].startswith("data:")
]
print("Found", len(filtered_videos),
"videos, upload their thumbnails as attachments")
upload_thumbnails(client, filtered_videos)
print("Updating records")
update_videos(client, filtered_videos)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Generate blocked item description files.',
default_collection=None,
default_bucket=constants.DESTINATION_BUCKET,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_collection=False)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-d', '--target-dir',
help='Destination directory to write files in.',
type=str, default=constants.TARGET_DIR)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
collections = [
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.addons_collection),
COLLECTION_FORMAT.format(bucket_id=args.bucket,
collection_id=args.plugins_collection),
]
generate(kinto_client, collections, args.target_dir, 'collection.tpl', 'record.tpl')
def main():
parser = cli_utils.add_parser_options(description='Upload files to Kinto',
default_server=DEFAULT_SERVER)
parser.add_argument('--gzip',
dest='gzip',
action='store_true',
help='Gzip files before upload')
parser.add_argument('--keep-filenames',
dest='randomize',
action='store_false',
help='Do not randomize file IDs on the server')
parser.add_argument('--force',
dest='force',
action='store_true',
help='Force upload even if the hash matches')
parser.add_argument('files', metavar='FILE', action='store', nargs='+')
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
existing = client.get_records()
to_upload = files_to_upload(existing, args.files, force=args.force)
upload_files(client,
to_upload,
compress=args.gzip,
randomize=args.randomize)
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password')
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False,
include_collection=False)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False, include_collection=False
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def main():
parser = cli_utils.add_parser_options(
description='Download files from Kinto')
parser.add_argument('-f', '--folder', help='Folder to download files in.',
type=str, default=".")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
# See if timestamp was saved from last run.
last_sync = None
timestamp_file = os.path.join(args.folder, '.last_sync')
if os.path.exists(args.folder):
if os.path.exists(timestamp_file):
last_sync = open(timestamp_file, 'r').read()
else:
os.makedirs(args.folder)
# Retrieve the collection of records.
existing = client.get_records(_since=last_sync, _sort="-last_modified")
if existing:
download_files(client, existing, args.folder)
timestamp = max([r['last_modified'] for r in existing])
# Save the highest timestamp for next runs.
with open(timestamp_file, 'w') as f:
f.write("%s" % timestamp)
def main():
parser = cli_utils.add_parser_options(
description="Add a creation_date field to the `upcoming` records",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
print("Fetching records")
videos = client.get_records()
print("Found", len(videos),
"videos, use last_modified date for the creation_date")
add_creation_date(videos)
print("Updating records")
update_videos(client, videos)
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/", )
args = parser.parse_args([
'--auth', 'user:password', '--bucket', 'blocklists',
'--collection', 'certificates', '--retry', '3'
])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates',
retry=3,
retry_after=None)
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
)
args = parser.parse_args(['--auth', 'user:password',
'--bucket', 'blocklists',
'--collection', 'certificates',
'--retry', '3'])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates',
retry=3,
retry_after=None)
def main():
parser = cli_utils.add_parser_options(
description='Upgrade the attachment structure',
default_server=DEFAULT_SERVER,
default_bucket='fennec',
default_collection='experiments')
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
push(client)
def main():
parser = argparse.ArgumentParser(
description="Wizard to setup Kinto with YAML")
subparsers = parser.add_subparsers(title='subcommand',
description='Load/Dump',
dest='subcommand',
help="Choose and run with --help")
subparsers.required = True
# load sub-command.
subparser = subparsers.add_parser('load')
subparser.set_defaults(which='load')
cli_utils.add_parser_options(subparser)
subparser.add_argument(dest='filepath', help='YAML file')
subparser.add_argument(
'--force',
help='Load the file using the CLIENT_WINS conflict resolution strategy',
action='store_true')
# dump sub-command.
subparser = subparsers.add_parser('dump')
subparser.set_defaults(which='dump')
cli_utils.add_parser_options(subparser)
subparser.add_argument('--full', help='Full output', action='store_true')
# Parse CLI args.
args = parser.parse_args()
cli_utils.setup_logger(logger, args)
kinto_logger = logging.getLogger('kinto_http')
cli_utils.setup_logger(kinto_logger, args)
logger.debug("Instantiate Kinto client.")
client = cli_utils.create_client_from_args(args)
# Run chosen subcommand.
if args.which == 'dump':
logger.debug("Start %sintrospection..." %
("full " if args.full else ""))
result = introspect_server(client,
bucket=args.bucket,
collection=args.collection,
full=args.full)
yaml_result = yaml.safe_dump(result, default_flow_style=False)
print(yaml_result, end=u'')
elif args.which == 'load':
logger.debug("Start initialization...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
initialize_server(client,
config,
bucket=args.bucket,
collection=args.collection,
force=args.force)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=("user", "password"),
include_bucket=False,
include_collection=False,
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url="https://firefox.settings.services.mozilla.com/",
auth=("user", "password"),
bucket=None,
collection=None,
ignore_batch_4xx=False,
retry=0,
retry_after=None,
)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(
server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection
)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def main():
parser = cli_utils.add_parser_options(
description="Update the 'videos' collection to accept new keywords",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
print("Updating the collection")
collections_metadata.update_videos(client)
def main():
parser = cli_utils.add_parser_options(
description="Change the location of the `upcoming` records attachments",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
default_collection=DEFAULT_COLLECTION,
)
parser.add_argument(
"--pg",
dest="connection_string",
help="""Connection string to the postgresql database:
"host=<host> dbname=<dbname> user=<username> password=<password>"
""",
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
print("Fetching records")
videos = client.get_records()
print("Found", len(videos), "videos, change their attachment's location")
change_location(videos)
# Connect to an existing database
conn = psycopg2.connect(args.connection_string)
# Open a cursor to perform database operations
cursor = conn.cursor()
print("Updating records")
update_videos(client, cursor, videos)
# Make the changes to the database persistent
conn.commit()
# Close communication with the database
cursor.close()
conn.close()
def main(args=None):
parser = cli_utils.add_parser_options(
description='Validate collection signature',
default_server=DEFAULT_SERVER,
default_bucket=DEST_BUCKET,
default_collection=DEST_COLLECTION)
args = parser.parse_args(args)
client = cli_utils.create_client_from_args(args)
# 1. Grab collection information
dest_col = client.get_collection()
# 2. Grab records
records = list(client.get_records(_sort='-last_modified'))
timestamp = client.get_records_timestamp()
# 3. Serialize
serialized = canonical_json(records, timestamp)
# 4. Compute the hash
computed_hash = compute_hash(serialized)
# 5. Grab the signature
signature = dest_col['data']['signature']
# 6. Grab the public key
with open('pub', 'w') as f:
f.write(signature['public_key'])
# 7. Verify the signature matches the hash
signer = ECDSASigner(public_key='pub')
try:
signer.verify(serialized, signature)
print('Signature OK')
except Exception:
print('Signature KO. Computed hash: %s' % computed_hash)
raise
def main():
parser = cli_utils.add_parser_options(
description="Update the collections metadata",
default_server=DEFAULT_SERVER,
default_bucket=DEFAULT_BUCKET,
)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
print("Updating the `upcoming` collection")
collections_metadata.update_upcoming(client)
print("Updating the `videos` collection")
collections_metadata.update_videos(client)
print("Updating the `contacts` collection")
collections_metadata.update_contacts(client)
print("Updating the `thumbnails` collection")
collections_metadata.update_thumbnails(client)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Validate collection signature',
default_server=DEFAULT_SERVER,
default_bucket=DEST_BUCKET,
default_collection=DEST_COLLECTION)
args = parser.parse_args(args)
client = cli_utils.create_client_from_args(args)
# 1. Grab collection information
dest_col = client.get_collection()
# 2. Grab records
records = list(client.get_records(_sort='-last_modified'))
timestamp = client.get_records_timestamp()
# 3. Serialize
serialized = canonical_json(records, timestamp)
# 4. Compute the hash
computed_hash = compute_hash(serialized)
# 5. Grab the signature
signature = dest_col['data']['signature']
# 6. Grab the public key
with open('pub', 'w') as f:
f.write(signature['public_key'])
# 7. Verify the signature matches the hash
signer = ECDSASigner(public_key='pub')
try:
signer.verify(serialized, signature)
print('Signature OK')
except Exception:
print('Signature KO. Computed hash: %s' % computed_hash)
raise
def main():
# Setup parser
parser = argparse.ArgumentParser(
description='Migrate Kinto databases from specification files.'
)
parser.add_argument('-f', '--file',
help='File describing Kinto resources.')
cli_utils.add_parser_options(parser)
args = parser.parse_args()
# Setup logging
logger = logging.getLogger(__name__)
cli_utils.setup_logger(logger, args)
# Create client
client = cli_utils.create_client_from_args(args)
# Load models file
models = yaml.load(open(args.file or 'models.yml'))
# Run migration
migrate_model(client, models)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Import the blocklists from the addons server into Kinto.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
type=str, default=constants.SCHEMA_FILE)
parser.add_argument('--no-schema', help='Should we handle schemas',
action="store_true")
parser.add_argument('--editor-auth',
help='Credentials to be used for requesting a review',
type=str, default=None)
parser.add_argument('--reviewer-auth',
help='Credentials to be used for validating the review',
type=str, default=None)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str, default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str, default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('-C', '--certificates',
help='Only import certificates',
action='store_true')
parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
action='store_true')
parser.add_argument('-A', '--addons', help='Only import addons',
action='store_true')
parser.add_argument('-P', '--plugins', help='Only import plugins',
action='store_true')
# Addons Server selection
parser.add_argument('--addons-server',
help='The addons server to import from',
type=str, default=constants.ADDONS_SERVER)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
editor_client = None
if args.editor_auth is not None:
args.editor_auth = tuple(args.editor_auth.split(':', 1))
editor_client = Client(server_url=args.server,
auth=args.editor_auth)
reviewer_client = None
if args.reviewer_auth is not None:
args.reviewer_auth = tuple(args.reviewer_auth.split(':', 1))
reviewer_client = Client(server_url=args.server,
auth=args.reviewer_auth)
# If none of the different "collections" were passed as parameter, then we
# want to import them all.
import_all = not any([
args.certificates,
args.gfx,
args.addons,
args.plugins])
# Check if the schema capability is activated
body, headers = kinto_client.session.request('get', '/')
if 'schema' not in body.get('capabilities', {}):
logger.warn('\t --- Server schema validation disabled --- \n')
logger.warn("The server at {} won't validate the records against "
"the collection JSON schema. More information "
"http://kinto.readthedocs.io/en/stable/api/1.x/"
"collections.html?highlight=json%20validation"
"#collection-json-schema\n".format(args.server))
# Load the schemas
schemas = {}
if not args.no_schema:
with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
schemas = json.load(f)['collections']
blocklists_url = urljoin(args.addons_server, '/blocked/blocklists.json')
resp = requests.get(blocklists_url)
resp.raise_for_status()
blocklists = resp.json()
for collection_type, records in six.iteritems(blocklists):
collection_type = collection_type.replace('-', '')
if hasattr(args, collection_type) and (
getattr(args, collection_type) or import_all):
bucket = getattr(args, '%s_bucket' % collection_type)
collection = getattr(args, '%s_collection' % collection_type)
config = None
if collection_type in schemas:
config = schemas[collection_type]['config']
sync_records(amo_records=records,
fields=FIELDS[collection_type],
kinto_client=kinto_client,
editor_client=editor_client,
reviewer_client=reviewer_client,
bucket=bucket,
collection=collection,
config=config,
permissions=constants.COLLECTION_PERMISSIONS)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Build a blocklists.xml file from Kinto blocklists.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str,
default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str,
default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket',
help='Bucket name for gfx',
type=str,
default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str,
default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket',
help='Bucket name for addons',
type=str,
default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str,
default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket',
help='Bucket name for plugins',
type=str,
default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str,
default=constants.PLUGINS_COLLECTION)
parser.add_argument('--api-version',
help='Targeted blocklists.xml APP id',
type=int,
default=3)
parser.add_argument('--app',
help='Targeted blocklists.xml APP id',
type=str,
default=constants.FIREFOX_APPID)
parser.add_argument('--app-version',
help='The targetted app version',
type=str,
default=None)
# Choose where to write the file down.
parser.add_argument('-o',
'--out',
help='Output XML file.',
type=str,
default=None)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
if not args.out:
out_fd = sys.stdout
close_out_fd = False
else:
out_fd = open(args.out, 'w+')
close_out_fd = True
client = cli_utils.create_client_from_args(args)
last_update = 0
# Retrieve the collection of records.
try:
addons_records = client.get_records(bucket=args.addons_bucket,
collection=args.addons_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.addons_bucket,
collection=args.addons_collection,
))
addons_records = []
if addons_records:
last_update = addons_records[-1]['last_modified']
try:
plugin_records = client.get_records(bucket=args.plugins_bucket,
collection=args.plugins_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
))
plugin_records = []
if plugin_records:
last_update = max(last_update, plugin_records[-1]['last_modified'])
try:
gfx_records = client.get_records(bucket=args.gfx_bucket,
collection=args.gfx_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
))
gfx_records = []
if gfx_records:
last_update = max(last_update, gfx_records[-1]['last_modified'])
cert_records = None
if should_include_certs(args.app, args.app_version):
try:
cert_records = client.get_records(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
enabled=True,
_sort="last_modified")
except Exception:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.
format(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
))
cert_records = []
if cert_records:
last_update = max(last_update, cert_records[-1]['last_modified'])
xml_tree = etree.Element(
'blocklist',
xmlns="http://www.mozilla.org/2006/addons-blocklist",
lastupdate='%s' % last_update)
write_addons_items(xml_tree,
addons_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_plugin_items(xml_tree,
plugin_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_gfx_items(xml_tree,
gfx_records,
api_ver=args.api_version,
app_id=args.app)
write_cert_items(xml_tree,
cert_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
doc = etree.ElementTree(xml_tree)
out_fd.write(
etree.tostring(doc,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8').decode('utf-8'))
if close_out_fd:
out_fd.close()
def main(args=None):
parser = cli_utils.add_parser_options(
description='Build a blocklists.xml file from Kinto blocklists.',
default_collection=None,
default_bucket=None,
default_server=constants.KINTO_SERVER,
default_auth=constants.AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--certificates-bucket',
help='Bucket name for certificates',
type=str, default=constants.CERT_BUCKET)
parser.add_argument('--certificates-collection',
help='Collection name for certificates',
type=str, default=constants.CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=constants.GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=constants.GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=constants.ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=constants.ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=constants.PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=constants.PLUGINS_COLLECTION)
parser.add_argument('--api-version', help='Targeted blocklists.xml APP id',
type=int, default=3)
parser.add_argument('--app', help='Targeted blocklists.xml APP id',
type=str, default=constants.FIREFOX_APPID)
parser.add_argument('--app-version', help='The targetted app version',
type=str, default=None)
# Choose where to write the file down.
parser.add_argument('-o', '--out', help='Output XML file.',
type=str, default=None)
args = parser.parse_args(args=args)
cli_utils.setup_logger(logger, args)
if not args.out:
out_fd = sys.stdout
close_out_fd = False
else:
out_fd = open(args.out, 'w+')
close_out_fd = True
client = cli_utils.create_client_from_args(args)
last_update = 0
# Retrieve the collection of records.
try:
addons_records = client.get_records(
bucket=args.addons_bucket,
collection=args.addons_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.addons_bucket,
collection=args.addons_collection,
)
)
addons_records = []
if addons_records:
last_update = addons_records[-1]['last_modified']
try:
plugin_records = client.get_records(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.plugins_bucket,
collection=args.plugins_collection,
)
)
plugin_records = []
if plugin_records:
last_update = max(last_update, plugin_records[-1]['last_modified'])
try:
gfx_records = client.get_records(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.gfx_bucket,
collection=args.gfx_collection,
)
)
gfx_records = []
if gfx_records:
last_update = max(last_update, gfx_records[-1]['last_modified'])
try:
cert_records = client.get_records(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
enabled=True,
_sort="last_modified")
except:
logger.warn(
'Unable to fetch the ``{bucket}/{collection}`` records.'.format(
bucket=args.certificates_bucket,
collection=args.certificates_collection,
)
)
cert_records = []
if cert_records:
last_update = max(last_update, cert_records[-1]['last_modified'])
xml_tree = etree.Element(
'blocklist',
xmlns="http://www.mozilla.org/2006/addons-blocklist",
lastupdate='%s' % last_update
)
write_addons_items(xml_tree, addons_records,
api_ver=args.api_version,
app_id=args.app)
write_plugin_items(xml_tree, plugin_records,
api_ver=args.api_version,
app_id=args.app,
app_ver=args.app_version)
write_gfx_items(xml_tree, gfx_records,
api_ver=args.api_version,
app_id=args.app)
write_cert_items(xml_tree, cert_records,
api_ver=args.api_version)
doc = etree.ElementTree(xml_tree)
out_fd.write(etree.tostring(
doc,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8').decode('utf-8'))
if close_out_fd:
out_fd.close()
def main(args=None):
parser = cli_utils.add_parser_options(description="Parse2Kinto importer",
default_bucket=None,
default_collection=None)
parser.add_argument('--parse-server',
help='Parse API server',
type=str,
default=PARSE_DEFAULT_SERVER)
parser.add_argument('--parse-app', help='Your Parse APP ID', type=str)
parser.add_argument('--parse-rest-key',
help='Your Parse APP ID REST API KEY',
type=str)
parser.add_argument('--parse-class',
help='The Parse APP class you want to import',
type=str)
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
parse_client = parse.create_client_from_args(args)
# Create bucket
kinto_client.create_bucket(if_not_exists=True)
# Create collection if doesn't exists yet
# try:
# kinto_client.create_collection(safe=True)
# except KintoException as e:
# if hasattr(e, 'response') and e.response.status_code == 412:
# raise KintoException("The collection already exists. Please "
# "delete it first to make a clean import.")
kinto_client.create_collection(if_not_exists=True)
# Count number of objects
count = parse_client.get_number_of_records()
pages = int(math.ceil(float(count) / RECORD_PER_PAGES))
widgets = [
'Import: ',
Percentage(), ' ',
BouncingBar(), ' ',
AdaptiveETA()
]
print("Importing %d records from %s" % (count, args.parse_class))
num_processed = 0
with ProgressBar(widgets=widgets, max_value=count) as progress:
# Get parse records
progress.update(num_processed)
# Manual batch management mode.
p = kinto_client.batch()
batch = p.__enter__()
send_every = batch.session.batch_max_requests
for page in range(pages):
records = parse_client.get_records(page, RECORD_PER_PAGES)
for record in records:
batch.create_record(data=parse.convert_record(record),
safe=True)
num_processed += 1
if num_processed % send_every == 0:
try:
batch.session.send()
except KintoException as e:
if e.response['status'] != 412:
raise
finally:
batch.session.reset()
progress.update(num_processed)
p.__exit__(None, None, None)
def main(args=None):
parser = cli_utils.add_parser_options(
description="Validate kinto-emailer setup",
default_server=DEFAULT_SERVER,
default_bucket=BUCKET,
default_collection=COLLECTION)
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
client = cli_utils.create_client_from_args(args)
# 1. Check capabilities
capabilities = client.server_info()["capabilities"]
if "emailer" not in capabilities:
print("Server doesn't have support for email notifications: \x1b[1;31m KO \x1b[0m")
return 2
print("Clear inbox")
clear_inbox(MAILBOX)
# 2. Configure collection metadata.
print("Configure emailing in %r collection" % args.collection)
setup_notifs_signing(client, RECIPIENT)
# 3. Create a record.
print("Create a dummy record")
client.create_record(data={"script": "validate kinto-emailer setup"})
# 4. If signing is enabled for this collection, then see integration with kinto-signer.
signed_resources = capabilities.get("signer", {"resources": []})["resources"]
ids = [(r["source"]["bucket"], r["source"]["collection"]) for r in signed_resources]
signing_enabled = (args.bucket, args.collection) in ids
if signing_enabled:
# 4. Request a review.
client.patch_collection(data={"status": "to-review"})
# 5. Check that emails were received.
print("Scan inbox...")
first_try = time.time()
while True:
emails = fetch_emails(MAILBOX)
subjects = [e["subject"] for e in emails]
if SUBJECT_EMAIL_CREATION not in subjects:
if time.time() - first_try < RETRY_TIMEOUT:
time.sleep(3)
sys.stdout.write('.')
sys.stdout.flush()
continue
break
if SUBJECT_EMAIL_CREATION in subjects:
print("Email received:\x1b[1;32m OK \x1b[0m")
if signing_enabled:
if "requested review" in subjects:
print("Review notification received:\x1b[1;32m OK \x1b[0m")
else:
print("Review notification not received:\x1b[1;31m KO \x1b[0m")
else:
print("Email not received:\x1b[1;31m KO \x1b[0m")
return 0
def main():
parser = argparse.ArgumentParser(
description="Wizard to setup Kinto with YAML")
subparsers = parser.add_subparsers(title='subcommand',
description='Load/Dump/Validate',
dest='subcommand',
help="Choose and run with --help")
subparsers.required = True
# load sub-command.
subparser = subparsers.add_parser('load')
subparser.set_defaults(which='load')
cli_utils.add_parser_options(subparser)
subparser.add_argument(dest='filepath', help='YAML file')
subparser.add_argument(
'--force',
help='Load the file using the CLIENT_WINS conflict resolution strategy',
action='store_true')
subparser.add_argument('--dry-run',
help="Do not apply write call to the server",
action='store_true')
subparser.add_argument('--delete-records',
help='Delete records that are not in the file.',
action='store_true')
# dump sub-command.
subparser = subparsers.add_parser('dump')
subparser.set_defaults(which='dump')
cli_utils.add_parser_options(subparser)
subparser.add_argument(
'--full',
help='Full output (same as with both --data and --records options)',
action='store_true')
subparser.add_argument('--data',
help='Export buckets, collections and groups data',
action='store_true')
subparser.add_argument('--records',
help="Export collections' records",
action='store_true')
# validate sub-command.
subparser = subparsers.add_parser('validate')
subparser.set_defaults(which='validate')
subparser.set_defaults(verbosity=logging.INFO)
subparser.add_argument(dest='filepath', help='YAML file to validate')
cli_utils.add_parser_options(subparser)
# Parse CLI args.
args = parser.parse_args()
cli_utils.setup_logger(logger, args)
kinto_logger = logging.getLogger('kinto_http')
cli_utils.setup_logger(kinto_logger, args)
if args.which == 'validate':
logger.debug("Start validation...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
logger.info("File loaded!")
fine = validate_export(config)
sys.exit(0 if fine else 1)
logger.debug("Instantiate Kinto client.")
client = cli_utils.create_client_from_args(args)
thread_pool = ThreadPoolExecutor()
event_loop = asyncio.get_event_loop()
async_client = AsyncKintoClient(client,
event_loop,
thread_pool,
dry_run=getattr(args, 'dry_run', False))
# Run chosen subcommand.
if args.which == 'dump':
if args.full:
data = True
records = True
else:
data = args.data
records = args.records
logger.debug(
"Start introspection with %s%s%s..." %
("data" if data else "", " and " if data and records else "",
"records" if records else ""))
result = event_loop.run_until_complete(
introspect_server(async_client,
bucket=args.bucket,
collection=args.collection,
data=data,
records=records))
yaml_result = yaml.safe_dump(result, default_flow_style=False)
print(yaml_result, end=u'')
elif args.which == 'load':
logger.debug("Start initialization...")
logger.info("Load YAML file {!r}".format(args.filepath))
with open(args.filepath, 'r') as f:
config = yaml.safe_load(f)
event_loop.run_until_complete(
initialize_server(async_client,
config,
bucket=args.bucket,
collection=args.collection,
force=args.force,
delete_missing_records=args.delete_records))
