def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password'))
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def main():
parser = cli_utils.add_parser_options(
description='Upgrade the attachment structure',
default_server=DEFAULT_SERVER)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
upgrade(client)
def main(args=None):
parser = cli_utils.add_parser_options(
description="Bootstrap switchboard bucket and collection",
default_bucket="switchboard",
default_collection="features")
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
client = cli_utils.create_client_from_args(args)
current_version = get_current_version(client)
current_version_str = str(current_version).zfill(3)
next_migration = ('migrate_%s_%s' % (
current_version_str,
str(current_version + 1).zfill(3)))
migrations = sorted([d for d in globals().keys()
if d.startswith('migrate_')])
try:
start_from = migrations.index(next_migration)
except ValueError:
logger.info("Nothing to migrate from version %s" % current_version_str)
else:
for migrate in migrations[start_from:]:
logger.info("Execute %s" % migrate)
globals()[migrate](client, args)
set_current_version(client, migrate.split('_')[-1])
def main():
parser = cli_utils.add_parser_options(
description='Download files from Kinto')
parser.add_argument('-f', '--folder', help='Folder to download files in.',
type=str, default=".")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
# See if timestamp was saved from last run.
last_sync = None
timestamp_file = os.path.join(args.folder, '.last_sync')
if os.path.exists(args.folder):
if os.path.exists(timestamp_file):
last_sync = open(timestamp_file, 'r').read()
else:
os.makedirs(args.folder)
# Retrieve the collection of records.
existing = client.get_records(_since=last_sync, _sort="-last_modified")
if existing:
download_files(client, existing, args.folder)
timestamp = max([r['last_modified'] for r in existing])
# Save the highest timestamp for next runs.
with open(timestamp_file, 'w') as f:
f.write("%s" % timestamp)
def main():
parser = cli_utils.add_parser_options(
description='Upload files to Kinto',
default_server=DEFAULT_SERVER)
parser.add_argument('--gzip', dest='gzip', action='store_true',
help='Gzip files before upload')
parser.add_argument('--keep-filenames', dest='randomize', action='store_false',
help='Do not randomize file IDs on the server')
parser.add_argument('files', metavar='FILE', action='store',
nargs='+')
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
existing = client.get_records()
to_upload = files_to_upload(existing, args.files)
upload_files(client, to_upload, compress=args.gzip,
randomize=args.randomize)
def main():
parser = cli_utils.add_parser_options(
description='Download files from Kinto')
parser.add_argument('-f',
'--folder',
help='Folder to download files in.',
type=str,
default=".")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
# See if timestamp was saved from last run.
last_sync = None
timestamp_file = os.path.join(args.folder, '.last_sync')
if os.path.exists(args.folder):
if os.path.exists(timestamp_file):
last_sync = open(timestamp_file, 'r').read()
else:
os.makedirs(args.folder)
# Retrieve the collection of records.
existing = client.get_records(_since=last_sync, _sort="-last_modified")
if existing:
download_files(client, existing, args.folder)
timestamp = max([r['last_modified'] for r in existing])
# Save the highest timestamp for next runs.
with open(timestamp_file, 'w') as f:
f.write("%s" % timestamp)
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False,
include_collection=False)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def main():
parser = cli_utils.add_parser_options(
description='Upgrade the attachment structure',
default_server=DEFAULT_SERVER)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
upgrade(client)
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password')
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False, include_collection=False
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def main(): # pragma: nocover
args = get_arguments()
cli_utils.setup_logger(logger, args)
origin = Client(
server_url=args.origin_server,
auth=args.origin_auth or args.auth,
bucket=args.origin_bucket or args.bucket,
collection=args.origin_collection or args.collection
)
destination = cli_utils.create_client_from_args(args)
replicate(origin, destination)
def main(args=None):
parser = cli_utils.add_parser_options(
description='Syncs a Kinto DB',
default_collection=None,
default_bucket=None,
default_server=KINTO_SERVER,
default_auth=AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--cert-bucket', help='Bucket name for certificates',
type=str, default=CERT_BUCKET)
parser.add_argument('--cert-collection',
help='Collection name for certificates',
type=str, default=CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=PLUGINS_COLLECTION)
parser.add_argument('-x', '--xml-file', help='XML Source file',
type=str, default=XML_FILE)
parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
type=str, default=SCHEMA_FILE)
parser.add_argument('--no-schema', help='Should we handle schemas',
action="store_true")
parser.add_argument('--with-scrapping', action="store_true",
help='Activate blocklist scrapping on AMO')
parser.add_argument('-C', '--certificates',
help='Only import certificates',
action='store_true')
parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
action='store_true')
parser.add_argument('-A', '--addons', help='Only import addons',
action='store_true')
parser.add_argument('-P', '--plugins', help='Only import plugins',
action='store_true')
args = parser.parse_args(args=args)
# If none of the different "collections" were passed as parameter, then we
# want to import them all.
import_all = not any([
args.certificates,
args.gfx,
args.addons,
args.plugins])
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
# Load the schemas
schemas = {}
if not args.no_schema:
with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
schemas = json.load(f)
# Import certificates
collections = {
# Certificates
'certificates': dict(
fields=CERT_ITEMS_FIELDS,
filename=args.xml_file,
xpath='certItems/*',
kinto_client=kinto_client,
bucket=args.cert_bucket,
collection=args.cert_collection,
schema=schemas.get(args.cert_collection,
schemas.get(CERT_COLLECTION))),
# GFX drivers
'gfx': dict(
fields=GFX_ITEMS_FIELDS,
filename=args.xml_file,
xpath='gfxItems/*',
kinto_client=kinto_client,
bucket=args.gfx_bucket,
collection=args.gfx_collection,
schema=schemas.get(args.gfx_collection,
schemas.get(GFX_COLLECTION))),
# Addons
'addons': dict(
fields=ADDONS_ITEMS_FIELDS,
filename=args.xml_file,
xpath='emItems/*',
kinto_client=kinto_client,
bucket=args.addons_bucket,
collection=args.addons_collection,
schema=schemas.get(args.addons_collection,
schemas.get(ADDONS_COLLECTION)),
with_scrapping=args.with_scrapping),
# Plugins
'plugins': dict(
fields=PLUGINS_ITEMS_FIELDS,
filename=args.xml_file,
xpath='pluginItems/*',
kinto_client=kinto_client,
bucket=args.plugins_bucket,
collection=args.plugins_collection,
schema=schemas.get(args.plugins_collection,
schemas.get(PLUGINS_COLLECTION)),
with_scrapping=args.with_scrapping)}
for collection_type, collection in collections.items():
if getattr(args, collection_type) or import_all:
sync_records(**collection)