def main():
parser = cli_utils.add_parser_options(
description='Download files from Kinto')
parser.add_argument('-f',
'--folder',
help='Folder to download files in.',
type=str,
default=".")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
# See if timestamp was saved from last run.
last_sync = None
timestamp_file = os.path.join(args.folder, '.last_sync')
if os.path.exists(args.folder):
if os.path.exists(timestamp_file):
last_sync = open(timestamp_file, 'r').read()
else:
os.makedirs(args.folder)
# Retrieve the collection of records.
existing = client.get_records(_since=last_sync, _sort="-last_modified")
if existing:
download_files(client, existing, args.folder)
timestamp = max([r['last_modified'] for r in existing])
# Save the highest timestamp for next runs.
with open(timestamp_file, 'w') as f:
f.write("%s" % timestamp)
def test_get_auth_is_called_by_argparse(self):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates")
args = parser.parse_args(['-a', 'user:password'])
assert args.auth == ('user', 'password')
def main(args=None):
parser = cli_utils.add_parser_options(
description="Bootstrap switchboard bucket and collection",
default_bucket="switchboard",
default_collection="features")
args = parser.parse_args(args)
cli_utils.setup_logger(logger, args)
client = cli_utils.create_client_from_args(args)
current_version = get_current_version(client)
current_version_str = str(current_version).zfill(3)
next_migration = ('migrate_%s_%s' % (
current_version_str,
str(current_version + 1).zfill(3)))
migrations = sorted([d for d in globals().keys()
if d.startswith('migrate_')])
try:
start_from = migrations.index(next_migration)
except ValueError:
logger.info("Nothing to migrate from version %s" % current_version_str)
else:
for migrate in migrations[start_from:]:
logger.info("Execute %s" % migrate)
globals()[migrate](client, args)
set_current_version(client, migrate.split('_')[-1])
def main():
parser = cli_utils.add_parser_options(
description='Download files from Kinto')
parser.add_argument('-f', '--folder', help='Folder to download files in.',
type=str, default=".")
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
# See if timestamp was saved from last run.
last_sync = None
timestamp_file = os.path.join(args.folder, '.last_sync')
if os.path.exists(args.folder):
if os.path.exists(timestamp_file):
last_sync = open(timestamp_file, 'r').read()
else:
os.makedirs(args.folder)
# Retrieve the collection of records.
existing = client.get_records(_since=last_sync, _sort="-last_modified")
if existing:
download_files(client, existing, args.folder)
timestamp = max([r['last_modified'] for r in existing])
# Save the highest timestamp for next runs.
with open(timestamp_file, 'w') as f:
f.write("%s" % timestamp)
def main():
parser = cli_utils.add_parser_options(
description='Upgrade the attachment structure',
default_server=DEFAULT_SERVER)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
upgrade(client)
def test_get_auth_is_called_by_argparse(self):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates")
args = parser.parse_args(['-a', 'user:password'])
assert args.auth == ('user', 'password')
def main():
parser = cli_utils.add_parser_options(
description='Upgrade the attachment structure',
default_server=DEFAULT_SERVER)
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
upgrade(client)
def main():
parser = cli_utils.add_parser_options(
description='Upload files to Kinto',
default_server=DEFAULT_SERVER)
parser.add_argument('--gzip', dest='gzip', action='store_true',
help='Gzip files before upload')
parser.add_argument('--keep-filenames', dest='randomize', action='store_false',
help='Do not randomize file IDs on the server')
parser.add_argument('files', metavar='FILE', action='store',
nargs='+')
args = parser.parse_args()
client = cli_utils.create_client_from_args(args)
try:
client.create_bucket(if_not_exists=True)
client.create_collection(if_not_exists=True)
except KintoException:
# Fail silently in case of 403
pass
existing = client.get_records()
to_upload = files_to_upload(existing, args.files)
upload_files(client, to_upload, compress=args.gzip,
randomize=args.randomize)
def test_add_parser_options_adds_arguments_on_existing_parser(self):
parser = argparse.ArgumentParser(prog="importer")
parser.add_argument('-t', '--type', help='File type',
type=str, default='xml')
parser = cli_utils.add_parser_options(parser)
self.assert_option_strings(parser, ['-t', '--type'],
*ALL_PARAMETERS)
assert len(parser._actions) == 9
def test_add_parser_options_adds_arguments_on_existing_parser(self):
parser = argparse.ArgumentParser(prog="importer")
parser.add_argument('-t',
'--type',
help='File type',
type=str,
default='xml')
parser = cli_utils.add_parser_options(parser)
self.assert_option_strings(parser, ['-t', '--type'], *ALL_PARAMETERS)
assert len(parser._actions) == 9
def test_add_parser_options_can_ignore_bucket_and_collection(self):
parser = cli_utils.add_parser_options(include_bucket=False,
include_collection=False)
parameters = [
['-h', '--help'],
['-s', '--server'],
['-a', '--auth'],
['-v', '--verbose'],
['-q', '--quiet'],
['-D', '--debug'],
]
self.assert_option_strings(parser, *parameters)
assert len(parser._actions) == 6
def test_add_parser_options_can_ignore_bucket_and_collection(self):
parser = cli_utils.add_parser_options(
include_bucket=False, include_collection=False)
parameters = [
['-h', '--help'],
['-s', '--server'],
['-a', '--auth'],
['-v', '--verbose'],
['-q', '--quiet'],
['-D', '--debug'],
]
self.assert_option_strings(parser, *parameters)
assert len(parser._actions) == 6
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password'))
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def test_can_change_default_values(self):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth='user:password')
args = vars(parser.parse_args([]))
assert args == {
'server': 'https://firefox.settings.services.mozilla.com/',
'auth': 'user:password',
'bucket': 'blocklists',
'collection': 'certificates',
'verbosity': None
}
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False,
include_collection=False)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def test_create_client_from_args_build_a_client(self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth=('user', 'password')
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket='blocklists',
collection='certificates')
def test_create_client_from_args_default_bucket_and_collection_to_none(
self, mocked_client):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_auth=('user', 'password'),
include_bucket=False, include_collection=False
)
args = parser.parse_args([])
cli_utils.create_client_from_args(args)
mocked_client.assert_called_with(
server_url='https://firefox.settings.services.mozilla.com/',
auth=('user', 'password'),
bucket=None,
collection=None)
def test_can_change_default_values(self):
parser = cli_utils.add_parser_options(
default_server="https://firefox.settings.services.mozilla.com/",
default_bucket="blocklists",
default_collection="certificates",
default_auth='user:password'
)
args = vars(parser.parse_args([]))
assert args == {
'server': 'https://firefox.settings.services.mozilla.com/',
'auth': 'user:password',
'bucket': 'blocklists',
'collection': 'certificates',
'verbosity': None
}
def main(args=None):
parser = cli_utils.add_parser_options(
description='Syncs a Kinto DB',
default_collection=None,
default_bucket=None,
default_server=KINTO_SERVER,
default_auth=AUTH,
include_bucket=False,
include_collection=False)
parser.add_argument('--cert-bucket', help='Bucket name for certificates',
type=str, default=CERT_BUCKET)
parser.add_argument('--cert-collection',
help='Collection name for certificates',
type=str, default=CERT_COLLECTION)
parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
type=str, default=GFX_BUCKET)
parser.add_argument('--gfx-collection',
help='Collection name for gfx',
type=str, default=GFX_COLLECTION)
parser.add_argument('--addons-bucket', help='Bucket name for addons',
type=str, default=ADDONS_BUCKET)
parser.add_argument('--addons-collection',
help='Collection name for addon',
type=str, default=ADDONS_COLLECTION)
parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
type=str, default=PLUGINS_BUCKET)
parser.add_argument('--plugins-collection',
help='Collection name for plugin',
type=str, default=PLUGINS_COLLECTION)
parser.add_argument('-x', '--xml-file', help='XML Source file',
type=str, default=XML_FILE)
parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
type=str, default=SCHEMA_FILE)
parser.add_argument('--no-schema', help='Should we handle schemas',
action="store_true")
parser.add_argument('--with-scrapping', action="store_true",
help='Activate blocklist scrapping on AMO')
parser.add_argument('-C', '--certificates',
help='Only import certificates',
action='store_true')
parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
action='store_true')
parser.add_argument('-A', '--addons', help='Only import addons',
action='store_true')
parser.add_argument('-P', '--plugins', help='Only import plugins',
action='store_true')
args = parser.parse_args(args=args)
# If none of the different "collections" were passed as parameter, then we
# want to import them all.
import_all = not any([
args.certificates,
args.gfx,
args.addons,
args.plugins])
cli_utils.setup_logger(logger, args)
kinto_client = cli_utils.create_client_from_args(args)
# Load the schemas
schemas = {}
if not args.no_schema:
with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
schemas = json.load(f)
# Import certificates
collections = {
# Certificates
'certificates': dict(
fields=CERT_ITEMS_FIELDS,
filename=args.xml_file,
xpath='certItems/*',
kinto_client=kinto_client,
bucket=args.cert_bucket,
collection=args.cert_collection,
schema=schemas.get(args.cert_collection,
schemas.get(CERT_COLLECTION))),
# GFX drivers
'gfx': dict(
fields=GFX_ITEMS_FIELDS,
filename=args.xml_file,
xpath='gfxItems/*',
kinto_client=kinto_client,
bucket=args.gfx_bucket,
collection=args.gfx_collection,
schema=schemas.get(args.gfx_collection,
schemas.get(GFX_COLLECTION))),
# Addons
'addons': dict(
fields=ADDONS_ITEMS_FIELDS,
filename=args.xml_file,
xpath='emItems/*',
kinto_client=kinto_client,
bucket=args.addons_bucket,
collection=args.addons_collection,
schema=schemas.get(args.addons_collection,
schemas.get(ADDONS_COLLECTION)),
with_scrapping=args.with_scrapping),
# Plugins
'plugins': dict(
fields=PLUGINS_ITEMS_FIELDS,
filename=args.xml_file,
xpath='pluginItems/*',
kinto_client=kinto_client,
bucket=args.plugins_bucket,
collection=args.plugins_collection,
schema=schemas.get(args.plugins_collection,
schemas.get(PLUGINS_COLLECTION)),
with_scrapping=args.with_scrapping)}
for collection_type, collection in collections.items():
if getattr(args, collection_type) or import_all:
sync_records(**collection)
def test_add_parser_options_create_a_parser_if_needed(self):
parser = cli_utils.add_parser_options()
self.assert_option_strings(parser, *ALL_PARAMETERS)
assert len(parser._actions) == 8
def test_add_parser_options_create_a_parser_if_needed(self):
parser = cli_utils.add_parser_options()
self.assert_option_strings(parser, *ALL_PARAMETERS)
assert len(parser._actions) == 8
