Example #1
0
    def test_create_client_from_args_build_a_client(self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth=('user', 'password'))

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket='blocklists',
            collection='certificates')
Example #2
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upgrade the attachment structure',
        default_server=DEFAULT_SERVER)
    args = parser.parse_args()
    client = cli_utils.create_client_from_args(args)
    upgrade(client)
def main(args=None):
    parser = cli_utils.add_parser_options(
        description="Bootstrap switchboard bucket and collection",
        default_bucket="switchboard",
        default_collection="features")

    args = parser.parse_args(args)
    cli_utils.setup_logger(logger, args)
    client = cli_utils.create_client_from_args(args)
    current_version = get_current_version(client)
    current_version_str = str(current_version).zfill(3)

    next_migration = ('migrate_%s_%s' % (
        current_version_str,
        str(current_version + 1).zfill(3)))

    migrations = sorted([d for d in globals().keys()
                         if d.startswith('migrate_')])

    try:
        start_from = migrations.index(next_migration)
    except ValueError:
        logger.info("Nothing to migrate from version %s" % current_version_str)
    else:
        for migrate in migrations[start_from:]:
            logger.info("Execute %s" % migrate)
            globals()[migrate](client, args)
            set_current_version(client, migrate.split('_')[-1])
Example #4
0
def main():
    parser = cli_utils.add_parser_options(
        description='Download files from Kinto')
    parser.add_argument('-f', '--folder', help='Folder to download files in.',
                        type=str, default=".")
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    # See if timestamp was saved from last run.
    last_sync = None
    timestamp_file = os.path.join(args.folder, '.last_sync')
    if os.path.exists(args.folder):
        if os.path.exists(timestamp_file):
            last_sync = open(timestamp_file, 'r').read()
    else:
        os.makedirs(args.folder)

    # Retrieve the collection of records.
    existing = client.get_records(_since=last_sync, _sort="-last_modified")

    if existing:
        download_files(client, existing, args.folder)

        timestamp = max([r['last_modified'] for r in existing])
        # Save the highest timestamp for next runs.
        with open(timestamp_file, 'w') as f:
            f.write("%s" % timestamp)
Example #5
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upload files to Kinto',
        default_server=DEFAULT_SERVER)

    parser.add_argument('--gzip', dest='gzip', action='store_true',
                        help='Gzip files before upload')
    parser.add_argument('--keep-filenames', dest='randomize', action='store_false',
                        help='Do not randomize file IDs on the server')
    parser.add_argument('files', metavar='FILE', action='store',
                        nargs='+')
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    try:
        client.create_bucket(if_not_exists=True)
        client.create_collection(if_not_exists=True)
    except KintoException:
        # Fail silently in case of 403
        pass

    existing = client.get_records()
    to_upload = files_to_upload(existing, args.files)
    upload_files(client, to_upload, compress=args.gzip,
                 randomize=args.randomize)
Example #6
0
def main():
    parser = cli_utils.add_parser_options(
        description='Download files from Kinto')
    parser.add_argument('-f',
                        '--folder',
                        help='Folder to download files in.',
                        type=str,
                        default=".")
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    # See if timestamp was saved from last run.
    last_sync = None
    timestamp_file = os.path.join(args.folder, '.last_sync')
    if os.path.exists(args.folder):
        if os.path.exists(timestamp_file):
            last_sync = open(timestamp_file, 'r').read()
    else:
        os.makedirs(args.folder)

    # Retrieve the collection of records.
    existing = client.get_records(_since=last_sync, _sort="-last_modified")

    if existing:
        download_files(client, existing, args.folder)

        timestamp = max([r['last_modified'] for r in existing])
        # Save the highest timestamp for next runs.
        with open(timestamp_file, 'w') as f:
            f.write("%s" % timestamp)
Example #7
0
    def test_create_client_from_args_default_bucket_and_collection_to_none(
            self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_auth=('user', 'password'),
            include_bucket=False,
            include_collection=False)

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket=None,
            collection=None)
Example #8
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upgrade the attachment structure',
        default_server=DEFAULT_SERVER)
    args = parser.parse_args()
    client = cli_utils.create_client_from_args(args)
    upgrade(client)
Example #9
0
    def test_create_client_from_args_build_a_client(self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth=('user', 'password')
        )

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket='blocklists',
            collection='certificates')
Example #10
0
    def test_create_client_from_args_default_bucket_and_collection_to_none(
            self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_auth=('user', 'password'),
            include_bucket=False, include_collection=False
        )

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket=None,
            collection=None)
Example #11
0
def main():  # pragma: nocover
    args = get_arguments()
    cli_utils.setup_logger(logger, args)

    origin = Client(server_url=args.origin_server,
                    auth=args.origin_auth or args.auth,
                    bucket=args.origin_bucket or args.bucket,
                    collection=args.origin_collection or args.collection)
    destination = cli_utils.create_client_from_args(args)

    replicate(origin, destination)
Example #12
0
def main():  # pragma: nocover
    args = get_arguments()
    cli_utils.setup_logger(logger, args)

    origin = Client(
        server_url=args.origin_server,
        auth=args.origin_auth or args.auth,
        bucket=args.origin_bucket or args.bucket,
        collection=args.origin_collection or args.collection
    )
    destination = cli_utils.create_client_from_args(args)

    replicate(origin, destination)
Example #13
0
def main(args=None):
    parser = cli_utils.add_parser_options(
        description='Syncs a Kinto DB',
        default_collection=None,
        default_bucket=None,
        default_server=KINTO_SERVER,
        default_auth=AUTH,
        include_bucket=False,
        include_collection=False)

    parser.add_argument('--cert-bucket', help='Bucket name for certificates',
                        type=str, default=CERT_BUCKET)

    parser.add_argument('--cert-collection',
                        help='Collection name for certificates',
                        type=str, default=CERT_COLLECTION)

    parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
                        type=str, default=GFX_BUCKET)

    parser.add_argument('--gfx-collection',
                        help='Collection name for gfx',
                        type=str, default=GFX_COLLECTION)

    parser.add_argument('--addons-bucket', help='Bucket name for addons',
                        type=str, default=ADDONS_BUCKET)

    parser.add_argument('--addons-collection',
                        help='Collection name for addon',
                        type=str, default=ADDONS_COLLECTION)

    parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
                        type=str, default=PLUGINS_BUCKET)

    parser.add_argument('--plugins-collection',
                        help='Collection name for plugin',
                        type=str, default=PLUGINS_COLLECTION)

    parser.add_argument('-x', '--xml-file', help='XML Source file',
                        type=str, default=XML_FILE)

    parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
                        type=str, default=SCHEMA_FILE)

    parser.add_argument('--no-schema', help='Should we handle schemas',
                        action="store_true")

    parser.add_argument('--with-scrapping', action="store_true",
                        help='Activate blocklist scrapping on AMO')

    parser.add_argument('-C', '--certificates',
                        help='Only import certificates',
                        action='store_true')

    parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
                        action='store_true')

    parser.add_argument('-A', '--addons', help='Only import addons',
                        action='store_true')

    parser.add_argument('-P', '--plugins', help='Only import plugins',
                        action='store_true')

    args = parser.parse_args(args=args)
    # If none of the different "collections" were passed as parameter, then we
    # want to import them all.
    import_all = not any([
        args.certificates,
        args.gfx,
        args.addons,
        args.plugins])

    cli_utils.setup_logger(logger, args)

    kinto_client = cli_utils.create_client_from_args(args)

    # Load the schemas
    schemas = {}
    if not args.no_schema:
        with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
            schemas = json.load(f)

    # Import certificates
    collections = {
        # Certificates
        'certificates': dict(
            fields=CERT_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='certItems/*',
            kinto_client=kinto_client,
            bucket=args.cert_bucket,
            collection=args.cert_collection,
            schema=schemas.get(args.cert_collection,
                               schemas.get(CERT_COLLECTION))),
        # GFX drivers
        'gfx': dict(
            fields=GFX_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='gfxItems/*',
            kinto_client=kinto_client,
            bucket=args.gfx_bucket,
            collection=args.gfx_collection,
            schema=schemas.get(args.gfx_collection,
                               schemas.get(GFX_COLLECTION))),
        # Addons
        'addons': dict(
            fields=ADDONS_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='emItems/*',
            kinto_client=kinto_client,
            bucket=args.addons_bucket,
            collection=args.addons_collection,
            schema=schemas.get(args.addons_collection,
                               schemas.get(ADDONS_COLLECTION)),
            with_scrapping=args.with_scrapping),
        # Plugins
        'plugins': dict(
            fields=PLUGINS_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='pluginItems/*',
            kinto_client=kinto_client,
            bucket=args.plugins_bucket,
            collection=args.plugins_collection,
            schema=schemas.get(args.plugins_collection,
                               schemas.get(PLUGINS_COLLECTION)),
            with_scrapping=args.with_scrapping)}

    for collection_type, collection in collections.items():
        if getattr(args, collection_type) or import_all:
            sync_records(**collection)