Exemplo n.º 1
0
def main():
    parser = cli_utils.add_parser_options(
        description='Download files from Kinto')
    parser.add_argument('-f',
                        '--folder',
                        help='Folder to download files in.',
                        type=str,
                        default=".")
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    # See if timestamp was saved from last run.
    last_sync = None
    timestamp_file = os.path.join(args.folder, '.last_sync')
    if os.path.exists(args.folder):
        if os.path.exists(timestamp_file):
            last_sync = open(timestamp_file, 'r').read()
    else:
        os.makedirs(args.folder)

    # Retrieve the collection of records.
    existing = client.get_records(_since=last_sync, _sort="-last_modified")

    if existing:
        download_files(client, existing, args.folder)

        timestamp = max([r['last_modified'] for r in existing])
        # Save the highest timestamp for next runs.
        with open(timestamp_file, 'w') as f:
            f.write("%s" % timestamp)
Exemplo n.º 2
0
 def test_get_auth_is_called_by_argparse(self):
     parser = cli_utils.add_parser_options(
         default_server="https://firefox.settings.services.mozilla.com/",
         default_bucket="blocklists",
         default_collection="certificates")
     args = parser.parse_args(['-a', 'user:password'])
     assert args.auth == ('user', 'password')
def main(args=None):
    parser = cli_utils.add_parser_options(
        description="Bootstrap switchboard bucket and collection",
        default_bucket="switchboard",
        default_collection="features")

    args = parser.parse_args(args)
    cli_utils.setup_logger(logger, args)
    client = cli_utils.create_client_from_args(args)
    current_version = get_current_version(client)
    current_version_str = str(current_version).zfill(3)

    next_migration = ('migrate_%s_%s' % (
        current_version_str,
        str(current_version + 1).zfill(3)))

    migrations = sorted([d for d in globals().keys()
                         if d.startswith('migrate_')])

    try:
        start_from = migrations.index(next_migration)
    except ValueError:
        logger.info("Nothing to migrate from version %s" % current_version_str)
    else:
        for migrate in migrations[start_from:]:
            logger.info("Execute %s" % migrate)
            globals()[migrate](client, args)
            set_current_version(client, migrate.split('_')[-1])
Exemplo n.º 4
0
def main():
    parser = cli_utils.add_parser_options(
        description='Download files from Kinto')
    parser.add_argument('-f', '--folder', help='Folder to download files in.',
                        type=str, default=".")
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    # See if timestamp was saved from last run.
    last_sync = None
    timestamp_file = os.path.join(args.folder, '.last_sync')
    if os.path.exists(args.folder):
        if os.path.exists(timestamp_file):
            last_sync = open(timestamp_file, 'r').read()
    else:
        os.makedirs(args.folder)

    # Retrieve the collection of records.
    existing = client.get_records(_since=last_sync, _sort="-last_modified")

    if existing:
        download_files(client, existing, args.folder)

        timestamp = max([r['last_modified'] for r in existing])
        # Save the highest timestamp for next runs.
        with open(timestamp_file, 'w') as f:
            f.write("%s" % timestamp)
Exemplo n.º 5
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upgrade the attachment structure',
        default_server=DEFAULT_SERVER)
    args = parser.parse_args()
    client = cli_utils.create_client_from_args(args)
    upgrade(client)
Exemplo n.º 6
0
 def test_get_auth_is_called_by_argparse(self):
     parser = cli_utils.add_parser_options(
         default_server="https://firefox.settings.services.mozilla.com/",
         default_bucket="blocklists",
         default_collection="certificates")
     args = parser.parse_args(['-a', 'user:password'])
     assert args.auth == ('user', 'password')
Exemplo n.º 7
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upgrade the attachment structure',
        default_server=DEFAULT_SERVER)
    args = parser.parse_args()
    client = cli_utils.create_client_from_args(args)
    upgrade(client)
Exemplo n.º 8
0
def main():
    parser = cli_utils.add_parser_options(
        description='Upload files to Kinto',
        default_server=DEFAULT_SERVER)

    parser.add_argument('--gzip', dest='gzip', action='store_true',
                        help='Gzip files before upload')
    parser.add_argument('--keep-filenames', dest='randomize', action='store_false',
                        help='Do not randomize file IDs on the server')
    parser.add_argument('files', metavar='FILE', action='store',
                        nargs='+')
    args = parser.parse_args()

    client = cli_utils.create_client_from_args(args)

    try:
        client.create_bucket(if_not_exists=True)
        client.create_collection(if_not_exists=True)
    except KintoException:
        # Fail silently in case of 403
        pass

    existing = client.get_records()
    to_upload = files_to_upload(existing, args.files)
    upload_files(client, to_upload, compress=args.gzip,
                 randomize=args.randomize)
Exemplo n.º 9
0
    def test_add_parser_options_adds_arguments_on_existing_parser(self):
        parser = argparse.ArgumentParser(prog="importer")
        parser.add_argument('-t', '--type', help='File type',
                            type=str, default='xml')

        parser = cli_utils.add_parser_options(parser)
        self.assert_option_strings(parser, ['-t', '--type'],
                                   *ALL_PARAMETERS)
        assert len(parser._actions) == 9
Exemplo n.º 10
0
    def test_add_parser_options_adds_arguments_on_existing_parser(self):
        parser = argparse.ArgumentParser(prog="importer")
        parser.add_argument('-t',
                            '--type',
                            help='File type',
                            type=str,
                            default='xml')

        parser = cli_utils.add_parser_options(parser)
        self.assert_option_strings(parser, ['-t', '--type'], *ALL_PARAMETERS)
        assert len(parser._actions) == 9
Exemplo n.º 11
0
 def test_add_parser_options_can_ignore_bucket_and_collection(self):
     parser = cli_utils.add_parser_options(include_bucket=False,
                                           include_collection=False)
     parameters = [
         ['-h', '--help'],
         ['-s', '--server'],
         ['-a', '--auth'],
         ['-v', '--verbose'],
         ['-q', '--quiet'],
         ['-D', '--debug'],
     ]
     self.assert_option_strings(parser, *parameters)
     assert len(parser._actions) == 6
Exemplo n.º 12
0
 def test_add_parser_options_can_ignore_bucket_and_collection(self):
     parser = cli_utils.add_parser_options(
         include_bucket=False, include_collection=False)
     parameters = [
         ['-h', '--help'],
         ['-s', '--server'],
         ['-a', '--auth'],
         ['-v', '--verbose'],
         ['-q', '--quiet'],
         ['-D', '--debug'],
     ]
     self.assert_option_strings(parser, *parameters)
     assert len(parser._actions) == 6
Exemplo n.º 13
0
    def test_create_client_from_args_build_a_client(self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth=('user', 'password'))

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket='blocklists',
            collection='certificates')
Exemplo n.º 14
0
    def test_can_change_default_values(self):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth='user:password')

        args = vars(parser.parse_args([]))

        assert args == {
            'server': 'https://firefox.settings.services.mozilla.com/',
            'auth': 'user:password',
            'bucket': 'blocklists',
            'collection': 'certificates',
            'verbosity': None
        }
Exemplo n.º 15
0
    def test_create_client_from_args_default_bucket_and_collection_to_none(
            self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_auth=('user', 'password'),
            include_bucket=False,
            include_collection=False)

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket=None,
            collection=None)
Exemplo n.º 16
0
    def test_create_client_from_args_build_a_client(self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth=('user', 'password')
        )

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket='blocklists',
            collection='certificates')
Exemplo n.º 17
0
    def test_create_client_from_args_default_bucket_and_collection_to_none(
            self, mocked_client):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_auth=('user', 'password'),
            include_bucket=False, include_collection=False
        )

        args = parser.parse_args([])

        cli_utils.create_client_from_args(args)
        mocked_client.assert_called_with(
            server_url='https://firefox.settings.services.mozilla.com/',
            auth=('user', 'password'),
            bucket=None,
            collection=None)
Exemplo n.º 18
0
    def test_can_change_default_values(self):
        parser = cli_utils.add_parser_options(
            default_server="https://firefox.settings.services.mozilla.com/",
            default_bucket="blocklists",
            default_collection="certificates",
            default_auth='user:password'
        )

        args = vars(parser.parse_args([]))

        assert args == {
            'server': 'https://firefox.settings.services.mozilla.com/',
            'auth': 'user:password',
            'bucket': 'blocklists',
            'collection': 'certificates',
            'verbosity': None
        }
Exemplo n.º 19
0
def main(args=None):
    parser = cli_utils.add_parser_options(
        description='Syncs a Kinto DB',
        default_collection=None,
        default_bucket=None,
        default_server=KINTO_SERVER,
        default_auth=AUTH,
        include_bucket=False,
        include_collection=False)

    parser.add_argument('--cert-bucket', help='Bucket name for certificates',
                        type=str, default=CERT_BUCKET)

    parser.add_argument('--cert-collection',
                        help='Collection name for certificates',
                        type=str, default=CERT_COLLECTION)

    parser.add_argument('--gfx-bucket', help='Bucket name for gfx',
                        type=str, default=GFX_BUCKET)

    parser.add_argument('--gfx-collection',
                        help='Collection name for gfx',
                        type=str, default=GFX_COLLECTION)

    parser.add_argument('--addons-bucket', help='Bucket name for addons',
                        type=str, default=ADDONS_BUCKET)

    parser.add_argument('--addons-collection',
                        help='Collection name for addon',
                        type=str, default=ADDONS_COLLECTION)

    parser.add_argument('--plugins-bucket', help='Bucket name for plugins',
                        type=str, default=PLUGINS_BUCKET)

    parser.add_argument('--plugins-collection',
                        help='Collection name for plugin',
                        type=str, default=PLUGINS_COLLECTION)

    parser.add_argument('-x', '--xml-file', help='XML Source file',
                        type=str, default=XML_FILE)

    parser.add_argument('-S', '--schema-file', help='JSON Schemas file',
                        type=str, default=SCHEMA_FILE)

    parser.add_argument('--no-schema', help='Should we handle schemas',
                        action="store_true")

    parser.add_argument('--with-scrapping', action="store_true",
                        help='Activate blocklist scrapping on AMO')

    parser.add_argument('-C', '--certificates',
                        help='Only import certificates',
                        action='store_true')

    parser.add_argument('-G', '--gfx', help='Only import GFX drivers',
                        action='store_true')

    parser.add_argument('-A', '--addons', help='Only import addons',
                        action='store_true')

    parser.add_argument('-P', '--plugins', help='Only import plugins',
                        action='store_true')

    args = parser.parse_args(args=args)
    # If none of the different "collections" were passed as parameter, then we
    # want to import them all.
    import_all = not any([
        args.certificates,
        args.gfx,
        args.addons,
        args.plugins])

    cli_utils.setup_logger(logger, args)

    kinto_client = cli_utils.create_client_from_args(args)

    # Load the schemas
    schemas = {}
    if not args.no_schema:
        with codecs.open(args.schema_file, 'r', encoding='utf-8') as f:
            schemas = json.load(f)

    # Import certificates
    collections = {
        # Certificates
        'certificates': dict(
            fields=CERT_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='certItems/*',
            kinto_client=kinto_client,
            bucket=args.cert_bucket,
            collection=args.cert_collection,
            schema=schemas.get(args.cert_collection,
                               schemas.get(CERT_COLLECTION))),
        # GFX drivers
        'gfx': dict(
            fields=GFX_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='gfxItems/*',
            kinto_client=kinto_client,
            bucket=args.gfx_bucket,
            collection=args.gfx_collection,
            schema=schemas.get(args.gfx_collection,
                               schemas.get(GFX_COLLECTION))),
        # Addons
        'addons': dict(
            fields=ADDONS_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='emItems/*',
            kinto_client=kinto_client,
            bucket=args.addons_bucket,
            collection=args.addons_collection,
            schema=schemas.get(args.addons_collection,
                               schemas.get(ADDONS_COLLECTION)),
            with_scrapping=args.with_scrapping),
        # Plugins
        'plugins': dict(
            fields=PLUGINS_ITEMS_FIELDS,
            filename=args.xml_file,
            xpath='pluginItems/*',
            kinto_client=kinto_client,
            bucket=args.plugins_bucket,
            collection=args.plugins_collection,
            schema=schemas.get(args.plugins_collection,
                               schemas.get(PLUGINS_COLLECTION)),
            with_scrapping=args.with_scrapping)}

    for collection_type, collection in collections.items():
        if getattr(args, collection_type) or import_all:
            sync_records(**collection)
Exemplo n.º 20
0
 def test_add_parser_options_create_a_parser_if_needed(self):
     parser = cli_utils.add_parser_options()
     self.assert_option_strings(parser, *ALL_PARAMETERS)
     assert len(parser._actions) == 8
Exemplo n.º 21
0
 def test_add_parser_options_create_a_parser_if_needed(self):
     parser = cli_utils.add_parser_options()
     self.assert_option_strings(parser, *ALL_PARAMETERS)
     assert len(parser._actions) == 8