Ejemplo n.º 1
0
    def handle(self, *args, **options):
        url = options['url']
        if url:
            es = get_es(urls=[url])
        else:
            es = get_es()
        # We define some custom analyzers that our mappings can use.
        index_settings = {'mappings': {}, 'settings': get_analyzers()}

        # Retrieve the mappings for the index-enabled models.
        for mappingClass in self.MAPPINGS:
            model_name = mappingClass.get_mapping_type_name()
            index_settings['mappings'].update({model_name: mappingClass.get_mapping()})

        # Create a new index.
        new_index = 'index_%s' % (int(time.time()))
        es.indices.create(new_index, body=index_settings)
        self.index(new_index)

        # The default index name, (we will use as an alias).
        index_name = settings.ES_INDEXES['default']

        # Check if we have a current index.
        old_index = None
        aliases = es.indices.get_aliases(name=index_name)
        for key, value in aliases.iteritems():
            if value['aliases']:
                old_index = key

        # Change the alias to point to our new index, and remove the old index.

        self.stdout.write('Changing alias "%s" from old index "%s" to new index "%s"' %
                          (index_name, old_index, new_index))
        if old_index:
            es.indices.update_aliases({'actions':
                                       [{'remove': {'index': old_index, 'alias': index_name}},
                                        {'add': {'index': new_index, 'alias': index_name}}]})
            es.indices.delete(old_index)
        else:
            if es.indices.exists(index_name):
                # This is a corner case. There was no alias named index_name, but
                # an index index_name nevertheless exists, this only happens when the index
                # was already created (because of ES auto creation features).
                es.indices.delete(index_name)
            es.indices.update_aliases({'actions':
                                       [{'add': {'index': new_index, 'alias': index_name}}]})

        # Finally re-index one more time, to pick up updates that were written during our command.
        # Note that this models that do not use the DeletedMixin will not work this way.
        self.index(index_name)
        self.unindex(index_name)
Ejemplo n.º 2
0
    def index(self):
        """
        Do the actual indexing for all specified targets.
        """
        for mapping in self.target_list:
            model_name = mapping.get_mapping_type_name()
            main_index_base = settings.ES_INDEXES['default']
            main_index = get_index_name(main_index_base, mapping)

            self.stdout.write('==> %s' % model_name)

            # Check if we currently have an index for this mapping.
            old_index = None
            aliases = self.es.indices.get_aliases(name=main_index)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key
                    self.stdout.write('Current index "%s"' % key)

            # Check any indices with no alias (leftovers from failed indexing).
            # Or it could be that it is still in progress,
            aliases = self.es.indices.get_aliases()
            for key, value in aliases.iteritems():
                if not key.endswith(model_name):
                    # Not the model we are looking after.
                    continue
                if key == main_index:
                    # This is an auto created index. Will be removed at end of command.
                    continue
                if not value['aliases']:
                    if self.force:
                        self.stdout.write('Removing leftover "%s"' % key)
                        self.es.indices.delete(key)
                    else:
                        raise Exception('Found leftover %s, proceed with -f to remove.'
                                        ' Make sure indexing this model is not already running!' % key)

            # Create new index.
            index_settings = {
                'mappings': {
                    model_name: mapping.get_mapping()
                },
                'settings': {
                    'analysis': get_analyzers()['analysis'],
                    'number_of_shards': 1,
                }
            }
            temp_index_base = 'index_%s' % (int(time.time()))
            temp_index = get_index_name(temp_index_base, mapping)

            self.stdout.write('Creating new index "%s"' % temp_index)
            self.es.indices.create(temp_index, body=index_settings)

            # Index documents.
            self.index_documents(mapping, temp_index_base)

            # Switch aliases.
            if old_index:
                self.es.indices.update_aliases({
                    'actions': [
                        {'remove': {'index': old_index, 'alias': main_index}},
                        {'remove': {'index': old_index, 'alias': main_index_base}},
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
                self.stdout.write('Removing previous index "%s"' % old_index)
                self.es.indices.delete(old_index)
            else:
                if self.es.indices.exists(main_index):
                    # This is a corner case. There was no alias named index_name, but
                    # an index index_name nevertheless exists, this only happens when the index
                    # was already created (because of ES auto creation features).
                    self.stdout.write('Removing previous (presumably auto created) index "%s"' % main_index)
                    self.es.indices.delete(main_index)
                self.es.indices.update_aliases({
                    'actions': [
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
            self.stdout.write('')

        self.stdout.write('Indexing finished.')
Ejemplo n.º 3
0
    def handle(self, *args, **options):
        es = get_es_client()

        if args:
            self.stdout.write('Aborting, unexpected arguments %s' % list(args))
            return

        if options['list']:
            self.stdout.write('Possible models to index:\n')
            for mapping in ModelMappings.get_model_mappings().values():
                self.stdout.write(mapping.get_mapping_type_name())
            return

        target = options['target']
        if target:
            targets = target.split(',')
        else:
            targets = []  # (meaning all)
        has_targets = targets != []

        self.stdout.write('Please remember that HelloLily needs to be in maintenance mode. \n\n')

        if has_targets:
            # Do a quick run to check if all targets are valid models.
            check_targets = list(targets)  # make a copy
            for target in check_targets:
                for mapping in ModelMappings.get_model_mappings().values():
                    if self.model_targetted(mapping, [target]):
                        check_targets.remove(target)
                        break
            if check_targets:
                self.stdout.write('Aborting, following targets not recognized: %s' % check_targets)
                return

        for mapping in ModelMappings.get_model_mappings().values():
            model_name = mapping.get_mapping_type_name()
            main_index_base = settings.ES_INDEXES['default']
            main_index = get_index_name(main_index_base, mapping)

            # Skip this model if there are specific targets and not specified.
            if has_targets and not self.model_targetted(mapping, targets):
                continue

            self.stdout.write('==> %s' % model_name)

            # Check if we currently have an index for this mapping.
            old_index = None
            aliases = es.indices.get_aliases(name=main_index)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key
                    self.stdout.write('Current index "%s"' % key)

            # Check any indices with no alias (leftovers from failed indexing).
            # Or it could be that it is still in progress,
            aliases = es.indices.get_aliases()
            for key, value in aliases.iteritems():
                if not key.endswith(model_name):
                    # Not the model we are looking after.
                    continue
                if key == main_index:
                    # This is an auto created index. Will be removed at end of command.
                    continue
                if not value['aliases']:
                    if options['force']:
                        self.stdout.write('Removing leftover "%s"' % key)
                        es.indices.delete(key)
                    else:
                        raise Exception('Found leftover %s, proceed with -f to remove.'
                                        ' Make sure indexing this model is not already running!' % key)

            # Create new index.
            index_settings = {
                'mappings': {
                    model_name: mapping.get_mapping()
                },
                'settings': {
                    'analysis': get_analyzers()['analysis'],
                    'number_of_shards': 1,
                }
            }
            temp_index_base = 'index_%s' % (int(time.time()))
            temp_index = get_index_name(temp_index_base, mapping)

            self.stdout.write('Creating new index "%s"' % temp_index)
            es.indices.create(temp_index, body=index_settings)

            # Index documents.
            self.index_documents(mapping, temp_index_base)

            # Switch aliases.
            if old_index:
                es.indices.update_aliases({
                    'actions': [
                        {'remove': {'index': old_index, 'alias': main_index}},
                        {'remove': {'index': old_index, 'alias': main_index_base}},
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
                self.stdout.write('Removing previous index "%s"' % old_index)
                es.indices.delete(old_index)
            else:
                if es.indices.exists(main_index):
                    # This is a corner case. There was no alias named index_name, but
                    # an index index_name nevertheless exists, this only happens when the index
                    # was already created (because of ES auto creation features).
                    self.stdout.write('Removing previous (presumably auto created) index "%s"' % main_index)
                    es.indices.delete(main_index)
                es.indices.update_aliases({
                    'actions': [
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
            self.stdout.write('')

        self.stdout.write('Indexing finished.')

        if options['queries']:
            from django.db import connection
            for query in connection.queries:
                print query
Ejemplo n.º 4
0
    def index(self):
        """
        Do the actual indexing for all specified targets.
        """
        for mapping in self.target_list:
            model_name = mapping.get_mapping_type_name()
            main_index_base = settings.ES_INDEXES['default']
            main_index = get_index_name(main_index_base, mapping)

            self.stdout.write('==> %s' % model_name)

            # Check if we currently have an index for this mapping.
            old_index = None
            aliases = self.es.indices.get_aliases(name=main_index)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key
                    self.stdout.write('Current index "%s"' % key)

            # Check any indices with no alias (leftovers from failed indexing).
            # Or it could be that it is still in progress,
            aliases = self.es.indices.get_aliases()
            for key, value in aliases.iteritems():
                if not key.endswith(model_name):
                    # Not the model we are looking after.
                    continue
                if key == main_index:
                    # This is an auto created index. Will be removed at end of command.
                    continue
                if not value['aliases']:
                    if self.force:
                        self.stdout.write('Removing leftover "%s"' % key)
                        self.es.indices.delete(key)
                    else:
                        raise Exception('Found leftover %s, proceed with -f to remove.'
                                        ' Make sure indexing this model is not already running!' % key)

            # Create new index.
            index_settings = {
                'mappings': {
                    model_name: mapping.get_mapping()
                },
                'settings': {
                    'analysis': get_analyzers()['analysis'],
                    'number_of_shards': 1,
                }
            }
            temp_index_base = 'index_%s' % (int(time.time()))
            temp_index = get_index_name(temp_index_base, mapping)

            self.stdout.write('Creating new index "%s"' % temp_index)
            self.es.indices.create(temp_index, body=index_settings)

            # Index documents.
            self.index_documents(mapping, temp_index_base)

            # Switch aliases.
            if old_index:
                self.es.indices.update_aliases({
                    'actions': [
                        {'remove': {'index': old_index, 'alias': main_index}},
                        {'remove': {'index': old_index, 'alias': main_index_base}},
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
                self.stdout.write('Removing previous index "%s"' % old_index)
                self.es.indices.delete(old_index)
            else:
                if self.es.indices.exists(main_index):
                    # This is a corner case. There was no alias named index_name, but
                    # an index index_name nevertheless exists, this only happens when the index
                    # was already created (because of ES auto creation features).
                    self.stdout.write('Removing previous (presumably auto created) index "%s"' % main_index)
                    self.es.indices.delete(main_index)
                self.es.indices.update_aliases({
                    'actions': [
                        {'add': {'index': temp_index, 'alias': main_index}},
                        {'add': {'index': temp_index, 'alias': main_index_base}},
                    ]
                })
            self.stdout.write('')

        self.stdout.write('Indexing finished.')
Ejemplo n.º 5
0
    def handle(self, *args, **options):
        es = get_es_client()

        # Check if specific targets specified to run, or otherwise run all.
        target = options['target']
        if target:
            targets = target.split(',')
            # Check specific IDs.
            if options['id']:
                ids = options['id'].split(',')
                self.index(settings.ES_INDEXES['default'], specific_targets=targets, ids=ids)
            else:
                self.index(settings.ES_INDEXES['default'], specific_targets=targets)

        else:
            if es.indices.exists(settings.ES_INDEXES['new_index']):
                if options['force']:
                    es.indices.delete(settings.ES_INDEXES['new_index'])
                else:
                    raise Exception('%s already exists, run with -f to remove old' % settings.ES_INDEXES['new_index'])

            # Define analyzers and set alias for new index.
            index_settings = {
                'mappings': {},
                'settings': get_analyzers(),
                'aliases': {
                    settings.ES_INDEXES['new_index']: {},
                },
            }

            # Retrieve the mappings for the index-enabled models.
            for mapping_class in ModelMappings.get_model_mappings().values():
                model_name = mapping_class.get_mapping_type_name()
                index_settings['mappings'].update({model_name: mapping_class.get_mapping()})

            new_index = 'index_%s' % (int(time.time()))
            self.stdout.write('Creating new index "%s"' % new_index)
            es.indices.create(new_index, body=index_settings)
            self.index(new_index)

            # The default index name, (we will use as an alias).
            index_name = settings.ES_INDEXES['default']

            # Check if we have a current index.
            old_index = None
            aliases = es.indices.get_aliases(name=index_name)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key

            # Change the alias to point to our new index, and remove the old index.

            self.stdout.write('Changing alias "%s" from old index "%s" to new index "%s"' %
                              (index_name, old_index, new_index))
            if old_index:
                es.indices.update_aliases({
                    'actions': [
                        {'remove': {'index': old_index, 'alias': index_name}},
                        {'add': {'index': new_index, 'alias': index_name}},
                    ]
                })
                es.indices.delete(old_index)
            else:
                if es.indices.exists(index_name):
                    # This is a corner case. There was no alias named index_name, but
                    # an index index_name nevertheless exists, this only happens when the index
                    # was already created (because of ES auto creation features).
                    es.indices.delete(index_name)
                es.indices.update_aliases({
                    'actions': [
                        {'add': {'index': new_index, 'alias': index_name}}
                    ]
                })
            es.indices.update_aliases({
                'actions': [
                    {'remove': {'index': new_index, 'alias': settings.ES_INDEXES['new_index']}},
                ]
            })
            # Compensate for the small race condition in the signal handlers:
            # They check if the index exists then updates documents, however when we
            # delete the alias in between these two commands, we can end up with
            # an auto created index named new_index, so we simply delete it again.
            time.sleep(5)
            try:
                es.indices.delete(settings.ES_INDEXES['new_index'])
            except NotFoundError:
                pass

        if options['queries']:
            from django.db import connection
            for query in connection.queries:
                print query
Ejemplo n.º 6
0
    def handle(self, *args, **options):
        es = get_es_client()

        # Check if specific targets specified to run, or otherwise run all.
        target = options['target']
        if target:
            targets = target.split(',')
            # Check specific IDs.
            if options['id']:
                ids = options['id'].split(',')
                self.index(settings.ES_INDEXES['default'],
                           specific_targets=targets,
                           ids=ids)
            else:
                self.index(settings.ES_INDEXES['default'],
                           specific_targets=targets)

        else:
            if es.indices.exists(settings.ES_INDEXES['new_index']):
                if options['force']:
                    es.indices.delete(settings.ES_INDEXES['new_index'])
                else:
                    raise Exception(
                        '%s already exists, run with -f to remove old' %
                        settings.ES_INDEXES['new_index'])

            # Define analyzers and set alias for new index.
            index_settings = {
                'mappings': {},
                'settings': get_analyzers(),
                'aliases': {
                    settings.ES_INDEXES['new_index']: {},
                },
            }

            # Retrieve the mappings for the index-enabled models.
            for mapping_class in ModelMappings.get_model_mappings().values():
                model_name = mapping_class.get_mapping_type_name()
                index_settings['mappings'].update(
                    {model_name: mapping_class.get_mapping()})

            new_index = 'index_%s' % (int(time.time()))
            self.stdout.write('Creating new index "%s"' % new_index)
            es.indices.create(new_index, body=index_settings)
            self.index(new_index)

            # The default index name, (we will use as an alias).
            index_name = settings.ES_INDEXES['default']

            # Check if we have a current index.
            old_index = None
            aliases = es.indices.get_aliases(name=index_name)
            for key, value in aliases.iteritems():
                if value['aliases']:
                    old_index = key

            # Change the alias to point to our new index, and remove the old index.

            self.stdout.write(
                'Changing alias "%s" from old index "%s" to new index "%s"' %
                (index_name, old_index, new_index))
            if old_index:
                es.indices.update_aliases({
                    'actions': [
                        {
                            'remove': {
                                'index': old_index,
                                'alias': index_name
                            }
                        },
                        {
                            'add': {
                                'index': new_index,
                                'alias': index_name
                            }
                        },
                    ]
                })
                es.indices.delete(old_index)
            else:
                if es.indices.exists(index_name):
                    # This is a corner case. There was no alias named index_name, but
                    # an index index_name nevertheless exists, this only happens when the index
                    # was already created (because of ES auto creation features).
                    es.indices.delete(index_name)
                es.indices.update_aliases({
                    'actions': [{
                        'add': {
                            'index': new_index,
                            'alias': index_name
                        }
                    }]
                })
            es.indices.update_aliases({
                'actions': [
                    {
                        'remove': {
                            'index': new_index,
                            'alias': settings.ES_INDEXES['new_index']
                        }
                    },
                ]
            })
            # Compensate for the small race condition in the signal handlers:
            # They check if the index exists then updates documents, however when we
            # delete the alias in between these two commands, we can end up with
            # an auto created index named new_index, so we simply delete it again.
            time.sleep(5)
            try:
                es.indices.delete(settings.ES_INDEXES['new_index'])
            except NotFoundError:
                pass

        if options['queries']:
            from django.db import connection
            for query in connection.queries:
                print query