def handle(self, *args, **options): if len(args) != 0: raise CommandError("This command doesn't expect arguments!") flip_all = options['flip_all'] code_red = options['code_red'] es = get_es_new() es_indices = list(get_all_expected_es_indices()) if code_red: if raw_input('\n'.join([ 'CODE RED!!!', 'Really delete ALL the elastic indices and pillow checkpoints?', 'The following indices will be affected:', '\n'.join([unicode(index_info) for index_info in es_indices]), 'This is a PERMANENT action. (Type "code red" to continue):', '', ])).lower() == 'code red': for index_info in es_indices: es.indices.delete(index_info.index) print 'deleted elastic index: {}'.format(index_info.index) else: print 'Safety first!' return if flip_all: for index_info in es_indices: assume_alias(es, index_info.index, index_info.alias) print simplejson.dumps(es.indices.get_aliases(), indent=4)
def handle(self, index_name, **options): noinput = options.pop('noinput') es_indices = list(get_all_expected_es_indices()) indexes = [ index for index in es_indices if index_name == index.alias or index_name == index.index ] if not indexes: raise CommandError( "No matching index found: {}".format(index_name)) index_info = indexes[0] es = get_es_new() if (noinput or _confirm( "Confirm that you want to update the mapping for '{}'".format( index_info.index))): mapping = copy(index_info.mapping) mapping['_meta']['created'] = datetime.utcnow().isoformat() mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index) if mapping_res.get('acknowledged', False): print("Index successfully updated") else: print(mapping_res)
def handle(self, *args, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist found_indices = set(es.indices.get_aliases().keys()) expected_indices = {info.index for info in get_all_expected_es_indices()} if options['verbose']: if expected_indices - found_indices: print 'the following indices were not found:\n{}\n'.format( '\n'.join(expected_indices - found_indices) ) print 'expecting {} indices:\n{}\n'.format(len(expected_indices), '\n'.join(sorted(expected_indices))) to_delete = set([index for index in found_indices if index not in expected_indices]) if to_delete: if options['noinput'] or raw_input( '\n'.join([ 'Really delete ALL the unrecognized elastic indices?', 'Here are the indices that will be deleted:', '\n'.join(sorted(to_delete)), 'This operation is not reversible and all data will be lost.', 'Type "delete indices" to continue:\n', ])).lower() == 'delete indices': for index in to_delete: es.indices.delete(index) else: print 'aborted' else: print 'no indices need pruning'
def handle(self, *args, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist # fixme: this can delete real indices if a reindex is in progress found_indices = set(es.indices.get_aliases().keys()) pillow_indices = {info.index for info in get_all_expected_es_indices()} ucr_indices = {index for index in get_ucr_es_indices()} expected_indices = pillow_indices | ucr_indices print expected_indices if options['verbose']: if expected_indices - found_indices: print 'the following indices were not found:\n{}\n'.format( '\n'.join(expected_indices - found_indices)) print 'expecting {} indices:\n{}\n'.format( len(expected_indices), '\n'.join(sorted(expected_indices))) unref_indices = set([ index for index in found_indices if index not in expected_indices ]) if unref_indices: if options['delete']: _delete_indices(es, unref_indices) else: _close_indices(es, unref_indices, options['noinput']) else: print 'no indices need pruning'
def handle(self, **options): es_manager = ElasticManageAdapter() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist # fixme: this can delete real indices if a reindex is in progress found_indices = set(es_manager.get_indices()) expected_indices = { info.index for info in get_all_expected_es_indices() } print(expected_indices) if options['verbose']: if expected_indices - found_indices: print('the following indices were not found:\n{}\n'.format( '\n'.join(expected_indices - found_indices))) print('expecting {} indices:\n{}\n'.format( len(expected_indices), '\n'.join(sorted(expected_indices)))) unref_indices = set([ index for index in found_indices if index not in expected_indices ]) if unref_indices: if options['delete']: _delete_indices(es_manager, unref_indices) else: _close_indices(es_manager, unref_indices, options['noinput']) else: print('no indices need pruning')
def handle(self, **options): runs = [] all_es_indices = get_all_expected_es_indices() es = get_es_new() indices_needing_reindex = [ info for info in all_es_indices if not es.indices.exists(info.index) ] if not indices_needing_reindex: print('Nothing needs to be reindexed') return print("Reindexing:\n\t", end=' ') print('\n\t'.join(map(str, indices_needing_reindex))) preindex_message = """ Heads up! %s is going to start preindexing the following indices:\n %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, '\n\t'.join( map(str, indices_needing_reindex))) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for index_info in indices_needing_reindex: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows reindex_command = get_reindex_commands(index_info.alias) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % index_info.alias) for index_info in indices_needing_reindex: print(index_info.alias) g = gevent.spawn(do_reindex, index_info.alias, options['reset']) runs.append(g) if len(indices_needing_reindex) > 0: gevent.joinall(runs) try: for job in runs: job.get() except Exception: mail_admins("Pillow preindexing failed", get_traceback_string()) raise else: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % (', '.join(map(str, indices_needing_reindex)), (datetime.utcnow() - start).seconds)) print("All pillowtop reindexing jobs completed")
def handle(self, **options): flip_all = options['flip_all'] code_red = options['code_red'] es = get_es_new() es_indices = list(get_all_expected_es_indices()) if code_red: if input('\n'.join([ 'CODE RED!!!', 'Really delete ALL the elastic indices and pillow checkpoints?', 'The following indices will be affected:', '\n'.join([ six.text_type(index_info) for index_info in es_indices ]), 'This is a PERMANENT action. (Type "code red" to continue):', '', ])).lower() == 'code red': for index_info in es_indices: try: es.indices.delete(index_info.index) except NotFoundError: print('elastic index not present: {}'.format( index_info.index)) else: print('deleted elastic index: {}'.format( index_info.index)) else: print('Safety first!') return if flip_all: for index_info in es_indices: assume_alias(es, index_info.index, index_info.alias) print(simplejson.dumps(es.indices.get_aliases(), indent=4))
def handle(self, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist # fixme: this can delete real indices if a reindex is in progress found_indices = set(es.indices.get_aliases().keys()) expected_indices = {info.index for info in get_all_expected_es_indices()} print(expected_indices) if options['verbose']: if expected_indices - found_indices: print('the following indices were not found:\n{}\n'.format( '\n'.join(expected_indices - found_indices) )) print('expecting {} indices:\n{}\n'.format(len(expected_indices), '\n'.join(sorted(expected_indices)))) unref_indices = set([index for index in found_indices if index not in expected_indices]) if unref_indices: if options['delete']: _delete_indices(es, unref_indices) else: _close_indices(es, unref_indices, options['noinput']) else: print('no indices need pruning')
def handle(self, **options): flip_all = options['flip_all'] code_red = options['code_red'] es = get_es_new() es_indices = list(get_all_expected_es_indices()) if code_red: if input('\n'.join([ 'CODE RED!!!', 'Really delete ALL the elastic indices and pillow checkpoints?', 'The following indices will be affected:', '\n'.join([six.text_type(index_info) for index_info in es_indices]), 'This is a PERMANENT action. (Type "code red" to continue):', '', ])).lower() == 'code red': for index_info in es_indices: try: es.indices.delete(index_info.index) except NotFoundError: print('elastic index not present: {}'.format(index_info.index)) else: print('deleted elastic index: {}'.format(index_info.index)) else: print('Safety first!') return if flip_all: for index_info in es_indices: assume_alias(es, index_info.index, index_info.alias) print(simplejson.dumps(es.indices.get_aliases(), indent=4))
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Please specify the index name to update") index_name = args[0] es_indices = list(get_all_expected_es_indices()) indexes = [ index for index in es_indices if index_name == index.alias or index_name == index.index ] if not indexes: raise CommandError( "No matching index found: {}".format(index_name)) index_info = indexes[0] es = get_es_new() if _confirm( "Confirm that you want to update the mapping for '{}'".format( index_info.index)): mapping = copy(index_info.mapping) mapping['_meta']['created'] = datetime.utcnow().isoformat() mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index) if mapping_res.get('acknowledged', False): print "Index successfully updated" else: print mapping_res
def test_prod_config(self): found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()] for info in found_prod_indices: # for now don't test these two properties, just ensure they exist self.assertTrue(info['meta']) del info['meta'] self.assertTrue(info['mapping']) del info['mapping'] found_prod_indices = sorted(found_prod_indices, key=lambda info: info['index']) self.assertEqual(EXPECTED_PROD_INDICES, found_prod_indices)
def handle(self, index_name, **options): noinput = options.pop('noinput') es_indices = list(get_all_expected_es_indices()) indexes = [index for index in es_indices if index_name == index.alias or index_name == index.index] if not indexes: raise CommandError("No matching index found: {}".format(index_name)) index_info = indexes[0] es = get_es_new() if (noinput or _confirm("Confirm that you want to update the mapping for '{}'".format(index_info.index))): mapping = copy(index_info.mapping) mapping['_meta']['created'] = datetime.utcnow().isoformat() mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index) if mapping_res.get('acknowledged', False): print("Index successfully updated") else: print(mapping_res)
def handle(self, *args, **options): if len(args) != 1: raise CommandError("Please specify the index name to update") index_name = args[0] es_indices = list(get_all_expected_es_indices()) indexes = [index for index in es_indices if index_name == index.alias or index_name == index.index] if not indexes: raise CommandError("No matching index found: {}".format(index_name)) index_info = indexes[0] es = get_es_new() if _confirm("Confirm that you want to update the mapping for '{}'".format(index_info.index)): mapping = copy(index_info.mapping) mapping['_meta']['created'] = datetime.utcnow().isoformat() mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index) if mapping_res.get('acknowledged', False): print "Index successfully updated" else: print mapping_res
def test_prod_config(self): found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()] for info in found_prod_indices: # for now don't test this property, just ensure it exist self.assertTrue(info['mapping']) del info['mapping'] def alias(info): return info['alias'] found_prod_indices = sorted(found_prod_indices, key=alias) expected_prod_indices = sorted(EXPECTED_PROD_INDICES, key=alias) # compare aliases to make it easier to spot the difference # when an index is added or removed self.assertEqual( [alias(info) for info in expected_prod_indices], [alias(info) for info in found_prod_indices], ) # do full comparison once we know the index aliases are the same self.assertEqual(expected_prod_indices, found_prod_indices)
def test_prod_config(self): # TODO: implement index verification in a way that is reindex-friendly found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()] for info in found_prod_indices: # for now don't test this property, just ensure it exist self.assertTrue(info['mapping']) del info['mapping'] # TODO: test mappings. Seems related, but different from # `corehq/pillows/mappings/tests`. The tests here in this module # should probably move over there some day. def alias(info): return info['alias'] found_prod_indices = sorted(found_prod_indices, key=alias) expected_prod_indices = sorted(EXPECTED_PROD_INDICES, key=alias) # compare aliases to make it easier to spot the difference # when an index is added or removed self.assertEqual( [alias(info) for info in expected_prod_indices], [alias(info) for info in found_prod_indices], ) # do full comparison once we know the index aliases are the same self.assertEqual(expected_prod_indices, found_prod_indices)
def handle(self, **options): noinput = options.pop('noinput') es_indices = list(get_all_expected_es_indices()) to_update = [] es = get_es_new() for index_info in es_indices: old_settings = es.indices.get_settings(index=index_info.index) old_number_of_replicas = int(old_settings[ index_info.index]['settings']['index']['number_of_replicas']) new_number_of_replicas = index_info.meta['settings'][ 'number_of_replicas'] if old_number_of_replicas != new_number_of_replicas: print( "{} [{}]:\n Number of replicas changing from {!r} to {!r}" .format(index_info.alias, index_info.index, old_number_of_replicas, new_number_of_replicas)) to_update.append((index_info, { 'number_of_replicas': new_number_of_replicas, })) if not to_update: print("There is nothing to update.") return if (noinput or _confirm( "Confirm that you want to update all the settings above?")): for index_info, settings in to_update: mapping_res = es.indices.put_settings(index=index_info.index, body=settings) if mapping_res.get('acknowledged', False): print("{} [{}]:\n Index settings successfully updated". format(index_info.alias, index_info.index)) else: print(mapping_res)
def handle(self, **options): runs = [] all_es_indices = get_all_expected_es_indices() es = get_es_new() indices_needing_reindex = [info for info in all_es_indices if not es.indices.exists(info.index)] if not indices_needing_reindex: print('Nothing needs to be reindexed') return print("Reindexing:\n\t", end=' ') print('\n\t'.join(map(six.text_type, indices_needing_reindex))) preindex_message = """ Heads up! %s is going to start preindexing the following indices:\n %s This may take a while, so don't deploy until all these have reported finishing. """ % ( settings.EMAIL_SUBJECT_PREFIX, '\n\t'.join(map(six.text_type, indices_needing_reindex)) ) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for index_info in indices_needing_reindex: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows reindex_command = get_reindex_commands(index_info.alias) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % index_info.alias ) for index_info in indices_needing_reindex: print(index_info.alias) g = gevent.spawn(do_reindex, index_info.alias, options['reset']) runs.append(g) if len(indices_needing_reindex) > 0: gevent.joinall(runs) try: for job in runs: job.get() except Exception: mail_admins("Pillow preindexing failed", get_traceback_string()) raise else: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join(map(six.text_type, indices_needing_reindex)), (datetime.utcnow() - start).seconds ) ) print("All pillowtop reindexing jobs completed")