def handle(self, *args, **options):
if len(args) != 0: raise CommandError("This command doesn't expect arguments!")
flip_all = options['flip_all']
code_red = options['code_red']
es = get_es_new()
es_indices = list(get_all_expected_es_indices())
if code_red:
if raw_input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following indices will be affected:',
'\n'.join([unicode(index_info) for index_info in es_indices]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for index_info in es_indices:
es.indices.delete(index_info.index)
print 'deleted elastic index: {}'.format(index_info.index)
else:
print 'Safety first!'
return
if flip_all:
for index_info in es_indices:
assume_alias(es, index_info.index, index_info.alias)
print simplejson.dumps(es.indices.get_aliases(), indent=4)
def handle(self, index_name, **options):
noinput = options.pop('noinput')
es_indices = list(get_all_expected_es_indices())
indexes = [
index for index in es_indices
if index_name == index.alias or index_name == index.index
]
if not indexes:
raise CommandError(
"No matching index found: {}".format(index_name))
index_info = indexes[0]
es = get_es_new()
if (noinput or _confirm(
"Confirm that you want to update the mapping for '{}'".format(
index_info.index))):
mapping = copy(index_info.mapping)
mapping['_meta']['created'] = datetime.utcnow().isoformat()
mapping_res = es.indices.put_mapping(index_info.type,
{index_info.type: mapping},
index=index_info.index)
if mapping_res.get('acknowledged', False):
print("Index successfully updated")
else:
print(mapping_res)
def handle(self, *args, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
found_indices = set(es.indices.get_aliases().keys())
expected_indices = {info.index for info in get_all_expected_es_indices()}
if options['verbose']:
if expected_indices - found_indices:
print 'the following indices were not found:\n{}\n'.format(
'\n'.join(expected_indices - found_indices)
)
print 'expecting {} indices:\n{}\n'.format(len(expected_indices),
'\n'.join(sorted(expected_indices)))
to_delete = set([index for index in found_indices if index not in expected_indices])
if to_delete:
if options['noinput'] or raw_input(
'\n'.join([
'Really delete ALL the unrecognized elastic indices?',
'Here are the indices that will be deleted:',
'\n'.join(sorted(to_delete)),
'This operation is not reversible and all data will be lost.',
'Type "delete indices" to continue:\n',
])).lower() == 'delete indices':
for index in to_delete:
es.indices.delete(index)
else:
print 'aborted'
else:
print 'no indices need pruning'
def handle(self, *args, **options):
if len(args) != 0: raise CommandError("This command doesn't expect arguments!")
flip_all = options['flip_all']
code_red = options['code_red']
es = get_es_new()
es_indices = list(get_all_expected_es_indices())
if code_red:
if raw_input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following indices will be affected:',
'\n'.join([unicode(index_info) for index_info in es_indices]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for index_info in es_indices:
es.indices.delete(index_info.index)
print 'deleted elastic index: {}'.format(index_info.index)
else:
print 'Safety first!'
return
if flip_all:
for index_info in es_indices:
assume_alias(es, index_info.index, index_info.alias)
print simplejson.dumps(es.indices.get_aliases(), indent=4)
def handle(self, *args, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
# fixme: this can delete real indices if a reindex is in progress
found_indices = set(es.indices.get_aliases().keys())
pillow_indices = {info.index for info in get_all_expected_es_indices()}
ucr_indices = {index for index in get_ucr_es_indices()}
expected_indices = pillow_indices | ucr_indices
print expected_indices
if options['verbose']:
if expected_indices - found_indices:
print 'the following indices were not found:\n{}\n'.format(
'\n'.join(expected_indices - found_indices))
print 'expecting {} indices:\n{}\n'.format(
len(expected_indices), '\n'.join(sorted(expected_indices)))
unref_indices = set([
index for index in found_indices if index not in expected_indices
])
if unref_indices:
if options['delete']:
_delete_indices(es, unref_indices)
else:
_close_indices(es, unref_indices, options['noinput'])
else:
print 'no indices need pruning'
def handle(self, **options):
es_manager = ElasticManageAdapter()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
# fixme: this can delete real indices if a reindex is in progress
found_indices = set(es_manager.get_indices())
expected_indices = {
info.index
for info in get_all_expected_es_indices()
}
print(expected_indices)
if options['verbose']:
if expected_indices - found_indices:
print('the following indices were not found:\n{}\n'.format(
'\n'.join(expected_indices - found_indices)))
print('expecting {} indices:\n{}\n'.format(
len(expected_indices), '\n'.join(sorted(expected_indices))))
unref_indices = set([
index for index in found_indices if index not in expected_indices
])
if unref_indices:
if options['delete']:
_delete_indices(es_manager, unref_indices)
else:
_close_indices(es_manager, unref_indices, options['noinput'])
else:
print('no indices need pruning')
def handle(self, **options):
runs = []
all_es_indices = get_all_expected_es_indices()
es = get_es_new()
indices_needing_reindex = [
info for info in all_es_indices
if not es.indices.exists(info.index)
]
if not indices_needing_reindex:
print('Nothing needs to be reindexed')
return
print("Reindexing:\n\t", end=' ')
print('\n\t'.join(map(str, indices_needing_reindex)))
preindex_message = """
Heads up!
%s is going to start preindexing the following indices:\n
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, '\n\t'.join(
map(str, indices_needing_reindex)))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for index_info in indices_needing_reindex:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
reindex_command = get_reindex_commands(index_info.alias)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % index_info.alias)
for index_info in indices_needing_reindex:
print(index_info.alias)
g = gevent.spawn(do_reindex, index_info.alias, options['reset'])
runs.append(g)
if len(indices_needing_reindex) > 0:
gevent.joinall(runs)
try:
for job in runs:
job.get()
except Exception:
mail_admins("Pillow preindexing failed",
get_traceback_string())
raise
else:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" %
(', '.join(map(str, indices_needing_reindex)),
(datetime.utcnow() - start).seconds))
print("All pillowtop reindexing jobs completed")
def handle(self, **options):
flip_all = options['flip_all']
code_red = options['code_red']
es = get_es_new()
es_indices = list(get_all_expected_es_indices())
if code_red:
if input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following indices will be affected:',
'\n'.join([
six.text_type(index_info) for index_info in es_indices
]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for index_info in es_indices:
try:
es.indices.delete(index_info.index)
except NotFoundError:
print('elastic index not present: {}'.format(
index_info.index))
else:
print('deleted elastic index: {}'.format(
index_info.index))
else:
print('Safety first!')
return
if flip_all:
for index_info in es_indices:
assume_alias(es, index_info.index, index_info.alias)
print(simplejson.dumps(es.indices.get_aliases(), indent=4))
def handle(self, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
# fixme: this can delete real indices if a reindex is in progress
found_indices = set(es.indices.get_aliases().keys())
expected_indices = {info.index for info in get_all_expected_es_indices()}
print(expected_indices)
if options['verbose']:
if expected_indices - found_indices:
print('the following indices were not found:\n{}\n'.format(
'\n'.join(expected_indices - found_indices)
))
print('expecting {} indices:\n{}\n'.format(len(expected_indices),
'\n'.join(sorted(expected_indices))))
unref_indices = set([index for index in found_indices if index not in expected_indices])
if unref_indices:
if options['delete']:
_delete_indices(es, unref_indices)
else:
_close_indices(es, unref_indices, options['noinput'])
else:
print('no indices need pruning')
def handle(self, **options):
flip_all = options['flip_all']
code_red = options['code_red']
es = get_es_new()
es_indices = list(get_all_expected_es_indices())
if code_red:
if input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following indices will be affected:',
'\n'.join([six.text_type(index_info) for index_info in es_indices]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for index_info in es_indices:
try:
es.indices.delete(index_info.index)
except NotFoundError:
print('elastic index not present: {}'.format(index_info.index))
else:
print('deleted elastic index: {}'.format(index_info.index))
else:
print('Safety first!')
return
if flip_all:
for index_info in es_indices:
assume_alias(es, index_info.index, index_info.alias)
print(simplejson.dumps(es.indices.get_aliases(), indent=4))
def handle(self, *args, **options):
if len(args) != 1:
raise CommandError("Please specify the index name to update")
index_name = args[0]
es_indices = list(get_all_expected_es_indices())
indexes = [
index for index in es_indices
if index_name == index.alias or index_name == index.index
]
if not indexes:
raise CommandError(
"No matching index found: {}".format(index_name))
index_info = indexes[0]
es = get_es_new()
if _confirm(
"Confirm that you want to update the mapping for '{}'".format(
index_info.index)):
mapping = copy(index_info.mapping)
mapping['_meta']['created'] = datetime.utcnow().isoformat()
mapping_res = es.indices.put_mapping(index_info.type,
{index_info.type: mapping},
index=index_info.index)
if mapping_res.get('acknowledged', False):
print "Index successfully updated"
else:
print mapping_res
def test_prod_config(self):
found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
for info in found_prod_indices:
# for now don't test these two properties, just ensure they exist
self.assertTrue(info['meta'])
del info['meta']
self.assertTrue(info['mapping'])
del info['mapping']
found_prod_indices = sorted(found_prod_indices, key=lambda info: info['index'])
self.assertEqual(EXPECTED_PROD_INDICES, found_prod_indices)
def handle(self, index_name, **options):
noinput = options.pop('noinput')
es_indices = list(get_all_expected_es_indices())
indexes = [index for index in es_indices if index_name == index.alias or index_name == index.index]
if not indexes:
raise CommandError("No matching index found: {}".format(index_name))
index_info = indexes[0]
es = get_es_new()
if (noinput or _confirm("Confirm that you want to update the mapping for '{}'".format(index_info.index))):
mapping = copy(index_info.mapping)
mapping['_meta']['created'] = datetime.utcnow().isoformat()
mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index)
if mapping_res.get('acknowledged', False):
print("Index successfully updated")
else:
print(mapping_res)
def handle(self, *args, **options):
if len(args) != 1: raise CommandError("Please specify the index name to update")
index_name = args[0]
es_indices = list(get_all_expected_es_indices())
indexes = [index for index in es_indices if index_name == index.alias or index_name == index.index]
if not indexes:
raise CommandError("No matching index found: {}".format(index_name))
index_info = indexes[0]
es = get_es_new()
if _confirm("Confirm that you want to update the mapping for '{}'".format(index_info.index)):
mapping = copy(index_info.mapping)
mapping['_meta']['created'] = datetime.utcnow().isoformat()
mapping_res = es.indices.put_mapping(index_info.type, {index_info.type: mapping}, index=index_info.index)
if mapping_res.get('acknowledged', False):
print "Index successfully updated"
else:
print mapping_res
def test_prod_config(self):
found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
for info in found_prod_indices:
# for now don't test this property, just ensure it exist
self.assertTrue(info['mapping'])
del info['mapping']
def alias(info):
return info['alias']
found_prod_indices = sorted(found_prod_indices, key=alias)
expected_prod_indices = sorted(EXPECTED_PROD_INDICES, key=alias)
# compare aliases to make it easier to spot the difference
# when an index is added or removed
self.assertEqual(
[alias(info) for info in expected_prod_indices],
[alias(info) for info in found_prod_indices],
)
# do full comparison once we know the index aliases are the same
self.assertEqual(expected_prod_indices, found_prod_indices)
def test_prod_config(self):
found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
for info in found_prod_indices:
# for now don't test this property, just ensure it exist
self.assertTrue(info['mapping'])
del info['mapping']
def alias(info):
return info['alias']
found_prod_indices = sorted(found_prod_indices, key=alias)
expected_prod_indices = sorted(EXPECTED_PROD_INDICES, key=alias)
# compare aliases to make it easier to spot the difference
# when an index is added or removed
self.assertEqual(
[alias(info) for info in expected_prod_indices],
[alias(info) for info in found_prod_indices],
)
# do full comparison once we know the index aliases are the same
self.assertEqual(expected_prod_indices, found_prod_indices)
def test_prod_config(self):
# TODO: implement index verification in a way that is reindex-friendly
found_prod_indices = [info.to_json() for info in get_all_expected_es_indices()]
for info in found_prod_indices:
# for now don't test this property, just ensure it exist
self.assertTrue(info['mapping'])
del info['mapping']
# TODO: test mappings. Seems related, but different from
# `corehq/pillows/mappings/tests`. The tests here in this module
# should probably move over there some day.
def alias(info):
return info['alias']
found_prod_indices = sorted(found_prod_indices, key=alias)
expected_prod_indices = sorted(EXPECTED_PROD_INDICES, key=alias)
# compare aliases to make it easier to spot the difference
# when an index is added or removed
self.assertEqual(
[alias(info) for info in expected_prod_indices],
[alias(info) for info in found_prod_indices],
)
# do full comparison once we know the index aliases are the same
self.assertEqual(expected_prod_indices, found_prod_indices)
def handle(self, **options):
noinput = options.pop('noinput')
es_indices = list(get_all_expected_es_indices())
to_update = []
es = get_es_new()
for index_info in es_indices:
old_settings = es.indices.get_settings(index=index_info.index)
old_number_of_replicas = int(old_settings[
index_info.index]['settings']['index']['number_of_replicas'])
new_number_of_replicas = index_info.meta['settings'][
'number_of_replicas']
if old_number_of_replicas != new_number_of_replicas:
print(
"{} [{}]:\n Number of replicas changing from {!r} to {!r}"
.format(index_info.alias, index_info.index,
old_number_of_replicas, new_number_of_replicas))
to_update.append((index_info, {
'number_of_replicas': new_number_of_replicas,
}))
if not to_update:
print("There is nothing to update.")
return
if (noinput or _confirm(
"Confirm that you want to update all the settings above?")):
for index_info, settings in to_update:
mapping_res = es.indices.put_settings(index=index_info.index,
body=settings)
if mapping_res.get('acknowledged', False):
print("{} [{}]:\n Index settings successfully updated".
format(index_info.alias, index_info.index))
else:
print(mapping_res)
def handle(self, **options):
runs = []
all_es_indices = get_all_expected_es_indices()
es = get_es_new()
indices_needing_reindex = [info for info in all_es_indices if not es.indices.exists(info.index)]
if not indices_needing_reindex:
print('Nothing needs to be reindexed')
return
print("Reindexing:\n\t", end=' ')
print('\n\t'.join(map(six.text_type, indices_needing_reindex)))
preindex_message = """
Heads up!
%s is going to start preindexing the following indices:\n
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (
settings.EMAIL_SUBJECT_PREFIX,
'\n\t'.join(map(six.text_type, indices_needing_reindex))
)
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for index_info in indices_needing_reindex:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
reindex_command = get_reindex_commands(index_info.alias)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % index_info.alias
)
for index_info in indices_needing_reindex:
print(index_info.alias)
g = gevent.spawn(do_reindex, index_info.alias, options['reset'])
runs.append(g)
if len(indices_needing_reindex) > 0:
gevent.joinall(runs)
try:
for job in runs:
job.get()
except Exception:
mail_admins("Pillow preindexing failed", get_traceback_string())
raise
else:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" % (
', '.join(map(six.text_type, indices_needing_reindex)),
(datetime.utcnow() - start).seconds
)
)
print("All pillowtop reindexing jobs completed")
