def handle(self, *args, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
found_indices = set(es.indices.get_aliases().keys())
existing_indices = set(pillow.es_index for pillow in filter(
lambda x: isinstance(x, AliasedElasticPillow), get_all_pillows()))
if options['verbose']:
if existing_indices - found_indices:
print 'the following indices were not found:\n{}\n'.format(
'\n'.join(existing_indices - found_indices))
print 'expecting {} indices:\n{}\n'.format(
len(existing_indices), '\n'.join(sorted(existing_indices)))
to_delete = set([
index for index in found_indices if index not in existing_indices
])
if to_delete:
if options['noinput'] or raw_input('\n'.join([
'Really delete ALL the unrecognized elastic indices?',
'Here are the indices that will be deleted:',
'\n'.join(sorted(to_delete)),
'This operation is not reversible and all data will be lost.',
'Type "delete indices" to continue:\n',
])).lower() == 'delete indices':
for index in to_delete:
es.indices.delete(index)
else:
print 'aborted'
else:
print 'no indices need pruning'
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillows(instantiate=False)
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows)
# mapped masters: ES indices as known in the current running code state that
# correctly have the alias applied to them
# unmapped masters: ES indices as known in the current running code state that
# do not have the alias applied to them
# stale indices: ES indices running on ES that are not part of the current source control.
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print "Reindexing master pillows that do not exist yet (ones with aliases skipped)"
preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows)
print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
#loop through pillows once before running greenlets to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_command(pillow_class_name)
if not reindex_command:
raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillows(instantiate=False)
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow),
pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
pillow_states = get_pillow_states(aliasable_pillows)
mapped_masters, unmapped_masters, stale_indices = pillow_states
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print(
"Reindexing master pillows that do not exist yet "
"(ones with aliases skipped)")
preindexable_pillows = filter(lambda x: not x.index_exists(),
aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
preindexable_pillows)
print "Reindexing:\n\t",
print '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join(
[x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_commands(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" %
(', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
found_indices = set(es.indices.get_aliases().keys())
existing_indices = set(
pillow.es_index for pillow in filter(lambda x: isinstance(x, AliasedElasticPillow), get_all_pillows())
)
if options['verbose']:
if existing_indices - found_indices:
print 'the following indices were not found:\n{}\n'.format(
'\n'.join(existing_indices - found_indices)
)
print 'expecting {} indices:\n{}\n'.format(len(existing_indices),
'\n'.join(sorted(existing_indices)))
to_delete = set([index for index in found_indices if index not in existing_indices])
if to_delete:
if options['noinput'] or raw_input(
'\n'.join([
'Really delete ALL the unrecognized elastic indices?',
'Here are the indices that will be deleted:',
'\n'.join(sorted(to_delete)),
'This operation is not reversible and all data will be lost.',
'Type "delete indices" to continue:\n',
])).lower() == 'delete indices':
for index in to_delete:
es.indices.delete(index)
else:
print 'aborted'
else:
print 'no indices need pruning'
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillows(instantiate=False)
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow),
pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
pillow_state_results = get_pillow_states(aliasable_pillows)
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print ("Reindexing master pillows that do not exist yet "
"(ones with aliases skipped)")
preindexable_pillows = filter(lambda x: not x.index_exists(),
aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
preindexable_pillows)
print "Reindexing:\n\t",
print '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (
settings.EMAIL_SUBJECT_PREFIX,
', '.join([x.__class__.__name__ for x in reindex_pillows])
)
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_commands(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % pillow_class_name
)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
if len(reindex_pillows) > 0:
gevent.joinall(runs)
try:
for job in runs:
job.get()
except Exception:
f = StringIO()
traceback.print_exc(file=f)
mail_admins("Pillow preindexing failed", f.getvalue())
raise
else:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
)
)
print "All pillowtop reindexing jobs completed"
