def test_index_operations(self):
pillow = TestElasticPillow()
self.assertTrue(self.es.indices.exists(self.index))
self.assertTrue(pillow_index_exists(pillow))
# delete and check
pillow.get_es_new().indices.delete(self.index)
self.assertFalse(self.es.indices.exists(self.index))
self.assertFalse(pillow_index_exists(pillow))
# create and check
create_index_for_pillow(pillow)
self.assertTrue(self.es.indices.exists(self.index))
self.assertTrue(pillow_index_exists(pillow))
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillow_classes()
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow),
pillow_classes)
aliasable_pillows = [p(online=False) for p in aliased_classes]
reindex_all = options['replace']
pillow_state_results = get_pillow_states(aliasable_pillows)
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print ("Reindexing master pillows that do not exist yet "
"(ones with aliases skipped)")
preindexable_pillows = [pillow for pillow in aliasable_pillows
if not pillow_index_exists(pillow)]
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
preindexable_pillows)
print "Reindexing:\n\t",
print '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (
settings.EMAIL_SUBJECT_PREFIX,
', '.join([x.__class__.__name__ for x in reindex_pillows])
)
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_commands(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % pillow_class_name
)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
if len(reindex_pillows) > 0:
gevent.joinall(runs)
try:
for job in runs:
job.get()
except Exception:
f = StringIO()
traceback.print_exc(file=f)
mail_admins("Pillow preindexing failed", f.getvalue())
raise
else:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
)
)
print "All pillowtop reindexing jobs completed"
