def test_index_operations(self): pillow = TestElasticPillow() self.assertTrue(self.es.indices.exists(self.index)) self.assertTrue(pillow_index_exists(pillow)) # delete and check pillow.get_es_new().indices.delete(self.index) self.assertFalse(self.es.indices.exists(self.index)) self.assertFalse(pillow_index_exists(pillow)) # create and check create_index_for_pillow(pillow) self.assertTrue(self.es.indices.exists(self.index)) self.assertTrue(pillow_index_exists(pillow))
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillow_classes() aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(online=False) for p in aliased_classes] reindex_all = options['replace'] pillow_state_results = get_pillow_states(aliasable_pillows) print "Master indices missing aliases:" unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print ("Reindexing master pillows that do not exist yet " "(ones with aliases skipped)") preindexable_pillows = [pillow for pillow in aliasable_pillows if not pillow_index_exists(pillow)] reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t", print '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % ( settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]) ) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_commands(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % pillow_class_name ) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) if len(reindex_pillows) > 0: gevent.joinall(runs) try: for job in runs: job.get() except Exception: f = StringIO() traceback.print_exc(file=f) mail_admins("Pillow preindexing failed", f.getvalue()) raise else: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds ) ) print "All pillowtop reindexing jobs completed"