def handle(self, *args, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist found_indices = set(es.indices.get_aliases().keys()) existing_indices = set(pillow.es_index for pillow in filter( lambda x: isinstance(x, AliasedElasticPillow), get_all_pillows())) if options['verbose']: if existing_indices - found_indices: print 'the following indices were not found:\n{}\n'.format( '\n'.join(existing_indices - found_indices)) print 'expecting {} indices:\n{}\n'.format( len(existing_indices), '\n'.join(sorted(existing_indices))) to_delete = set([ index for index in found_indices if index not in existing_indices ]) if to_delete: if options['noinput'] or raw_input('\n'.join([ 'Really delete ALL the unrecognized elastic indices?', 'Here are the indices that will be deleted:', '\n'.join(sorted(to_delete)), 'This operation is not reversible and all data will be lost.', 'Type "delete indices" to continue:\n', ])).lower() == 'delete indices': for index in to_delete: es.indices.delete(index) else: print 'aborted' else: print 'no indices need pruning'
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillows(instantiate=False) aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows) # mapped masters: ES indices as known in the current running code state that # correctly have the alias applied to them # unmapped masters: ES indices as known in the current running code state that # do not have the alias applied to them # stale indices: ES indices running on ES that are not part of the current source control. print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print "Reindexing master pillows that do not exist yet (ones with aliases skipped)" preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: #loop through pillows once before running greenlets to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_command(pillow_class_name) if not reindex_command: raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds )) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillows(instantiate=False) aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] pillow_states = get_pillow_states(aliasable_pillows) mapped_masters, unmapped_masters, stale_indices = pillow_states print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print( "Reindexing master pillows that do not exist yet " "(ones with aliases skipped)") preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t", print '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join( [x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_commands(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % (', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds)) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist found_indices = set(es.indices.get_aliases().keys()) existing_indices = set( pillow.es_index for pillow in filter(lambda x: isinstance(x, AliasedElasticPillow), get_all_pillows()) ) if options['verbose']: if existing_indices - found_indices: print 'the following indices were not found:\n{}\n'.format( '\n'.join(existing_indices - found_indices) ) print 'expecting {} indices:\n{}\n'.format(len(existing_indices), '\n'.join(sorted(existing_indices))) to_delete = set([index for index in found_indices if index not in existing_indices]) if to_delete: if options['noinput'] or raw_input( '\n'.join([ 'Really delete ALL the unrecognized elastic indices?', 'Here are the indices that will be deleted:', '\n'.join(sorted(to_delete)), 'This operation is not reversible and all data will be lost.', 'Type "delete indices" to continue:\n', ])).lower() == 'delete indices': for index in to_delete: es.indices.delete(index) else: print 'aborted' else: print 'no indices need pruning'
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillows(instantiate=False) aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] pillow_state_results = get_pillow_states(aliasable_pillows) print "Master indices missing aliases:" unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print ("Reindexing master pillows that do not exist yet " "(ones with aliases skipped)") preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t", print '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % ( settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]) ) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_commands(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % pillow_class_name ) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) if len(reindex_pillows) > 0: gevent.joinall(runs) try: for job in runs: job.get() except Exception: f = StringIO() traceback.print_exc(file=f) mail_admins("Pillow preindexing failed", f.getvalue()) raise else: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds ) ) print "All pillowtop reindexing jobs completed"