def handle(self, *args, **options): if len(args) != 0: raise CommandError("This command doesn't expect arguments!") show_info = options['show_info'] list_pillows = options['list_pillows'] flip_all = options['flip_all'] flip_single = options['pillow_class'] code_red = options['code_red'] es = get_es() pillows = get_all_pillow_instances() aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows) if code_red: if raw_input('\n'.join([ 'CODE RED!!!', 'Really delete ALL the elastic indices and pillow checkpoints?', 'The following pillows will be affected:', '\n'.join([type(p).__name__ for p in aliased_pillows]), 'This is a PERMANENT action. (Type "code red" to continue):', '', ])).lower() == 'code red': for pillow in aliased_pillows: pillow.get_es_new().indices.delete(pillow.es_index) print 'deleted elastic index: {}'.format(pillow.es_index) checkpoint_id = pillow.checkpoint.checkpoint_id if pillow.couch_db.doc_exist(checkpoint_id): pillow.couch_db.delete_doc(checkpoint_id) print 'deleted checkpoint: {}'.format(checkpoint_id) else: print 'Safety first!' return if show_info: get_pillow_states(aliased_pillows).dump_info() if list_pillows: print aliased_pillows if flip_all: for pillow in aliased_pillows: assume_alias_for_pillow(pillow) print simplejson.dumps(es.get('_aliases'), indent=4) if flip_single is not None: pillow_class_name = flip_single pillow_to_use = filter(lambda x: x.__class__.__name__ == pillow_class_name, aliased_pillows) if len(pillow_to_use) != 1: print "Unknown pillow (option --pillow <name>) class string, the options are: \n\t%s" % ', '.join( [x.__class__.__name__ for x in aliased_pillows]) sys.exit() target_pillow = pillow_to_use[0] assume_alias_for_pillow(target_pillow) print es.get('_aliases')
def handle(self, *args, **options): runs = [] pillows = import_pillows() aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows) mapped_masters, unmapped_masters, stale_indices = get_pillow_states( pillows) print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, aliased_pillows) print reindex_pillows if len(reindex_pillows) > 0: preindex_message = """ Heads up! %sis going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join( [x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: #loop through pillows once before running greenlets to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_command(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % (', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds)) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): runs = [] pillows = import_pillows() aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows) mapped_masters, unmapped_masters, stale_indices = get_pillow_states(pillows) print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, aliased_pillows) print reindex_pillows if len(reindex_pillows) > 0: preindex_message = """ Heads up! %sis going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: #loop through pillows once before running greenlets to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_command(pillow_class_name) if not reindex_command: raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds )) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillows(instantiate=False) aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows) # mapped masters: ES indices as known in the current running code state that # correctly have the alias applied to them # unmapped masters: ES indices as known in the current running code state that # do not have the alias applied to them # stale indices: ES indices running on ES that are not part of the current source control. print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print "Reindexing master pillows that do not exist yet (ones with aliases skipped)" preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: #loop through pillows once before running greenlets to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_command(pillow_class_name) if not reindex_command: raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds )) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillows(instantiate=False) aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] pillow_states = get_pillow_states(aliasable_pillows) mapped_masters, unmapped_masters, stale_indices = pillow_states print "Master indices missing aliases:" unmapped_indices = [x[0] for x in unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print( "Reindexing master pillows that do not exist yet " "(ones with aliases skipped)") preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t", print '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % (settings.EMAIL_SUBJECT_PREFIX, ', '.join( [x.__class__.__name__ for x in reindex_pillows])) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_commands(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % pillow_class_name) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) gevent.joinall(runs) if len(reindex_pillows) > 0: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % (', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds)) print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options): runs = [] pillow_classes = get_all_pillow_classes() aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes) aliasable_pillows = [p(create_index=False) for p in aliased_classes] reindex_all = options['replace'] pillow_state_results = get_pillow_states(aliasable_pillows) print "Master indices missing aliases:" unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters] print unmapped_indices if reindex_all: print "Reindexing ALL master pillows that are not aliased" preindexable_pillows = aliasable_pillows else: print ("Reindexing master pillows that do not exist yet " "(ones with aliases skipped)") preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows) reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows) print "Reindexing:\n\t", print '\n\t'.join(x.es_index for x in reindex_pillows) if len(reindex_pillows) > 0: preindex_message = """ Heads up! %s is going to start preindexing the following pillows: %s This may take a while, so don't deploy until all these have reported finishing. """ % ( settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]) ) mail_admins("Pillow preindexing starting", preindex_message) start = datetime.utcnow() for pillow in reindex_pillows: # loop through pillows once before running greenlets # to fail hard on misconfigured pillows pillow_class_name = pillow.__class__.__name__ reindex_command = get_reindex_commands(pillow_class_name) if not reindex_command: raise Exception( "Error, pillow [%s] is not configured " "with its own management command reindex command " "- it needs one" % pillow_class_name ) for pillow in reindex_pillows: print pillow.__class__.__name__ g = gevent.spawn(do_reindex, pillow.__class__.__name__) runs.append(g) if len(reindex_pillows) > 0: gevent.joinall(runs) try: for job in runs: job.get() except Exception: f = StringIO() traceback.print_exc(file=f) mail_admins("Pillow preindexing failed", f.getvalue()) raise else: mail_admins( "Pillow preindexing completed", "Reindexing %s took %s seconds" % ( ', '.join([x.__class__.__name__ for x in reindex_pillows]), (datetime.utcnow() - start).seconds ) ) print "All pillowtop reindexing jobs completed"