def handle(self, *args, **options):
if len(args) != 0: raise CommandError("This command doesn't expect arguments!")
show_info = options['show_info']
list_pillows = options['list_pillows']
flip_all = options['flip_all']
flip_single = options['pillow_class']
code_red = options['code_red']
es = get_es()
pillows = get_all_pillow_instances()
aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows)
if code_red:
if raw_input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following pillows will be affected:',
'\n'.join([type(p).__name__ for p in aliased_pillows]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for pillow in aliased_pillows:
pillow.get_es_new().indices.delete(pillow.es_index)
print 'deleted elastic index: {}'.format(pillow.es_index)
checkpoint_id = pillow.checkpoint.checkpoint_id
if pillow.couch_db.doc_exist(checkpoint_id):
pillow.couch_db.delete_doc(checkpoint_id)
print 'deleted checkpoint: {}'.format(checkpoint_id)
else:
print 'Safety first!'
return
if show_info:
get_pillow_states(aliased_pillows).dump_info()
if list_pillows:
print aliased_pillows
if flip_all:
for pillow in aliased_pillows:
assume_alias_for_pillow(pillow)
print simplejson.dumps(es.get('_aliases'), indent=4)
if flip_single is not None:
pillow_class_name = flip_single
pillow_to_use = filter(lambda x: x.__class__.__name__ == pillow_class_name, aliased_pillows)
if len(pillow_to_use) != 1:
print "Unknown pillow (option --pillow <name>) class string, the options are: \n\t%s" % ', '.join(
[x.__class__.__name__ for x in aliased_pillows])
sys.exit()
target_pillow = pillow_to_use[0]
assume_alias_for_pillow(target_pillow)
print es.get('_aliases')
def handle(self, *args, **options):
runs = []
pillows = import_pillows()
aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow),
pillows)
mapped_masters, unmapped_masters, stale_indices = get_pillow_states(
pillows)
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
aliased_pillows)
print reindex_pillows
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%sis going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join(
[x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
#loop through pillows once before running greenlets to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_command(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured with its own management command reindex command - it needs one"
% pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" %
(', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
runs = []
pillows = import_pillows()
aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows)
mapped_masters, unmapped_masters, stale_indices = get_pillow_states(pillows)
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, aliased_pillows)
print reindex_pillows
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%sis going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
#loop through pillows once before running greenlets to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_command(pillow_class_name)
if not reindex_command:
raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillows(instantiate=False)
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow), pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
mapped_masters, unmapped_masters, stale_indices = get_pillow_states(aliasable_pillows)
# mapped masters: ES indices as known in the current running code state that
# correctly have the alias applied to them
# unmapped masters: ES indices as known in the current running code state that
# do not have the alias applied to them
# stale indices: ES indices running on ES that are not part of the current source control.
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print "Reindexing master pillows that do not exist yet (ones with aliases skipped)"
preindexable_pillows = filter(lambda x: not x.index_exists(), aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices, preindexable_pillows)
print "Reindexing:\n\t%s" % '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join([x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
#loop through pillows once before running greenlets to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_command(pillow_class_name)
if not reindex_command:
raise Exception("Error, pillow [%s] is not configured with its own management command reindex command - it needs one" % pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins("Pillow preindexing completed", "Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillows(instantiate=False)
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow),
pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
pillow_states = get_pillow_states(aliasable_pillows)
mapped_masters, unmapped_masters, stale_indices = pillow_states
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print(
"Reindexing master pillows that do not exist yet "
"(ones with aliases skipped)")
preindexable_pillows = filter(lambda x: not x.index_exists(),
aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
preindexable_pillows)
print "Reindexing:\n\t",
print '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (settings.EMAIL_SUBJECT_PREFIX, ', '.join(
[x.__class__.__name__ for x in reindex_pillows]))
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_commands(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % pillow_class_name)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
gevent.joinall(runs)
if len(reindex_pillows) > 0:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" %
(', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds))
print "All pillowtop reindexing jobs completed"
def handle(self, *args, **options):
runs = []
pillow_classes = get_all_pillow_classes()
aliased_classes = filter(lambda x: issubclass(x, AliasedElasticPillow),
pillow_classes)
aliasable_pillows = [p(create_index=False) for p in aliased_classes]
reindex_all = options['replace']
pillow_state_results = get_pillow_states(aliasable_pillows)
print "Master indices missing aliases:"
unmapped_indices = [x[0] for x in pillow_state_results.unmapped_masters]
print unmapped_indices
if reindex_all:
print "Reindexing ALL master pillows that are not aliased"
preindexable_pillows = aliasable_pillows
else:
print ("Reindexing master pillows that do not exist yet "
"(ones with aliases skipped)")
preindexable_pillows = filter(lambda x: not x.index_exists(),
aliasable_pillows)
reindex_pillows = filter(lambda x: x.es_index in unmapped_indices,
preindexable_pillows)
print "Reindexing:\n\t",
print '\n\t'.join(x.es_index for x in reindex_pillows)
if len(reindex_pillows) > 0:
preindex_message = """
Heads up!
%s is going to start preindexing the following pillows:
%s
This may take a while, so don't deploy until all these have reported finishing.
""" % (
settings.EMAIL_SUBJECT_PREFIX,
', '.join([x.__class__.__name__ for x in reindex_pillows])
)
mail_admins("Pillow preindexing starting", preindex_message)
start = datetime.utcnow()
for pillow in reindex_pillows:
# loop through pillows once before running greenlets
# to fail hard on misconfigured pillows
pillow_class_name = pillow.__class__.__name__
reindex_command = get_reindex_commands(pillow_class_name)
if not reindex_command:
raise Exception(
"Error, pillow [%s] is not configured "
"with its own management command reindex command "
"- it needs one" % pillow_class_name
)
for pillow in reindex_pillows:
print pillow.__class__.__name__
g = gevent.spawn(do_reindex, pillow.__class__.__name__)
runs.append(g)
if len(reindex_pillows) > 0:
gevent.joinall(runs)
try:
for job in runs:
job.get()
except Exception:
f = StringIO()
traceback.print_exc(file=f)
mail_admins("Pillow preindexing failed", f.getvalue())
raise
else:
mail_admins(
"Pillow preindexing completed",
"Reindexing %s took %s seconds" % (
', '.join([x.__class__.__name__ for x in reindex_pillows]),
(datetime.utcnow() - start).seconds
)
)
print "All pillowtop reindexing jobs completed"
