def handle(self, *args, **options):
es = get_es_new()
# call this before getting existing indices because apparently getting the pillow will create the index
# if it doesn't exist
found_indices = set(es.indices.get_aliases().keys())
existing_indices = set(
pillow.es_index for pillow in filter(lambda x: isinstance(x, AliasedElasticPillow),
get_all_pillow_instances())
)
if options['verbose']:
if existing_indices - found_indices:
print 'the following indices were not found:\n{}\n'.format(
'\n'.join(existing_indices - found_indices)
)
print 'expecting {} indices:\n{}\n'.format(len(existing_indices),
'\n'.join(sorted(existing_indices)))
to_delete = set([index for index in found_indices if index not in existing_indices])
if to_delete:
if options['noinput'] or raw_input(
'\n'.join([
'Really delete ALL the unrecognized elastic indices?',
'Here are the indices that will be deleted:',
'\n'.join(sorted(to_delete)),
'This operation is not reversible and all data will be lost.',
'Type "delete indices" to continue:\n',
])).lower() == 'delete indices':
for index in to_delete:
es.indices.delete(index)
else:
print 'aborted'
else:
print 'no indices need pruning'
def test_get_pillow_instances(self):
pillows = get_all_pillow_instances()
self.assertEquals(len(pillows), 1)
pillow = pillows[0]
self.assertFalse(isclass(pillow))
self.assertEqual(FakeConstructedPillow, type(pillow))
self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
def test_get_pillow_instances(self):
pillows = get_all_pillow_instances()
self.assertEquals(len(pillows), 1)
pillow = pillows[0]
self.assertFalse(isclass(pillow))
self.assertEqual(FakeConstructedPillow, type(pillow))
self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
def test_all_pillows(self):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
current_seq = checkpoint.get_current_sequence_id()
HistoricalPillowCheckpoint.create_checkpoint_snapshot(checkpoint)
latest = HistoricalPillowCheckpoint.get_latest(checkpoint.checkpoint_id)
checkpoint.reset()
checkpoint.update_to(latest.seq)
if checkpoint.get_current_sequence_id() != current_seq:
self.assertDictEqual(json.loads(checkpoint.get_current_sequence_id()), json.loads(current_seq))
def test_all_pillows(self):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
current_seq = checkpoint.get_current_sequence_id()
HistoricalPillowCheckpoint.create_checkpoint_snapshot(checkpoint)
latest = HistoricalPillowCheckpoint.get_latest(checkpoint.checkpoint_id)
checkpoint.reset()
checkpoint.update_to(latest.seq)
if checkpoint.get_current_sequence_id() != current_seq:
self.assertDictEqual(json.loads(checkpoint.get_current_sequence_id()), json.loads(current_seq))
def handle(self, *args, **options):
if len(args) != 0: raise CommandError("This command doesn't expect arguments!")
show_info = options['show_info']
list_pillows = options['list_pillows']
flip_all = options['flip_all']
flip_single = options['pillow_class']
code_red = options['code_red']
es = get_es()
pillows = get_all_pillow_instances()
aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows)
if code_red:
if raw_input('\n'.join([
'CODE RED!!!',
'Really delete ALL the elastic indices and pillow checkpoints?',
'The following pillows will be affected:',
'\n'.join([type(p).__name__ for p in aliased_pillows]),
'This is a PERMANENT action. (Type "code red" to continue):',
'',
])).lower() == 'code red':
for pillow in aliased_pillows:
pillow.get_es_new().indices.delete(pillow.es_index)
print 'deleted elastic index: {}'.format(pillow.es_index)
checkpoint_id = pillow.checkpoint.checkpoint_id
if pillow.couch_db.doc_exist(checkpoint_id):
pillow.couch_db.delete_doc(checkpoint_id)
print 'deleted checkpoint: {}'.format(checkpoint_id)
else:
print 'Safety first!'
return
if show_info:
get_pillow_states(aliased_pillows).dump_info()
if list_pillows:
print aliased_pillows
if flip_all:
for pillow in aliased_pillows:
assume_alias_for_pillow(pillow)
print simplejson.dumps(es.get('_aliases'), indent=4)
if flip_single is not None:
pillow_class_name = flip_single
pillow_to_use = filter(lambda x: x.__class__.__name__ == pillow_class_name, aliased_pillows)
if len(pillow_to_use) != 1:
print "Unknown pillow (option --pillow <name>) class string, the options are: \n\t%s" % ', '.join(
[x.__class__.__name__ for x in aliased_pillows])
sys.exit()
target_pillow = pillow_to_use[0]
assume_alias_for_pillow(target_pillow)
print es.get('_aliases')
def validate_checkpoints(print_only):
for pillow in get_all_pillow_instances():
if isinstance(pillow.get_change_feed(), KafkaChangeFeed):
checkpoint_dict = _get_checkpoint_dict(pillow)
try:
validate_offsets(checkpoint_dict)
except UnavailableKafkaOffset as e:
message = u'Problem with checkpoint for {}: {}'.format(
pillow.pillow_id, e)
if print_only:
print message
else:
raise Exception(message)
def validate_checkpoints(print_only):
for pillow in get_all_pillow_instances():
if isinstance(pillow.get_change_feed(), KafkaChangeFeed):
checkpoint_dict = _get_checkpoint_dict(pillow)
try:
validate_offsets(checkpoint_dict)
except UnavailableKafkaOffset as e:
message = u'Problem with checkpoint for {}: {}'.format(
pillow.pillow_id, e
)
if print_only:
print message
else:
raise Exception(message)
def validate_checkpoints(print_only):
for pillow in get_all_pillow_instances():
if (pillow.pillow_id in getattr(settings, 'ACTIVE_PILLOW_NAMES', [pillow.pillow_id])
and isinstance(pillow.get_change_feed(), KafkaChangeFeed)):
checkpoint_dict = _get_checkpoint_dict(pillow)
try:
validate_offsets(checkpoint_dict)
except UnavailableKafkaOffset as e:
message = 'Problem with checkpoint for {}: {}'.format(
pillow.pillow_id, e
)
if print_only:
print(message)
else:
raise Exception(message)
def validate_checkpoints(print_only):
for pillow in get_all_pillow_instances():
if (pillow.pillow_id in getattr(settings, 'ACTIVE_PILLOW_NAMES',
[pillow.pillow_id])
and isinstance(pillow.get_change_feed(), KafkaChangeFeed)):
checkpoint_dict = _get_checkpoint_dict(pillow)
try:
validate_offsets(checkpoint_dict)
except UnavailableKafkaOffset as e:
message = 'Problem with checkpoint for {}: {}'.format(
pillow.pillow_id, e)
if print_only:
print(message)
else:
raise Exception(message)
def copy_checkpoints(apps, schema_editor):
from pillowtop import get_all_pillow_instances
DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint')
current_checkoint_ids = set(
[pillow_instance.checkpoint.checkpoint_id for pillow_instance in get_all_pillow_instances()]
)
for old_id, new_id in RENAMES:
try:
checkpoint = DjangoPillowCheckpoint.objects.get(checkpoint_id=old_id)
# since checkpoint_id is the primary key, this should make a new model
# which is good in case we need to rollback
checkpoint.checkpoint_id = new_id
checkpoint.save()
except DjangoPillowCheckpoint.DoesNotExist:
if not settings.UNIT_TESTING:
print 'warning: legacy pillow checkpoint with ID {} not found'.format(old_id)
if new_id not in current_checkoint_ids and not settings.UNIT_TESTING:
print 'warning: no current pillow found with checkpoint ID {}'.format(new_id)
def start_pillows(pillows=None):
"""
Actual runner for running pillow processes. Use this to run pillows.
"""
run_pillows = pillows or get_all_pillow_instances()
try:
while True:
jobs = []
print("[pillowtop] Starting pillow processes")
for pillow_class in run_pillows:
p = multiprocessing.Process(target=pillow_class.run)
p.start()
jobs.append(p)
print("[pillowtop] all processes started, pids: %s" % ([x.pid for x in jobs]))
for j in jobs:
j.join()
print("[pillowtop] All processes complete, restarting")
except KeyboardInterrupt:
sys.exit()
def handle(self, pillow_class=None, **options):
if pillow_class:
pillows = [get_pillow_by_name(pillow_class)]
else:
pillows = get_all_pillow_instances()
for pillow in pillows:
last_sequence = pillow.get_last_checkpoint_sequence()
filepath = 'pillow_changes_{}_{}.gz'.format(
pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat()
)
filepath = filepath.replace(':', '')
self.stdout.write("\n Writing changes to {}\n\n".format(filepath))
with gzip.open(filepath, 'wb') as file:
for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False):
if change:
doc = change.to_dict()
if change.metadata:
doc['metadata'] = change.metadata.to_json()
doc.pop('doc', None)
file.write('{}\n'.format(json.dumps(doc)))
def handle(self, pillow_class=None, **options):
if pillow_class:
pillows = [get_pillow_by_name(pillow_class)]
else:
pillows = get_all_pillow_instances()
for pillow in pillows:
last_sequence = pillow.get_last_checkpoint_sequence()
filepath = 'pillow_changes_{}_{}.gz'.format(
pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat()
)
filepath = filepath.replace(':', '')
self.stdout.write("\n Writing changes to {}\n\n".format(filepath))
with gzip.open(filepath, 'wb') as file:
for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False):
if change:
doc = change.to_dict()
if change.metadata:
doc['metadata'] = change.metadata.to_json()
doc.pop('doc', None)
file.write('{}\n'.format(json.dumps(doc)))
def handle(self, pillow_name, process_count, **options):
all_pillows = get_all_pillow_instances()
pillows = [
pillow for pillow in all_pillows if pillow.pillow_id == pillow_name
]
if not pillows:
all_names = '\n\t'.join(
[pillow.pillow_id for pillow in all_pillows])
raise CommandError(
f'No pillow found: {pillow_name}.\n\t{all_names}')
pillow = pillows[0]
change_feed = pillow.get_change_feed()
if not isinstance(change_feed, KafkaChangeFeed):
raise CommandError(f"Pillow '{pillow_name}' is not a Kafka pillow")
topic_partitions = []
for topic in change_feed.topics:
for partition in change_feed.consumer.partitions_for_topic(topic):
topic_partitions.append(TopicPartition(topic, partition))
topic_partitions.sort()
process_assignments = [
topic_partitions[num::process_count]
for num in range(process_count)
]
if options['csv']:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([len(pillow_name) + 4, 10])
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Pillow Process', 'Kafka Topics'], [[
f'{pillow_name}:{process_num}', ', '.join(
[f'{tp.topic}-{tp.partition}' for tp in assignments])
] for process_num, assignments in enumerate(process_assignments)])
def test_get_all_pillow_instances(self):
pillows = get_all_pillow_instances()
self.assertEquals(len(pillows), 1)
self.assertFalse(isclass(pillows[0]))
def test_get_all_pillow_instances(self):
pillows = get_all_pillow_instances()
self.assertEquals(len(pillows), 1)
self.assertFalse(isclass(pillows[0]))
