def handle(self, *args, **options): es = get_es_new() # call this before getting existing indices because apparently getting the pillow will create the index # if it doesn't exist found_indices = set(es.indices.get_aliases().keys()) existing_indices = set( pillow.es_index for pillow in filter(lambda x: isinstance(x, AliasedElasticPillow), get_all_pillow_instances()) ) if options['verbose']: if existing_indices - found_indices: print 'the following indices were not found:\n{}\n'.format( '\n'.join(existing_indices - found_indices) ) print 'expecting {} indices:\n{}\n'.format(len(existing_indices), '\n'.join(sorted(existing_indices))) to_delete = set([index for index in found_indices if index not in existing_indices]) if to_delete: if options['noinput'] or raw_input( '\n'.join([ 'Really delete ALL the unrecognized elastic indices?', 'Here are the indices that will be deleted:', '\n'.join(sorted(to_delete)), 'This operation is not reversible and all data will be lost.', 'Type "delete indices" to continue:\n', ])).lower() == 'delete indices': for index in to_delete: es.indices.delete(index) else: print 'aborted' else: print 'no indices need pruning'
def test_get_pillow_instances(self): pillows = get_all_pillow_instances() self.assertEquals(len(pillows), 1) pillow = pillows[0] self.assertFalse(isclass(pillow)) self.assertEqual(FakeConstructedPillow, type(pillow)) self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
def test_all_pillows(self): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint current_seq = checkpoint.get_current_sequence_id() HistoricalPillowCheckpoint.create_checkpoint_snapshot(checkpoint) latest = HistoricalPillowCheckpoint.get_latest(checkpoint.checkpoint_id) checkpoint.reset() checkpoint.update_to(latest.seq) if checkpoint.get_current_sequence_id() != current_seq: self.assertDictEqual(json.loads(checkpoint.get_current_sequence_id()), json.loads(current_seq))
def handle(self, *args, **options): if len(args) != 0: raise CommandError("This command doesn't expect arguments!") show_info = options['show_info'] list_pillows = options['list_pillows'] flip_all = options['flip_all'] flip_single = options['pillow_class'] code_red = options['code_red'] es = get_es() pillows = get_all_pillow_instances() aliased_pillows = filter(lambda x: isinstance(x, AliasedElasticPillow), pillows) if code_red: if raw_input('\n'.join([ 'CODE RED!!!', 'Really delete ALL the elastic indices and pillow checkpoints?', 'The following pillows will be affected:', '\n'.join([type(p).__name__ for p in aliased_pillows]), 'This is a PERMANENT action. (Type "code red" to continue):', '', ])).lower() == 'code red': for pillow in aliased_pillows: pillow.get_es_new().indices.delete(pillow.es_index) print 'deleted elastic index: {}'.format(pillow.es_index) checkpoint_id = pillow.checkpoint.checkpoint_id if pillow.couch_db.doc_exist(checkpoint_id): pillow.couch_db.delete_doc(checkpoint_id) print 'deleted checkpoint: {}'.format(checkpoint_id) else: print 'Safety first!' return if show_info: get_pillow_states(aliased_pillows).dump_info() if list_pillows: print aliased_pillows if flip_all: for pillow in aliased_pillows: assume_alias_for_pillow(pillow) print simplejson.dumps(es.get('_aliases'), indent=4) if flip_single is not None: pillow_class_name = flip_single pillow_to_use = filter(lambda x: x.__class__.__name__ == pillow_class_name, aliased_pillows) if len(pillow_to_use) != 1: print "Unknown pillow (option --pillow <name>) class string, the options are: \n\t%s" % ', '.join( [x.__class__.__name__ for x in aliased_pillows]) sys.exit() target_pillow = pillow_to_use[0] assume_alias_for_pillow(target_pillow) print es.get('_aliases')
def validate_checkpoints(print_only): for pillow in get_all_pillow_instances(): if isinstance(pillow.get_change_feed(), KafkaChangeFeed): checkpoint_dict = _get_checkpoint_dict(pillow) try: validate_offsets(checkpoint_dict) except UnavailableKafkaOffset as e: message = u'Problem with checkpoint for {}: {}'.format( pillow.pillow_id, e) if print_only: print message else: raise Exception(message)
def validate_checkpoints(print_only): for pillow in get_all_pillow_instances(): if isinstance(pillow.get_change_feed(), KafkaChangeFeed): checkpoint_dict = _get_checkpoint_dict(pillow) try: validate_offsets(checkpoint_dict) except UnavailableKafkaOffset as e: message = u'Problem with checkpoint for {}: {}'.format( pillow.pillow_id, e ) if print_only: print message else: raise Exception(message)
def validate_checkpoints(print_only): for pillow in get_all_pillow_instances(): if (pillow.pillow_id in getattr(settings, 'ACTIVE_PILLOW_NAMES', [pillow.pillow_id]) and isinstance(pillow.get_change_feed(), KafkaChangeFeed)): checkpoint_dict = _get_checkpoint_dict(pillow) try: validate_offsets(checkpoint_dict) except UnavailableKafkaOffset as e: message = 'Problem with checkpoint for {}: {}'.format( pillow.pillow_id, e ) if print_only: print(message) else: raise Exception(message)
def validate_checkpoints(print_only): for pillow in get_all_pillow_instances(): if (pillow.pillow_id in getattr(settings, 'ACTIVE_PILLOW_NAMES', [pillow.pillow_id]) and isinstance(pillow.get_change_feed(), KafkaChangeFeed)): checkpoint_dict = _get_checkpoint_dict(pillow) try: validate_offsets(checkpoint_dict) except UnavailableKafkaOffset as e: message = 'Problem with checkpoint for {}: {}'.format( pillow.pillow_id, e) if print_only: print(message) else: raise Exception(message)
def copy_checkpoints(apps, schema_editor): from pillowtop import get_all_pillow_instances DjangoPillowCheckpoint = apps.get_model('pillowtop', 'DjangoPillowCheckpoint') current_checkoint_ids = set( [pillow_instance.checkpoint.checkpoint_id for pillow_instance in get_all_pillow_instances()] ) for old_id, new_id in RENAMES: try: checkpoint = DjangoPillowCheckpoint.objects.get(checkpoint_id=old_id) # since checkpoint_id is the primary key, this should make a new model # which is good in case we need to rollback checkpoint.checkpoint_id = new_id checkpoint.save() except DjangoPillowCheckpoint.DoesNotExist: if not settings.UNIT_TESTING: print 'warning: legacy pillow checkpoint with ID {} not found'.format(old_id) if new_id not in current_checkoint_ids and not settings.UNIT_TESTING: print 'warning: no current pillow found with checkpoint ID {}'.format(new_id)
def start_pillows(pillows=None): """ Actual runner for running pillow processes. Use this to run pillows. """ run_pillows = pillows or get_all_pillow_instances() try: while True: jobs = [] print("[pillowtop] Starting pillow processes") for pillow_class in run_pillows: p = multiprocessing.Process(target=pillow_class.run) p.start() jobs.append(p) print("[pillowtop] all processes started, pids: %s" % ([x.pid for x in jobs])) for j in jobs: j.join() print("[pillowtop] All processes complete, restarting") except KeyboardInterrupt: sys.exit()
def handle(self, pillow_class=None, **options): if pillow_class: pillows = [get_pillow_by_name(pillow_class)] else: pillows = get_all_pillow_instances() for pillow in pillows: last_sequence = pillow.get_last_checkpoint_sequence() filepath = 'pillow_changes_{}_{}.gz'.format( pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat() ) filepath = filepath.replace(':', '') self.stdout.write("\n Writing changes to {}\n\n".format(filepath)) with gzip.open(filepath, 'wb') as file: for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False): if change: doc = change.to_dict() if change.metadata: doc['metadata'] = change.metadata.to_json() doc.pop('doc', None) file.write('{}\n'.format(json.dumps(doc)))
def handle(self, pillow_name, process_count, **options): all_pillows = get_all_pillow_instances() pillows = [ pillow for pillow in all_pillows if pillow.pillow_id == pillow_name ] if not pillows: all_names = '\n\t'.join( [pillow.pillow_id for pillow in all_pillows]) raise CommandError( f'No pillow found: {pillow_name}.\n\t{all_names}') pillow = pillows[0] change_feed = pillow.get_change_feed() if not isinstance(change_feed, KafkaChangeFeed): raise CommandError(f"Pillow '{pillow_name}' is not a Kafka pillow") topic_partitions = [] for topic in change_feed.topics: for partition in change_feed.consumer.partitions_for_topic(topic): topic_partitions.append(TopicPartition(topic, partition)) topic_partitions.sort() process_assignments = [ topic_partitions[num::process_count] for num in range(process_count) ] if options['csv']: row_formatter = CSVRowFormatter() else: row_formatter = TableRowFormatter([len(pillow_name) + 4, 10]) writer = SimpleTableWriter(self.stdout, row_formatter) writer.write_table(['Pillow Process', 'Kafka Topics'], [[ f'{pillow_name}:{process_num}', ', '.join( [f'{tp.topic}-{tp.partition}' for tp in assignments]) ] for process_num, assignments in enumerate(process_assignments)])
def test_get_all_pillow_instances(self): pillows = get_all_pillow_instances() self.assertEquals(len(pillows), 1) self.assertFalse(isclass(pillows[0]))