def handle(self, pillow_class, **options): """ More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint """ pillow_to_use = get_pillow_by_name(pillow_class) if not pillow_to_use: print("") print( "\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_class) sys.exit() if not options.get('interactive'): confirm = raw_input(""" You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible operation, and may take a long time, and cause extraneous updates to the requisite consumers of the _changes feeds Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % pillow_class) else: confirm = 'yes' if confirm != 'yes': print("Reset cancelled.") return print("Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id) print("\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().wrapped_sequence) pillow_to_use.checkpoint.reset() print("\n\tNew checkpoint reset to zero")
def handle(self, *labels, **options): """ More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint """ if not labels: pillow_names = [config.name for config in get_all_pillow_configs()] print "\nNo pillow specified, options are:\n\t%s\n" % ('\n\t'.join(pillow_names)) sys.exit() pillow_name = labels[0] pillow_to_use = get_pillow_by_name(pillow_name) if not pillow_to_use: print "" print "\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_name sys.exit() if not options.get('interactive'): confirm = raw_input(""" You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible operation, and may take a long time, and cause extraneous updates to the requisite consumers of the _changes feeds Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % pillow_name) else: confirm = 'yes' if confirm != 'yes': print "Reset cancelled." return print "Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id print "\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().sequence pillow_to_use.checkpoint.reset() print "\n\tNew checkpoint reset to zero"
def handle(self, pillow_class, **options): """ More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint """ pillow_to_use = get_pillow_by_name(pillow_class) if not pillow_to_use: print("") print("\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_class) sys.exit() if not options.get('interactive'): confirm = input(""" You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible operation, and may take a long time, and cause extraneous updates to the requisite consumers of the _changes feeds Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % pillow_class) else: confirm = 'yes' if confirm != 'yes': print("Reset cancelled.") return print("Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id) print("\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().wrapped_sequence) pillow_to_use.checkpoint.reset() print("\n\tNew checkpoint reset to zero")
def pillow(self): if not self._pillow: with real_pillow_settings(), override_settings( PTOP_CHECKPOINT_DELAY_OVERRIDE=None): self._pillow = get_pillow_by_name(self.pillow_name_or_instance, instantiate=True) return self._pillow
def _get_topic_to_pillow_map(): all_pillow_names = {config.name for config in get_all_pillow_configs()} return { topic: get_pillow_by_name(pillow_name) for topic, pillow_name in TOPIC_TO_PILLOW_NAME_MAP.items() if pillow_name in all_pillow_names }
def pillow_operation_api(request): pillow_name = request.POST["pillow_name"] operation = request.POST["operation"] pillow = get_pillow_by_name(pillow_name) def get_response(error=None): response = { 'pillow_name': pillow_name, 'operation': operation, 'success': error is None, 'message': error, } response.update(pillow_supervisor_status(pillow_name)) if pillow: response.update(get_pillow_json(pillow)) return json_response(response) @any_toggle_enabled(SUPPORT) def reset_pillow(request): pillow.reset_checkpoint() if supervisor.restart_pillow(pillow_name): return get_response() else: return get_response("Checkpoint reset but failed to restart pillow. " "Restart manually to complete reset.") @any_toggle_enabled(SUPPORT) def start_pillow(request): if supervisor.start_pillow(pillow_name): return get_response() else: return get_response('Unknown error') @any_toggle_enabled(SUPPORT) def stop_pillow(request): if supervisor.stop_pillow(pillow_name): return get_response() else: return get_response('Unknown error') if pillow: try: supervisor = PillowtopSupervisorApi() except Exception as e: return get_response(str(e)) try: if operation == 'reset_checkpoint': reset_pillow(request) if operation == 'start': start_pillow(request) if operation == 'stop': stop_pillow(request) if operation == 'refresh': return get_response() except SupervisorException as e: return get_response(str(e)) else: return get_response("No pillow found with name '{}'".format(pillow_name))
def __init__(self, pillow_name_or_instance): if isinstance(pillow_name_or_instance, basestring): with real_pillow_settings(), override_settings(PTOP_CHECKPOINT_DELAY_OVERRIDE=None): self.pillow = get_pillow_by_name(pillow_name_or_instance, instantiate=True) else: self.pillow = pillow_name_or_instance self.topics = self.pillow.get_change_feed().topics
def handle(self, pillow_names, **options): from_pillow = pillow_names[0] to_pillows = pillow_names[1:] logging.info(f"Attempting to split {from_pillow} into {to_pillows}") from_checkpoint = get_pillow_by_name( from_pillow).checkpoint.checkpoint_id to_checkpoints = [ get_pillow_by_name(pillow).checkpoint.checkpoint_id for pillow in to_pillows ] existing_checkpoints = list( KafkaCheckpoint.objects.filter(checkpoint_id__in=to_checkpoints). values('checkpoint_id').annotate(Max('last_modified'))) if existing_checkpoints: print(f'Some of {to_checkpoints} already exist:') for checkpoint in existing_checkpoints: print( f"{checkpoint['checkpoint_id']} last updated {checkpoint['last_modified__max']}" ) if not confirm( "Do you want to continue and overwrite existing checkpoints? [y/n]" ): sys.exit(1) if not confirm( "Are you sure you want to DELETE existing checkpoints? [y/n]" ): sys.exit(1) KafkaCheckpoint.objects.filter( checkpoint_id__in=to_checkpoints).delete() checkpoints = KafkaCheckpoint.objects.filter( checkpoint_id=from_checkpoint) for checkpoint in checkpoints: for to_checkpoint in to_checkpoints: KafkaCheckpoint.objects.update_or_create( checkpoint_id=to_checkpoint, topic=checkpoint.topic, partition=checkpoint.partition, defaults={'offset': checkpoint.offset})
def __init__(self, pillow_name_or_instance): if isinstance(pillow_name_or_instance, basestring): with real_pillow_settings(): self.pillow = get_pillow_by_name(pillow_name_or_instance, instantiate=True) else: self.pillow = pillow_name_or_instance self.topics = self.pillow.get_change_feed().topics
def handle(self, pillow_name, **options): self.pool = Pool(10) self.pillow_name = pillow_name try: pillow = get_pillow_by_name(pillow_name) except PillowNotFoundError: raise CommandError(f"Unknown pillow: {pillow_name}") if not isinstance(pillow.get_change_feed(), KafkaChangeFeed): raise CommandError(f"Only Kafka pillows are supported") self.count = 0 self.start = time.time() self.producer = ChangeProducer(auto_flush=False) for errors in self.get_next_errors(): self.pool.spawn(self._process_errors, errors)
def handle(self, filename, *args, **options): with open(filename) as f: checkpoint_map = json.loads(f.read()) for pillow_name in sorted(checkpoint_map.keys()): checkpoint_to_set = checkpoint_map[pillow_name] pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError("No pillow found with name: {}".format(pillow_name)) old_seq = pillow.get_checkpoint().sequence msg = "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format( pillow_name, old_seq, checkpoint_to_set, ) if not options['noinput'] and \ raw_input("{} Type ['y', 'yes'] to continue.\n".format(msg)) not in ['y', 'yes']: print 'skipped' continue pillow.checkpoint.update_to(checkpoint_to_set)
def handle(self, pillow_class=None, **options): if pillow_class: pillows = [get_pillow_by_name(pillow_class)] else: pillows = get_all_pillow_instances() for pillow in pillows: last_sequence = pillow.get_last_checkpoint_sequence() filepath = 'pillow_changes_{}_{}.gz'.format( pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat() ) filepath = filepath.replace(':', '') self.stdout.write("\n Writing changes to {}\n\n".format(filepath)) with gzip.open(filepath, 'wb') as file: for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False): if change: doc = change.to_dict() if change.metadata: doc['metadata'] = change.metadata.to_json() doc.pop('doc', None) file.write('{}\n'.format(json.dumps(doc)))
def handle(self, *args, **options): if options['pillow_class']: pillow = get_pillow_by_name(options['pillow_class']) else: raise CommandError('--pillow argument is required') if options['docs_filename']: docs_filename = options['docs_filename'] else: raise CommandError('--docs argument is required') skip_check = options['skip_check'] if not skip_check: if not self.check_file(docs_filename): return def doc_ids(): with open(docs_filename) as f: for line in f: line = line.strip() if line: yield line self.handle_all(pillow, doc_ids())
def handle(self, filename, **options): with open(filename) as f: checkpoint_map = json.loads(f.read()) for pillow_name in sorted(checkpoint_map.keys()): checkpoint_to_set = checkpoint_map[pillow_name] pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError( "No pillow found with name: {}".format(pillow_name)) old_seq = pillow.get_checkpoint().wrapped_sequence msg = "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format( pillow_name, old_seq, checkpoint_to_set, ) if not options['noinput'] and \ raw_input("{} Type ['y', 'yes'] to continue.\n".format(msg)) not in ['y', 'yes']: print('skipped') continue pillow.checkpoint.update_to(checkpoint_to_set)
def handle(self, *labels, **options): """ More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint """ if not labels: pillow_names = [config.name for config in get_all_pillow_configs()] print "\nNo pillow specified, options are:\n\t%s\n" % ( '\n\t'.join(pillow_names)) sys.exit() pillow_name = labels[0] pillow_to_use = get_pillow_by_name(pillow_name) if not pillow_to_use: print "" print "\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_name sys.exit() if not options.get('interactive'): confirm = raw_input(""" You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible operation, and may take a long time, and cause extraneous updates to the requisite consumers of the _changes feeds Are you sure you want to do this? Type 'yes' to continue, or 'no' to cancel: """ % pillow_name) else: confirm = 'yes' if confirm != 'yes': print "Reset cancelled." return print "Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id print "\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().sequence pillow_to_use.checkpoint.reset() print "\n\tNew checkpoint reset to zero"
raise return result def prepare_metadata(doc_ids_by_domain): domain_id_rev_list = [] for domain, all_doc_ids in doc_ids_by_domain.items(): for doc_ids in chunked(all_doc_ids, 500): doc_id_rev_list = bulk_get_revs(XFormInstance.get_db(), doc_ids) assert len(doc_id_rev_list) == len(doc_ids) domain_id_rev_list.extend([[domain, doc_id, doc_rev] for doc_id, doc_rev in doc_id_rev_list]) return domain_id_rev_list PROCESSOR, = get_pillow_by_name('DefaultChangeFeedPillow').processors PRODUCER = PROCESSOR._producer DATA_SOURCE_TYPE = PROCESSOR._data_source_type DATA_SOURCE_NAME = PROCESSOR._data_source_name def iter_case_changes(case_metadata): for domain, doc_id, doc_rev in case_metadata: yield create_case_change_meta(domain, doc_id, doc_rev) def iter_form_changes(form_metadata): for domain, doc_id, doc_rev in form_metadata: yield create_form_change_meta(domain, doc_id, doc_rev)
def pillow(self): if not self._pillow: with real_pillow_settings(), override_settings(PTOP_CHECKPOINT_DELAY_OVERRIDE=None): self._pillow = get_pillow_by_name( self.pillow_name_or_instance, instantiate=True, **self._pillow_kwargs) return self._pillow
def handle(self, pillow_names, **options): from_pillows = pillow_names[:-1] to_pillow = pillow_names[-1] logging.info(f"Attempting to merge {from_pillows} into {to_pillow}") from_checkpoints = [ get_pillow_by_name(pillow).checkpoint.checkpoint_id for pillow in from_pillows ] to_checkpoint = get_pillow_by_name(to_pillow).checkpoint.checkpoint_id existing_checkpoints = list( KafkaCheckpoint.objects.filter(checkpoint_id=to_checkpoint) .values('checkpoint_id').annotate(Max('last_modified')) ) if existing_checkpoints: print(f'{to_checkpoint} already exists:') for checkpoint in existing_checkpoints: print(f"{checkpoint['checkpoint_id']} last updated {checkpoint['last_modified__max']}") if not confirm("Do you want to continue and overwrite existing checkpoints? [y/n]"): sys.exit(1) if not confirm("Are you sure you want to DELETE existing checkpoints? [y/n]"): sys.exit(1) KafkaCheckpoint.objects.filter(checkpoint_id=to_checkpoint).delete() checkpoint_info = ( KafkaCheckpoint.objects .filter(checkpoint_id__in=from_checkpoints) .values('topic', 'partition') .annotate(Min('offset'), Max('offset'), Count('checkpoint_id')) ) number_checkpoints = checkpoint_info[0]['checkpoint_id__count'] number_nonstandard_checkpoints = sum( 1 for info in checkpoint_info if info['checkpoint_id__count'] != number_checkpoints ) if number_nonstandard_checkpoints > 0: logger.error( f'Not all checkpoints have the same topics and partitions specified. ' 'Aborting pillow merging' ) sys.exit(2) minimum_difference = min( info['offset__max'] - info['offset__min'] for info in checkpoint_info ) if minimum_difference < 0: logger.error("The minimum difference between checkpoints between pillows is less than zero") sys.exit(4) maximum_difference = max( info['offset__max'] - info['offset__min'] for info in checkpoint_info ) if maximum_difference > 0: logger.warning(f"At least one checkpoint will need to reprocess {maximum_difference} changes") if confirm("Is this amount of reprocessing acceptable y/N?"): sys.exit(3) else: logger.info("All pillows have the same offsets") for info in checkpoint_info: KafkaCheckpoint.objects.update_or_create( checkpoint_id=to_checkpoint, topic=info['topic'], partition=info['partition'], defaults={ 'offset': info['offset__min'] } ) logger.info(f"{to_checkpoint} checkpoints created")
def __init__(self, pillow_name): with real_pillow_settings(): self.pillow = get_pillow_by_name(pillow_name, instantiate=True)
def test_get_pillow_by_name_instantiate(self): self.assertEqual( FakePillow, type(get_pillow_by_name('FakePillow', instantiate=True)))
def test_get_pillow_by_name_instantiate(self): pillow = get_pillow_by_name('FakeConstructedPillowName', instantiate=True) self.assertFalse(isclass(pillow)) self.assertEqual(FakeConstructedPillow, type(pillow)) self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
def test_get_pillow_by_name_missing(self): with self.assertRaises(PillowNotFoundError): get_pillow_by_name('MissingPillow')
def reset_pillow_checkpoint(request): pillow = get_pillow_by_name(request.POST["pillow_name"]) if pillow: pillow.reset_checkpoint() return redirect("system_info")
def _get_pillow(self, pillow_name, pillow_kwargs): with real_pillow_settings(), override_settings( PTOP_CHECKPOINT_DELAY_OVERRIDE=None): return get_pillow_by_name(pillow_name, instantiate=True, **pillow_kwargs)
raise return result def prepare_metadata(doc_ids_by_domain): domain_id_rev_list = [] for domain, all_doc_ids in doc_ids_by_domain.items(): for doc_ids in chunked(all_doc_ids, 500): doc_id_rev_list = _bulk_get_revs(XFormInstance.get_db(), doc_ids) assert len(doc_id_rev_list) == len(doc_ids) domain_id_rev_list.extend([[domain, doc_id, doc_rev] for doc_id, doc_rev in doc_id_rev_list]) return domain_id_rev_list PROCESSOR, = get_pillow_by_name('DefaultChangeFeedPillow').processors PRODUCER = PROCESSOR._producer DATA_SOURCE_TYPE = PROCESSOR._data_source_type DATA_SOURCE_NAME = PROCESSOR._data_source_name def iter_case_changes(case_metadata): for domain, doc_id, doc_rev in case_metadata: yield create_case_change_meta(domain, doc_id, doc_rev) def iter_form_changes(form_metadata): for domain, doc_id, doc_rev in form_metadata: yield create_form_change_meta(domain, doc_id, doc_rev)
def test_get_pillow_by_name(self): self.assertEqual(FakePillow, get_pillow_by_name('FakePillow', instantiate=False))
def test_get_pillow_by_name_instantiate(self): self.assertEqual(FakePillow, type(get_pillow_by_name('FakePillow', instantiate=True)))
def test_get_pillow_class_by_name(self): pillow = get_pillow_by_name('FakeConstructedPillowName', instantiate=False) self.assertEqual(FakeConstructedPillow, pillow)