def create_pillow_checkpoint_snapshots(cls): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint db_seq = checkpoint.get_current_sequence_id() cls.objects.create(seq=db_seq, checkpoint_id=checkpoint.checkpoint_id, date_updated=date.today())
def handle(self, **options): date = options['date'] pillow_args = set(options['pillows'] or []) if not pillow_args and not confirm('Reset checkpoints ALL pillows?'): raise CommandError('Abort') def _pillow_match(pillow_id): return ( pillow_id in pillow_args or any(re.match(arg, pillow_id, re.IGNORECASE) for arg in pillow_args) ) all_pillows = get_all_pillow_instances() if not pillow_args: pillows = all_pillows else: pillows = [ pillow for pillow in all_pillows if _pillow_match(pillow.pillow_id) ] if not pillows: raise CommandError('No pillows match: {}'.format(options['pillows'])) if not confirm('Update checkpoints for {}?'.format('\n '.join(p.pillow_id for p in pillows))): raise CommandError('abort') for pillow in pillows: checkpoint = pillow.checkpoint historical_checkpoint = HistoricalPillowCheckpoint.objects.filter( checkpoint_id=checkpoint.checkpoint_id, date_updated__lt=date).first() if not historical_checkpoint: print(self.style.ERROR('No historical checkpoints for {} before {}'.format( checkpoint.checkpoint_id, date)) ) continue old_seq = pillow.get_last_checkpoint_sequence() new_seq = historical_checkpoint.seq if checkpoint.sequence_format == 'json' and isinstance(old_seq, dict): new_seq = str_to_kafka_seq(new_seq) diff = ('\n'.join(difflib.ndiff( pprint.pformat(old_seq).splitlines(), pprint.pformat(new_seq).splitlines()))) else: diff = 'from: {}\nto : {}'.format(old_seq, new_seq) pillow_id = pillow.pillow_id if old_seq == new_seq: print('Sequences for {} are identical, moving on.'.format(pillow_id)) continue if confirm("\nReset checkpoint for '{}' pillow to sequence from {}:\n\n{}\n".format( pillow_id, historical_checkpoint.date_updated, diff )): pillow.checkpoint.update_to(new_seq) print(self.style.SUCCESS("Checkpoint for {} updated\n".format(pillow_id)))
def handle(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] num_processes = options['num_processes'] process_number = options['process_number'] processor_chunk_size = options['processor_chunk_size'] dedicated_migration_process = options['dedicated_migration_process'] assert 0 <= process_number < num_processes assert processor_chunk_size if list_all: print("\nPillows registered in system:") for config in get_all_pillow_configs(): print('{}: {}'.format(config.section, config.name)) print("\n\tRun with --pillow-name <name> to run a pillow") print( "\n\tRun with --pillow-key <key> to run a group of pillows together\n" ) sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, list(settings.PILLOWTOPS))) sys.exit() else: pillows_to_run = [ get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key] ] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name( pillow_name, num_processes=num_processes, process_num=process_number, processor_chunk_size=processor_chunk_size, dedicated_migration_process=dedicated_migration_process) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print(pillow.checkpoint.checkpoint_id) sys.exit() else: print( "\nNo command set, please see --help for runtime instructions") sys.exit() start_pillows(pillows=[ pillow_config.get_instance() for pillow_config in pillows_to_run ])
def rewind_pillows(date): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint try: checkpoint = HistoricalPillowCheckpoint.objects.get(checkpoint_id=checkpoint.checkpoint_id, date_updated=date) seq = checkpoint.seq except HistoricalPillowCheckpoint.DoesNotExist: seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE_FOR_RESTORE[pillow.checkpoint.sequence_format] pillow.checkpoint.update_to(seq)
def rewind_pillows(date): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint try: checkpoint = ESRestorePillowCheckpoints.objects.get( checkpoint_id=checkpoint.checkpoint_id, date_updated=date) seq = checkpoint.seq except ESRestorePillowCheckpoints.DoesNotExist: seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE pillow.checkpoint.update_to(seq)
def rewind_pillows(date): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint try: checkpoint = ESRestorePillowCheckpoints.objects.get(checkpoint_id=checkpoint.checkpoint_id, date_updated=date) seq = checkpoint.seq except ESRestorePillowCheckpoints.DoesNotExist: seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE pillow.checkpoint.update_to(seq)
def rewind_pillows(date): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint try: checkpoint = HistoricalPillowCheckpoint.objects.get(checkpoint_id=checkpoint.checkpoint_id, date_updated=date) if pillow.checkpoint.sequence_format == 'json': seq = str_to_kafka_seq(checkpoint.seq) else: seq = checkpoint.seq except HistoricalPillowCheckpoint.DoesNotExist: seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE_FOR_RESTORE[pillow.checkpoint.sequence_format] pillow.checkpoint.update_to(seq)
def pillow_seq_store(): for pillow in get_all_pillow_instances(): checkpoint = pillow.get_checkpoint() store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint['_id']) if not created and force_seq_int(checkpoint['seq']) < force_seq_int(store.seq) - EPSILON: notify_exception( None, message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format(store.checkpoint_id), details={ 'pillow checkpoint seq': checkpoint['seq'], 'stored seq': store.seq }) else: store.seq = checkpoint['seq'] store.save()
def handle(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] if list_all: print("\nPillows registered in system:") for config in get_all_pillow_configs(): print(u'{}: {}'.format(config.section, config.name)) print("\n\tRun with --pillow-name <name> to run a pillow") print( "\n\tRun with --pillow-key <key> to run a group of pillows together\n" ) sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, settings.PILLOWTOPS.keys())) sys.exit() else: pillows_to_run = [ get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key] ] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name(pillow_name) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print(pillow.checkpoint.checkpoint_id) sys.exit() else: print( "\nNo command set, please see --help for runtime instructions") sys.exit() start_pillows(pillows=[ pillow_config.get_instance() for pillow_config in pillows_to_run ])
def pillow_seq_store(): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint.checkpoint_id) db_seq = checkpoint.get_current_sequence_id() store_seq = force_seq_int(store.seq) or 0 if not created and force_seq_int(db_seq) < store_seq - EPSILON: notify_exception( None, message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format(store.checkpoint_id), details={ 'pillow checkpoint seq': db_seq, 'stored seq': store.seq }) else: store.seq = db_seq store.save()
def pillow_seq_store(): for pillow in get_all_pillow_instances(): checkpoint = pillow.checkpoint store, created = PillowCheckpointSeqStore.objects.get_or_create( checkpoint_id=checkpoint.checkpoint_id) db_seq = checkpoint.get_current_sequence_id() store_seq = force_seq_int(store.seq) or 0 if not created and force_seq_int(db_seq) < store_seq - EPSILON: notify_exception( None, message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format( store.checkpoint_id), details={ 'pillow checkpoint seq': db_seq, 'stored seq': store.seq }) else: store.seq = db_seq store.save()
def handle_noargs(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] if list_all: print "\nPillows registered in system:" for config in get_all_pillow_configs(): print u'{}: {}'.format(config.section, config.name) print "\n\tRun with --pillow-name <name> to run a pillow" print "\n\tRun with --pillow-key <key> to run a group of pillows together\n" sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print "\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, settings.PILLOWTOPS.keys()) sys.exit() else: pillows_to_run = [get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key]] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name(pillow_name) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print pillow.checkpoint.checkpoint_id sys.exit() else: print "\nNo command set, please see --help for runtime instructions" sys.exit() start_pillows(pillows=[pillow_config.get_instance() for pillow_config in pillows_to_run])
def create_pillow_checkpoint_snapshots(cls): for pillow in get_all_pillow_instances(): cls.create_checkpoint_snapshot(pillow.checkpoint)
def handle(self, **options): date = options['date'] pillow_args = set(options['pillows'] or []) if not pillow_args and not confirm('Reset checkpoints ALL pillows?'): raise CommandError('Abort') def _pillow_match(pillow_id): return (pillow_id in pillow_args or any( re.match(arg, pillow_id, re.IGNORECASE) for arg in pillow_args)) all_pillows = get_all_pillow_instances() if not pillow_args: pillows = all_pillows else: pillows = [ pillow for pillow in all_pillows if _pillow_match(pillow.pillow_id) ] if not pillows: raise CommandError('No pillows match: {}'.format( options['pillows'])) if not confirm('Update checkpoints for {}?'.format('\n '.join( p.pillow_id for p in pillows))): raise CommandError('abort') for pillow in pillows: checkpoint = pillow.checkpoint historical_checkpoint = HistoricalPillowCheckpoint.objects.filter( checkpoint_id=checkpoint.checkpoint_id, date_updated__lt=date).first() if not historical_checkpoint: print( self.style.ERROR( 'No historical checkpoints for {} before {}'.format( checkpoint.checkpoint_id, date))) continue old_seq = pillow.get_last_checkpoint_sequence() new_seq = historical_checkpoint.seq if checkpoint.sequence_format == 'json' and isinstance( old_seq, dict): new_seq = str_to_kafka_seq(new_seq) diff = ('\n'.join( difflib.ndiff( pprint.pformat(old_seq).splitlines(), pprint.pformat(new_seq).splitlines()))) else: diff = 'from: {}\nto : {}'.format(old_seq, new_seq) pillow_id = pillow.pillow_id if old_seq == new_seq: print('Sequences for {} are identical, moving on.'.format( pillow_id)) continue if confirm( "\nReset checkpoint for '{}' pillow to sequence from {}:\n\n{}\n" .format(pillow_id, historical_checkpoint.date_updated, diff)): pillow.checkpoint.update_to(new_seq) print( self.style.SUCCESS( "Checkpoint for {} updated\n".format(pillow_id)))