def create_pillow_checkpoint_snapshots(cls):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
db_seq = checkpoint.get_current_sequence_id()
cls.objects.create(seq=db_seq,
checkpoint_id=checkpoint.checkpoint_id,
date_updated=date.today())
def handle(self, **options):
date = options['date']
pillow_args = set(options['pillows'] or [])
if not pillow_args and not confirm('Reset checkpoints ALL pillows?'):
raise CommandError('Abort')
def _pillow_match(pillow_id):
return (
pillow_id in pillow_args
or any(re.match(arg, pillow_id, re.IGNORECASE) for arg in pillow_args)
)
all_pillows = get_all_pillow_instances()
if not pillow_args:
pillows = all_pillows
else:
pillows = [
pillow for pillow in all_pillows
if _pillow_match(pillow.pillow_id)
]
if not pillows:
raise CommandError('No pillows match: {}'.format(options['pillows']))
if not confirm('Update checkpoints for {}?'.format('\n '.join(p.pillow_id for p in pillows))):
raise CommandError('abort')
for pillow in pillows:
checkpoint = pillow.checkpoint
historical_checkpoint = HistoricalPillowCheckpoint.objects.filter(
checkpoint_id=checkpoint.checkpoint_id,
date_updated__lt=date).first()
if not historical_checkpoint:
print(self.style.ERROR('No historical checkpoints for {} before {}'.format(
checkpoint.checkpoint_id, date))
)
continue
old_seq = pillow.get_last_checkpoint_sequence()
new_seq = historical_checkpoint.seq
if checkpoint.sequence_format == 'json' and isinstance(old_seq, dict):
new_seq = str_to_kafka_seq(new_seq)
diff = ('\n'.join(difflib.ndiff(
pprint.pformat(old_seq).splitlines(),
pprint.pformat(new_seq).splitlines())))
else:
diff = 'from: {}\nto : {}'.format(old_seq, new_seq)
pillow_id = pillow.pillow_id
if old_seq == new_seq:
print('Sequences for {} are identical, moving on.'.format(pillow_id))
continue
if confirm("\nReset checkpoint for '{}' pillow to sequence from {}:\n\n{}\n".format(
pillow_id, historical_checkpoint.date_updated, diff
)):
pillow.checkpoint.update_to(new_seq)
print(self.style.SUCCESS("Checkpoint for {} updated\n".format(pillow_id)))
def create_pillow_checkpoint_snapshots(cls):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
db_seq = checkpoint.get_current_sequence_id()
cls.objects.create(seq=db_seq,
checkpoint_id=checkpoint.checkpoint_id,
date_updated=date.today())
def handle(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
num_processes = options['num_processes']
process_number = options['process_number']
processor_chunk_size = options['processor_chunk_size']
dedicated_migration_process = options['dedicated_migration_process']
assert 0 <= process_number < num_processes
assert processor_chunk_size
if list_all:
print("\nPillows registered in system:")
for config in get_all_pillow_configs():
print('{}: {}'.format(config.section, config.name))
print("\n\tRun with --pillow-name <name> to run a pillow")
print(
"\n\tRun with --pillow-key <key> to run a group of pillows together\n"
)
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, list(settings.PILLOWTOPS)))
sys.exit()
else:
pillows_to_run = [
get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]
]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(
pillow_name,
num_processes=num_processes,
process_num=process_number,
processor_chunk_size=processor_chunk_size,
dedicated_migration_process=dedicated_migration_process)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print(pillow.checkpoint.checkpoint_id)
sys.exit()
else:
print(
"\nNo command set, please see --help for runtime instructions")
sys.exit()
start_pillows(pillows=[
pillow_config.get_instance() for pillow_config in pillows_to_run
])
def rewind_pillows(date):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
try:
checkpoint = HistoricalPillowCheckpoint.objects.get(checkpoint_id=checkpoint.checkpoint_id,
date_updated=date)
seq = checkpoint.seq
except HistoricalPillowCheckpoint.DoesNotExist:
seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE_FOR_RESTORE[pillow.checkpoint.sequence_format]
pillow.checkpoint.update_to(seq)
def rewind_pillows(date):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
try:
checkpoint = ESRestorePillowCheckpoints.objects.get(
checkpoint_id=checkpoint.checkpoint_id, date_updated=date)
seq = checkpoint.seq
except ESRestorePillowCheckpoints.DoesNotExist:
seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE
pillow.checkpoint.update_to(seq)
def rewind_pillows(date):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
try:
checkpoint = ESRestorePillowCheckpoints.objects.get(checkpoint_id=checkpoint.checkpoint_id,
date_updated=date)
seq = checkpoint.seq
except ESRestorePillowCheckpoints.DoesNotExist:
seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE
pillow.checkpoint.update_to(seq)
def rewind_pillows(date):
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
try:
checkpoint = HistoricalPillowCheckpoint.objects.get(checkpoint_id=checkpoint.checkpoint_id,
date_updated=date)
if pillow.checkpoint.sequence_format == 'json':
seq = str_to_kafka_seq(checkpoint.seq)
else:
seq = checkpoint.seq
except HistoricalPillowCheckpoint.DoesNotExist:
seq = DEFAULT_EMPTY_CHECKPOINT_SEQUENCE_FOR_RESTORE[pillow.checkpoint.sequence_format]
pillow.checkpoint.update_to(seq)
def pillow_seq_store():
for pillow in get_all_pillow_instances():
checkpoint = pillow.get_checkpoint()
store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint['_id'])
if not created and force_seq_int(checkpoint['seq']) < force_seq_int(store.seq) - EPSILON:
notify_exception(
None,
message='Found seq number lower than previous for {}. '
'This could mean we are in a rewind state'.format(store.checkpoint_id),
details={
'pillow checkpoint seq': checkpoint['seq'],
'stored seq': store.seq
})
else:
store.seq = checkpoint['seq']
store.save()
def handle(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
if list_all:
print("\nPillows registered in system:")
for config in get_all_pillow_configs():
print(u'{}: {}'.format(config.section, config.name))
print("\n\tRun with --pillow-name <name> to run a pillow")
print(
"\n\tRun with --pillow-key <key> to run a group of pillows together\n"
)
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, settings.PILLOWTOPS.keys()))
sys.exit()
else:
pillows_to_run = [
get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]
]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(pillow_name)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print(pillow.checkpoint.checkpoint_id)
sys.exit()
else:
print(
"\nNo command set, please see --help for runtime instructions")
sys.exit()
start_pillows(pillows=[
pillow_config.get_instance() for pillow_config in pillows_to_run
])
def pillow_seq_store():
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint.checkpoint_id)
db_seq = checkpoint.get_current_sequence_id()
store_seq = force_seq_int(store.seq) or 0
if not created and force_seq_int(db_seq) < store_seq - EPSILON:
notify_exception(
None,
message='Found seq number lower than previous for {}. '
'This could mean we are in a rewind state'.format(store.checkpoint_id),
details={
'pillow checkpoint seq': db_seq,
'stored seq': store.seq
})
else:
store.seq = db_seq
store.save()
def pillow_seq_store():
for pillow in get_all_pillow_instances():
checkpoint = pillow.checkpoint
store, created = PillowCheckpointSeqStore.objects.get_or_create(
checkpoint_id=checkpoint.checkpoint_id)
db_seq = checkpoint.get_current_sequence_id()
store_seq = force_seq_int(store.seq) or 0
if not created and force_seq_int(db_seq) < store_seq - EPSILON:
notify_exception(
None,
message='Found seq number lower than previous for {}. '
'This could mean we are in a rewind state'.format(
store.checkpoint_id),
details={
'pillow checkpoint seq': db_seq,
'stored seq': store.seq
})
else:
store.seq = db_seq
store.save()
def handle_noargs(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
if list_all:
print "\nPillows registered in system:"
for config in get_all_pillow_configs():
print u'{}: {}'.format(config.section, config.name)
print "\n\tRun with --pillow-name <name> to run a pillow"
print "\n\tRun with --pillow-key <key> to run a group of pillows together\n"
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print "\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, settings.PILLOWTOPS.keys())
sys.exit()
else:
pillows_to_run = [get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(pillow_name)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print pillow.checkpoint.checkpoint_id
sys.exit()
else:
print "\nNo command set, please see --help for runtime instructions"
sys.exit()
start_pillows(pillows=[pillow_config.get_instance() for pillow_config in pillows_to_run])
def create_pillow_checkpoint_snapshots(cls):
for pillow in get_all_pillow_instances():
cls.create_checkpoint_snapshot(pillow.checkpoint)
def create_pillow_checkpoint_snapshots(cls):
for pillow in get_all_pillow_instances():
cls.create_checkpoint_snapshot(pillow.checkpoint)
def handle(self, **options):
date = options['date']
pillow_args = set(options['pillows'] or [])
if not pillow_args and not confirm('Reset checkpoints ALL pillows?'):
raise CommandError('Abort')
def _pillow_match(pillow_id):
return (pillow_id in pillow_args or any(
re.match(arg, pillow_id, re.IGNORECASE)
for arg in pillow_args))
all_pillows = get_all_pillow_instances()
if not pillow_args:
pillows = all_pillows
else:
pillows = [
pillow for pillow in all_pillows
if _pillow_match(pillow.pillow_id)
]
if not pillows:
raise CommandError('No pillows match: {}'.format(
options['pillows']))
if not confirm('Update checkpoints for {}?'.format('\n '.join(
p.pillow_id for p in pillows))):
raise CommandError('abort')
for pillow in pillows:
checkpoint = pillow.checkpoint
historical_checkpoint = HistoricalPillowCheckpoint.objects.filter(
checkpoint_id=checkpoint.checkpoint_id,
date_updated__lt=date).first()
if not historical_checkpoint:
print(
self.style.ERROR(
'No historical checkpoints for {} before {}'.format(
checkpoint.checkpoint_id, date)))
continue
old_seq = pillow.get_last_checkpoint_sequence()
new_seq = historical_checkpoint.seq
if checkpoint.sequence_format == 'json' and isinstance(
old_seq, dict):
new_seq = str_to_kafka_seq(new_seq)
diff = ('\n'.join(
difflib.ndiff(
pprint.pformat(old_seq).splitlines(),
pprint.pformat(new_seq).splitlines())))
else:
diff = 'from: {}\nto : {}'.format(old_seq, new_seq)
pillow_id = pillow.pillow_id
if old_seq == new_seq:
print('Sequences for {} are identical, moving on.'.format(
pillow_id))
continue
if confirm(
"\nReset checkpoint for '{}' pillow to sequence from {}:\n\n{}\n"
.format(pillow_id, historical_checkpoint.date_updated,
diff)):
pillow.checkpoint.update_to(new_seq)
print(
self.style.SUCCESS(
"Checkpoint for {} updated\n".format(pillow_id)))