def handle(self, pillow_class, **options):
"""
More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint
"""
pillow_to_use = get_pillow_by_name(pillow_class)
if not pillow_to_use:
print("")
print(
"\n\tPillow class [%s] not in configuration, what are you trying to do?\n"
% pillow_class)
sys.exit()
if not options.get('interactive'):
confirm = raw_input("""
You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible
operation, and may take a long time, and cause extraneous updates to the requisite
consumers of the _changes feeds Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % pillow_class)
else:
confirm = 'yes'
if confirm != 'yes':
print("Reset cancelled.")
return
print("Resetting checkpoint for %s" %
pillow_to_use.checkpoint.checkpoint_id)
print("\tOld checkpoint: %s" %
pillow_to_use.get_checkpoint().wrapped_sequence)
pillow_to_use.checkpoint.reset()
print("\n\tNew checkpoint reset to zero")
def handle(self, *labels, **options):
"""
More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint
"""
if not labels:
pillow_names = [config.name for config in get_all_pillow_configs()]
print "\nNo pillow specified, options are:\n\t%s\n" % ('\n\t'.join(pillow_names))
sys.exit()
pillow_name = labels[0]
pillow_to_use = get_pillow_by_name(pillow_name)
if not pillow_to_use:
print ""
print "\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_name
sys.exit()
if not options.get('interactive'):
confirm = raw_input("""
You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible
operation, and may take a long time, and cause extraneous updates to the requisite
consumers of the _changes feeds Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % pillow_name)
else:
confirm = 'yes'
if confirm != 'yes':
print "Reset cancelled."
return
print "Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id
print "\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().sequence
pillow_to_use.checkpoint.reset()
print "\n\tNew checkpoint reset to zero"
def handle(self, pillow_class, **options):
"""
More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint
"""
pillow_to_use = get_pillow_by_name(pillow_class)
if not pillow_to_use:
print("")
print("\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_class)
sys.exit()
if not options.get('interactive'):
confirm = input("""
You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible
operation, and may take a long time, and cause extraneous updates to the requisite
consumers of the _changes feeds Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % pillow_class)
else:
confirm = 'yes'
if confirm != 'yes':
print("Reset cancelled.")
return
print("Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id)
print("\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().wrapped_sequence)
pillow_to_use.checkpoint.reset()
print("\n\tNew checkpoint reset to zero")
def pillow(self):
if not self._pillow:
with real_pillow_settings(), override_settings(
PTOP_CHECKPOINT_DELAY_OVERRIDE=None):
self._pillow = get_pillow_by_name(self.pillow_name_or_instance,
instantiate=True)
return self._pillow
def _get_topic_to_pillow_map():
all_pillow_names = {config.name for config in get_all_pillow_configs()}
return {
topic: get_pillow_by_name(pillow_name)
for topic, pillow_name in TOPIC_TO_PILLOW_NAME_MAP.items()
if pillow_name in all_pillow_names
}
def pillow_operation_api(request):
pillow_name = request.POST["pillow_name"]
operation = request.POST["operation"]
pillow = get_pillow_by_name(pillow_name)
def get_response(error=None):
response = {
'pillow_name': pillow_name,
'operation': operation,
'success': error is None,
'message': error,
}
response.update(pillow_supervisor_status(pillow_name))
if pillow:
response.update(get_pillow_json(pillow))
return json_response(response)
@any_toggle_enabled(SUPPORT)
def reset_pillow(request):
pillow.reset_checkpoint()
if supervisor.restart_pillow(pillow_name):
return get_response()
else:
return get_response("Checkpoint reset but failed to restart pillow. "
"Restart manually to complete reset.")
@any_toggle_enabled(SUPPORT)
def start_pillow(request):
if supervisor.start_pillow(pillow_name):
return get_response()
else:
return get_response('Unknown error')
@any_toggle_enabled(SUPPORT)
def stop_pillow(request):
if supervisor.stop_pillow(pillow_name):
return get_response()
else:
return get_response('Unknown error')
if pillow:
try:
supervisor = PillowtopSupervisorApi()
except Exception as e:
return get_response(str(e))
try:
if operation == 'reset_checkpoint':
reset_pillow(request)
if operation == 'start':
start_pillow(request)
if operation == 'stop':
stop_pillow(request)
if operation == 'refresh':
return get_response()
except SupervisorException as e:
return get_response(str(e))
else:
return get_response("No pillow found with name '{}'".format(pillow_name))
def __init__(self, pillow_name_or_instance):
if isinstance(pillow_name_or_instance, basestring):
with real_pillow_settings(), override_settings(PTOP_CHECKPOINT_DELAY_OVERRIDE=None):
self.pillow = get_pillow_by_name(pillow_name_or_instance, instantiate=True)
else:
self.pillow = pillow_name_or_instance
self.topics = self.pillow.get_change_feed().topics
def handle(self, pillow_names, **options):
from_pillow = pillow_names[0]
to_pillows = pillow_names[1:]
logging.info(f"Attempting to split {from_pillow} into {to_pillows}")
from_checkpoint = get_pillow_by_name(
from_pillow).checkpoint.checkpoint_id
to_checkpoints = [
get_pillow_by_name(pillow).checkpoint.checkpoint_id
for pillow in to_pillows
]
existing_checkpoints = list(
KafkaCheckpoint.objects.filter(checkpoint_id__in=to_checkpoints).
values('checkpoint_id').annotate(Max('last_modified')))
if existing_checkpoints:
print(f'Some of {to_checkpoints} already exist:')
for checkpoint in existing_checkpoints:
print(
f"{checkpoint['checkpoint_id']} last updated {checkpoint['last_modified__max']}"
)
if not confirm(
"Do you want to continue and overwrite existing checkpoints? [y/n]"
):
sys.exit(1)
if not confirm(
"Are you sure you want to DELETE existing checkpoints? [y/n]"
):
sys.exit(1)
KafkaCheckpoint.objects.filter(
checkpoint_id__in=to_checkpoints).delete()
checkpoints = KafkaCheckpoint.objects.filter(
checkpoint_id=from_checkpoint)
for checkpoint in checkpoints:
for to_checkpoint in to_checkpoints:
KafkaCheckpoint.objects.update_or_create(
checkpoint_id=to_checkpoint,
topic=checkpoint.topic,
partition=checkpoint.partition,
defaults={'offset': checkpoint.offset})
def __init__(self, pillow_name_or_instance):
if isinstance(pillow_name_or_instance, basestring):
with real_pillow_settings():
self.pillow = get_pillow_by_name(pillow_name_or_instance,
instantiate=True)
else:
self.pillow = pillow_name_or_instance
self.topics = self.pillow.get_change_feed().topics
def handle(self, pillow_name, **options):
self.pool = Pool(10)
self.pillow_name = pillow_name
try:
pillow = get_pillow_by_name(pillow_name)
except PillowNotFoundError:
raise CommandError(f"Unknown pillow: {pillow_name}")
if not isinstance(pillow.get_change_feed(), KafkaChangeFeed):
raise CommandError(f"Only Kafka pillows are supported")
self.count = 0
self.start = time.time()
self.producer = ChangeProducer(auto_flush=False)
for errors in self.get_next_errors():
self.pool.spawn(self._process_errors, errors)
def handle(self, filename, *args, **options):
with open(filename) as f:
checkpoint_map = json.loads(f.read())
for pillow_name in sorted(checkpoint_map.keys()):
checkpoint_to_set = checkpoint_map[pillow_name]
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError("No pillow found with name: {}".format(pillow_name))
old_seq = pillow.get_checkpoint().sequence
msg = "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format(
pillow_name, old_seq, checkpoint_to_set,
)
if not options['noinput'] and \
raw_input("{} Type ['y', 'yes'] to continue.\n".format(msg)) not in ['y', 'yes']:
print 'skipped'
continue
pillow.checkpoint.update_to(checkpoint_to_set)
def handle(self, pillow_class=None, **options):
if pillow_class:
pillows = [get_pillow_by_name(pillow_class)]
else:
pillows = get_all_pillow_instances()
for pillow in pillows:
last_sequence = pillow.get_last_checkpoint_sequence()
filepath = 'pillow_changes_{}_{}.gz'.format(
pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat()
)
filepath = filepath.replace(':', '')
self.stdout.write("\n Writing changes to {}\n\n".format(filepath))
with gzip.open(filepath, 'wb') as file:
for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False):
if change:
doc = change.to_dict()
if change.metadata:
doc['metadata'] = change.metadata.to_json()
doc.pop('doc', None)
file.write('{}\n'.format(json.dumps(doc)))
def handle(self, pillow_class=None, **options):
if pillow_class:
pillows = [get_pillow_by_name(pillow_class)]
else:
pillows = get_all_pillow_instances()
for pillow in pillows:
last_sequence = pillow.get_last_checkpoint_sequence()
filepath = 'pillow_changes_{}_{}.gz'.format(
pillow.get_name(), datetime.utcnow().replace(microsecond=0).isoformat()
)
filepath = filepath.replace(':', '')
self.stdout.write("\n Writing changes to {}\n\n".format(filepath))
with gzip.open(filepath, 'wb') as file:
for change in pillow.get_change_feed().iter_changes(since=last_sequence, forever=False):
if change:
doc = change.to_dict()
if change.metadata:
doc['metadata'] = change.metadata.to_json()
doc.pop('doc', None)
file.write('{}\n'.format(json.dumps(doc)))
def handle(self, *args, **options):
if options['pillow_class']:
pillow = get_pillow_by_name(options['pillow_class'])
else:
raise CommandError('--pillow argument is required')
if options['docs_filename']:
docs_filename = options['docs_filename']
else:
raise CommandError('--docs argument is required')
skip_check = options['skip_check']
if not skip_check:
if not self.check_file(docs_filename):
return
def doc_ids():
with open(docs_filename) as f:
for line in f:
line = line.strip()
if line:
yield line
self.handle_all(pillow, doc_ids())
def handle(self, *args, **options):
if options['pillow_class']:
pillow = get_pillow_by_name(options['pillow_class'])
else:
raise CommandError('--pillow argument is required')
if options['docs_filename']:
docs_filename = options['docs_filename']
else:
raise CommandError('--docs argument is required')
skip_check = options['skip_check']
if not skip_check:
if not self.check_file(docs_filename):
return
def doc_ids():
with open(docs_filename) as f:
for line in f:
line = line.strip()
if line:
yield line
self.handle_all(pillow, doc_ids())
def handle(self, filename, **options):
with open(filename) as f:
checkpoint_map = json.loads(f.read())
for pillow_name in sorted(checkpoint_map.keys()):
checkpoint_to_set = checkpoint_map[pillow_name]
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError(
"No pillow found with name: {}".format(pillow_name))
old_seq = pillow.get_checkpoint().wrapped_sequence
msg = "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format(
pillow_name,
old_seq,
checkpoint_to_set,
)
if not options['noinput'] and \
raw_input("{} Type ['y', 'yes'] to continue.\n".format(msg)) not in ['y', 'yes']:
print('skipped')
continue
pillow.checkpoint.update_to(checkpoint_to_set)
def handle(self, *labels, **options):
"""
More targeted pillow checkpoint reset system - must specify the pillow class_name to reset the checkpoint
"""
if not labels:
pillow_names = [config.name for config in get_all_pillow_configs()]
print "\nNo pillow specified, options are:\n\t%s\n" % (
'\n\t'.join(pillow_names))
sys.exit()
pillow_name = labels[0]
pillow_to_use = get_pillow_by_name(pillow_name)
if not pillow_to_use:
print ""
print "\n\tPillow class [%s] not in configuration, what are you trying to do?\n" % pillow_name
sys.exit()
if not options.get('interactive'):
confirm = raw_input("""
You have requested to reset the checkpoints for the pillow [%s]. This is an irreversible
operation, and may take a long time, and cause extraneous updates to the requisite
consumers of the _changes feeds Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % pillow_name)
else:
confirm = 'yes'
if confirm != 'yes':
print "Reset cancelled."
return
print "Resetting checkpoint for %s" % pillow_to_use.checkpoint.checkpoint_id
print "\tOld checkpoint: %s" % pillow_to_use.get_checkpoint().sequence
pillow_to_use.checkpoint.reset()
print "\n\tNew checkpoint reset to zero"
raise
return result
def prepare_metadata(doc_ids_by_domain):
domain_id_rev_list = []
for domain, all_doc_ids in doc_ids_by_domain.items():
for doc_ids in chunked(all_doc_ids, 500):
doc_id_rev_list = bulk_get_revs(XFormInstance.get_db(), doc_ids)
assert len(doc_id_rev_list) == len(doc_ids)
domain_id_rev_list.extend([[domain, doc_id, doc_rev]
for doc_id, doc_rev in doc_id_rev_list])
return domain_id_rev_list
PROCESSOR, = get_pillow_by_name('DefaultChangeFeedPillow').processors
PRODUCER = PROCESSOR._producer
DATA_SOURCE_TYPE = PROCESSOR._data_source_type
DATA_SOURCE_NAME = PROCESSOR._data_source_name
def iter_case_changes(case_metadata):
for domain, doc_id, doc_rev in case_metadata:
yield create_case_change_meta(domain, doc_id, doc_rev)
def iter_form_changes(form_metadata):
for domain, doc_id, doc_rev in form_metadata:
yield create_form_change_meta(domain, doc_id, doc_rev)
def pillow(self):
if not self._pillow:
with real_pillow_settings(), override_settings(PTOP_CHECKPOINT_DELAY_OVERRIDE=None):
self._pillow = get_pillow_by_name(
self.pillow_name_or_instance, instantiate=True, **self._pillow_kwargs)
return self._pillow
def handle(self, pillow_names, **options):
from_pillows = pillow_names[:-1]
to_pillow = pillow_names[-1]
logging.info(f"Attempting to merge {from_pillows} into {to_pillow}")
from_checkpoints = [
get_pillow_by_name(pillow).checkpoint.checkpoint_id
for pillow in from_pillows
]
to_checkpoint = get_pillow_by_name(to_pillow).checkpoint.checkpoint_id
existing_checkpoints = list(
KafkaCheckpoint.objects.filter(checkpoint_id=to_checkpoint)
.values('checkpoint_id').annotate(Max('last_modified'))
)
if existing_checkpoints:
print(f'{to_checkpoint} already exists:')
for checkpoint in existing_checkpoints:
print(f"{checkpoint['checkpoint_id']} last updated {checkpoint['last_modified__max']}")
if not confirm("Do you want to continue and overwrite existing checkpoints? [y/n]"):
sys.exit(1)
if not confirm("Are you sure you want to DELETE existing checkpoints? [y/n]"):
sys.exit(1)
KafkaCheckpoint.objects.filter(checkpoint_id=to_checkpoint).delete()
checkpoint_info = (
KafkaCheckpoint.objects
.filter(checkpoint_id__in=from_checkpoints)
.values('topic', 'partition')
.annotate(Min('offset'), Max('offset'), Count('checkpoint_id'))
)
number_checkpoints = checkpoint_info[0]['checkpoint_id__count']
number_nonstandard_checkpoints = sum(
1 for info in checkpoint_info
if info['checkpoint_id__count'] != number_checkpoints
)
if number_nonstandard_checkpoints > 0:
logger.error(
f'Not all checkpoints have the same topics and partitions specified. '
'Aborting pillow merging'
)
sys.exit(2)
minimum_difference = min(
info['offset__max'] - info['offset__min']
for info in checkpoint_info
)
if minimum_difference < 0:
logger.error("The minimum difference between checkpoints between pillows is less than zero")
sys.exit(4)
maximum_difference = max(
info['offset__max'] - info['offset__min']
for info in checkpoint_info
)
if maximum_difference > 0:
logger.warning(f"At least one checkpoint will need to reprocess {maximum_difference} changes")
if confirm("Is this amount of reprocessing acceptable y/N?"):
sys.exit(3)
else:
logger.info("All pillows have the same offsets")
for info in checkpoint_info:
KafkaCheckpoint.objects.update_or_create(
checkpoint_id=to_checkpoint,
topic=info['topic'],
partition=info['partition'],
defaults={
'offset': info['offset__min']
}
)
logger.info(f"{to_checkpoint} checkpoints created")
def __init__(self, pillow_name):
with real_pillow_settings():
self.pillow = get_pillow_by_name(pillow_name, instantiate=True)
def test_get_pillow_by_name_instantiate(self):
self.assertEqual(
FakePillow, type(get_pillow_by_name('FakePillow',
instantiate=True)))
def test_get_pillow_by_name_instantiate(self):
pillow = get_pillow_by_name('FakeConstructedPillowName', instantiate=True)
self.assertFalse(isclass(pillow))
self.assertEqual(FakeConstructedPillow, type(pillow))
self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
def test_get_pillow_by_name_missing(self):
with self.assertRaises(PillowNotFoundError):
get_pillow_by_name('MissingPillow')
def reset_pillow_checkpoint(request):
pillow = get_pillow_by_name(request.POST["pillow_name"])
if pillow:
pillow.reset_checkpoint()
return redirect("system_info")
def _get_pillow(self, pillow_name, pillow_kwargs):
with real_pillow_settings(), override_settings(
PTOP_CHECKPOINT_DELAY_OVERRIDE=None):
return get_pillow_by_name(pillow_name,
instantiate=True,
**pillow_kwargs)
def test_get_pillow_by_name_instantiate(self):
pillow = get_pillow_by_name('FakeConstructedPillowName',
instantiate=True)
self.assertFalse(isclass(pillow))
self.assertEqual(FakeConstructedPillow, type(pillow))
self.assertEqual(pillow.get_name(), 'FakeConstructedPillowName')
raise
return result
def prepare_metadata(doc_ids_by_domain):
domain_id_rev_list = []
for domain, all_doc_ids in doc_ids_by_domain.items():
for doc_ids in chunked(all_doc_ids, 500):
doc_id_rev_list = _bulk_get_revs(XFormInstance.get_db(), doc_ids)
assert len(doc_id_rev_list) == len(doc_ids)
domain_id_rev_list.extend([[domain, doc_id, doc_rev]
for doc_id, doc_rev in doc_id_rev_list])
return domain_id_rev_list
PROCESSOR, = get_pillow_by_name('DefaultChangeFeedPillow').processors
PRODUCER = PROCESSOR._producer
DATA_SOURCE_TYPE = PROCESSOR._data_source_type
DATA_SOURCE_NAME = PROCESSOR._data_source_name
def iter_case_changes(case_metadata):
for domain, doc_id, doc_rev in case_metadata:
yield create_case_change_meta(domain, doc_id, doc_rev)
def iter_form_changes(form_metadata):
for domain, doc_id, doc_rev in form_metadata:
yield create_form_change_meta(domain, doc_id, doc_rev)
def test_get_pillow_by_name_missing(self):
with self.assertRaises(PillowNotFoundError):
get_pillow_by_name('MissingPillow')
def test_get_pillow_by_name(self):
self.assertEqual(FakePillow, get_pillow_by_name('FakePillow', instantiate=False))
def test_get_pillow_by_name_instantiate(self):
self.assertEqual(FakePillow, type(get_pillow_by_name('FakePillow', instantiate=True)))
def reset_pillow_checkpoint(request):
pillow = get_pillow_by_name(request.POST["pillow_name"])
if pillow:
pillow.reset_checkpoint()
return redirect("system_info")
def test_get_pillow_class_by_name(self):
pillow = get_pillow_by_name('FakeConstructedPillowName', instantiate=False)
self.assertEqual(FakeConstructedPillow, pillow)
def __init__(self, pillow_name):
with real_pillow_settings():
self.pillow = get_pillow_by_name(pillow_name, instantiate=True)
def test_get_pillow_class_by_name(self):
pillow = get_pillow_by_name('FakeConstructedPillowName',
instantiate=False)
self.assertEqual(FakeConstructedPillow, pillow)
def test_get_pillow_by_name(self):
self.assertEqual(FakePillow,
get_pillow_by_name('FakePillow', instantiate=False))
