def test_checkpoint_creation(self, reindex_id, pillow_name):
# checks that checkpoipnts are set to the latest checkpoints after reindexing
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
# set the offets to something obviously wrong
current_offsets = pillow.checkpoint.get_current_sequence_as_dict()
bad_offsets = {
tp: (offset + 38014)
for tp, offset in current_offsets.items()
}
pillow.checkpoint.update_to(bad_offsets)
self.assertNotEqual(current_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
reindex_and_clean(reindex_id)
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_or_create_wrapped().wrapped_sequence,
)
self.assertEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_current_sequence_as_dict(),
)
def test_no_checkpoint_creation(self, reindex_id, pillow_name):
# these pillows should not touch checkpoints since they are run with other
# reindexers
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
# set these to something obviously wrong
current_offsets = pillow.checkpoint.get_current_sequence_as_dict()
bad_offsets = {tp: (offset + 38014) for tp, offset in current_offsets.items()}
pillow.checkpoint.update_to(bad_offsets)
self.assertNotEqual(current_offsets, pillow.checkpoint.get_current_sequence_as_dict())
self.assertEqual(bad_offsets, pillow.checkpoint.get_current_sequence_as_dict())
reindex_and_clean(reindex_id)
# make sure they are still bad
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(
current_offsets,
pillow.checkpoint.get_current_sequence_as_dict(),
)
self.assertEqual(
bad_offsets,
pillow.checkpoint.get_current_sequence_as_dict(),
)
self.assertNotEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_or_create_wrapped().wrapped_sequence,
)
self.assertNotEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_current_sequence_as_dict(),
)
def test_checkpoint_creation(self, reindex_id, pillow_name):
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
random_seq = uuid.uuid4().hex
pillow.checkpoint.update_to(random_seq)
self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True)
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
self.assertEqual(
pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(),
pillow.checkpoint.get_or_create_wrapped().wrapped_sequence,
)
def test_checkpoint_creation(self, reindex_id, pillow_name):
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
random_seq = uuid.uuid4().hex
pillow.checkpoint.update_to(random_seq)
self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True)
pillow = get_pillow_by_name(pillow_name)
self.assertNotEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
self.assertEqual(
str(pillow.get_change_feed().get_checkpoint_value()),
pillow.checkpoint.get_current_sequence_id(),
)
def pillow_to_checkpoint_id_mapping(reorg_mapping):
checkpoint_mapping = {}
for new_pillow_name, old_pillows in reorg_mapping.items():
new_pillow = get_pillow_by_name(new_pillow_name)
checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = []
checkpoints = []
for pillow_name in old_pillows:
old_pillow = get_pillow_by_name(pillow_name)
checkpoints.append(old_pillow.checkpoint.checkpoint_id)
checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = checkpoints, list(new_pillow.topics)
return checkpoint_mapping
def pillow_to_checkpoint_id_mapping():
checkpoint_mapping = {}
for new_pillow_name, old_pillows in six.iteritems(PILLOW_REORG_MAPPING):
new_pillow = get_pillow_by_name(new_pillow_name)
checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = []
checkpoints = []
for pillow_name in old_pillows:
old_pillow = get_pillow_by_name(pillow_name)
checkpoints.append(old_pillow.checkpoint.checkpoint_id)
checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = checkpoints, list(new_pillow.topics)
return checkpoint_mapping
def test_no_checkpoint_creation(self, reindex_id, pillow_name):
# these pillows should not touch checkpoints since they are run with other
# reindexers
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
random_seq = uuid.uuid4().hex
pillow.checkpoint.update_to(random_seq)
self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True)
pillow = get_pillow_by_name(pillow_name)
self.assertEqual(
random_seq,
pillow.checkpoint.get_current_sequence_id(),
)
def test_no_checkpoint_creation(self, reindex_id, pillow_name):
# these pillows should not touch checkpoints since they are run with other
# reindexers
with real_pillow_settings():
pillow = get_pillow_by_name(pillow_name)
random_seq = uuid.uuid4().hex
pillow.checkpoint.update_to(random_seq)
self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id())
call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True)
pillow = get_pillow_by_name(pillow_name)
self.assertEqual(
random_seq,
pillow.checkpoint.get_current_sequence_id(),
)
def pillow_to_checkpoint_id_mapping():
checkpoint_mapping = {}
for new_pillow_name, old_pillows in six.iteritems(PILLOW_REORG_MAPPING):
new_pillow = get_pillow_by_name(new_pillow_name)
checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = []
checkpoints = []
for pillow_name in old_pillows:
old_pillow = get_pillow_by_name(pillow_name)
checkpoints.append(old_pillow.checkpoint.checkpoint_id)
checkpoint_mapping[
new_pillow.checkpoint.checkpoint_id] = checkpoints, list(
new_pillow.topics)
return checkpoint_mapping
def handle(self, pillow_name, **options):
confirm(
"Are you sure you want to reset the checkpoint for the '{}' pillow?"
.format(pillow_name))
confirm("Have you stopped the pillow?")
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError(
"No pillow found with name: {}".format(pillow_name))
checkpoint = pillow.checkpoint
store = HistoricalPillowCheckpoint.get_historical_max(
checkpoint.checkpoint_id)
if not store:
print(
"No new sequence exists for that pillow. You'll have to do it manually."
)
exit()
old_seq = pillow.get_last_checkpoint_sequence()
new_seq = store.seq
confirm(
"\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".
format(pillow_name, old_seq, new_seq))
checkpoint.update_to(new_seq)
print("Checkpoint updated")
def handle(self, pillow_name, **options):
fluff_configs = {config.name: config for config in get_fluff_pillow_configs()}
if pillow_name not in fluff_configs:
raise CommandError('Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format(
pillow_name, '\n\t'.join(fluff_configs)))
pillow_getter = get_pillow_by_name(pillow_name, instantiate=False)
pillow = pillow_getter(delete_filtered=True)
domains = options.get('domain') or pillow.domains
domains_not_in_pillow = set(domains) - set(pillow.domains)
if domains_not_in_pillow:
bad_domains = ', '.join(domains_not_in_pillow)
available_domains = ', '.join(pillow.domains)
raise CommandError(
"The following domains aren't for this pillow: {}.\nAvailable domains are: {}".format(
bad_domains, available_domains
))
if pillow.kafka_topic in (topics.CASE, topics.FORM):
couch_db = couch_config.get_db(None)
elif pillow.kafka_topic == topics.COMMCARE_USER:
couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB)
else:
raise CommandError('Reindexer not configured for topic: {}'.format(pillow.kafka_topic))
change_provider = CouchDomainDocTypeChangeProvider(
couch_db=couch_db,
domains=domains,
doc_types=[pillow.doc_type],
event_handler=ReindexEventHandler(pillow_name),
)
PillowChangeProviderReindexer(pillow, change_provider).reindex()
def handle(self, *args, **options):
pillow_name = args[0]
confirm(
"Are you sure you want to reset the checkpoint for the '{}' pillow?"
.format(pillow_name))
confirm("Have you stopped the pillow?")
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError(
"No pillow found with name: {}".format(pillow_name))
checkpoint = pillow.get_checkpoint()
try:
seq = PillowCheckpointSeqStore.objects.get(
checkpoint_id=checkpoint.checkpoint_id)
except PillowCheckpointSeqStore.DoesNotExist:
print(
"No new sequence exists for that pillow. You'll have to do it manually."
)
exit()
old_seq = checkpoint.sequence
new_seq = seq.seq
confirm(
"\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".
format(pillow_name, old_seq, new_seq))
pillow.checkpoint.update_to(new_seq)
print("Checkpoint updated")
def process_pillow_retry(error_doc_id):
# Redis error logged in get_redis_client
try:
client = cache_core.get_redis_client()
except cache_core.RedisClientError:
return
# Prevent more than one task from processing this error, just in case
# it got enqueued twice.
lock = client.lock(
"pillow-retry-processing-%s" % error_doc_id,
timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60
)
if lock.acquire(blocking=False):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
release_lock(lock, True)
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
release_lock(lock, True)
return
change = error_doc.change_object
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain
)
change.document_store = document_store
pillow.process_change(change)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.queued = False
error_doc.save()
else:
error_doc.delete()
finally:
release_lock(lock, True)
def process_pillow_retry(error_doc_id):
# Redis error logged in get_redis_client
try:
client = cache_core.get_redis_client()
except cache_core.RedisClientError:
return
# Prevent more than one task from processing this error, just in case
# it got enqueued twice.
lock = client.lock("pillow-retry-processing-%s" % error_doc_id,
timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT *
60)
if lock.acquire(blocking=False):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
release_lock(lock, True)
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") %
pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff(
) + 1
error_doc.save()
finally:
release_lock(lock, True)
return
change = error_doc.change_object
try:
try:
from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow
if isinstance(pillow, ConfigurableReportKafkaPillow):
raise Exception('this is temporarily not supported!')
except ImportError:
pass
pillow.process_change(change)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.queued = False
error_doc.save()
else:
error_doc.delete()
finally:
release_lock(lock, True)
def handle(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
num_processes = options['num_processes']
process_number = options['process_number']
processor_chunk_size = options['processor_chunk_size']
dedicated_migration_process = options['dedicated_migration_process']
assert 0 <= process_number < num_processes
assert processor_chunk_size
if list_all:
print("\nPillows registered in system:")
for config in get_all_pillow_configs():
print('{}: {}'.format(config.section, config.name))
print("\n\tRun with --pillow-name <name> to run a pillow")
print(
"\n\tRun with --pillow-key <key> to run a group of pillows together\n"
)
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, list(settings.PILLOWTOPS)))
sys.exit()
else:
pillows_to_run = [
get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]
]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(
pillow_name,
num_processes=num_processes,
process_num=process_number,
processor_chunk_size=processor_chunk_size,
dedicated_migration_process=dedicated_migration_process)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print(pillow.checkpoint.checkpoint_id)
sys.exit()
else:
print(
"\nNo command set, please see --help for runtime instructions")
sys.exit()
start_pillows(pillows=[
pillow_config.get_instance() for pillow_config in pillows_to_run
])
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(
change_metadata.document_type,
data_source_type
),
change_metadata
)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def _try_legacy_import(pillow_name_or_class):
try:
return get_pillow_instance(pillow_name_or_class)
except ValueError:
# all fluff pillows have module path of 'fluff' so can't be imported directly
if '.' in pillow_name_or_class:
_, pillow_name_or_class = pillow_name_or_class.rsplit('.', 1)
try:
return get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
return None
def get_latest_for_pillow(cls, pillow_name):
try:
pillow = get_pillow_by_name(pillow_name)
except PillowNotFoundError:
# Could not find the pillow
return None
if not pillow:
return None
return cls.get_latest(pillow.checkpoint.checkpoint_id)
def get_latest_for_pillow(cls, pillow_name):
try:
pillow = get_pillow_by_name(pillow_name)
except PillowNotFoundError:
# Could not find the pillow
return None
if not pillow:
return None
return cls.get_latest(pillow.checkpoint.checkpoint_id)
def get_active_pillows(pillows):
# return active pillows based on the heuristic
# that have their checkpoints updated in last 30 days
active_pillows = []
for pillow in pillows:
pillow = get_pillow_by_name(pillow)
last_modified = KafkaCheckpoint.objects.filter(
checkpoint_id=pillow.checkpoint.checkpoint_id).order_by(
'-last_modified')[0].last_modified
if last_modified > datetime.today() - timedelta(days=30):
active_pillows.append(pillow)
return active_pillows
def process_pillow_retry(error_doc):
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") % pillow_name_or_class)
try:
error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1
error_doc.save()
finally:
return
change = error_doc.change_object
delete_all_for_doc = False
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain,
load_source="pillow_retry",
)
change.document_store = document_store
if isinstance(pillow.get_change_feed(), CouchChangeFeed):
pillow.process_change(change)
else:
if change_metadata.data_source_type in ('couch', 'sql'):
data_source_type = change_metadata.data_source_type
else:
# legacy metadata will have other values for non-sql
# can remove this once all legacy errors have been processed
data_source_type = 'sql'
producer.send_change(
get_topic_for_doc_type(change_metadata.document_type,
data_source_type), change_metadata)
delete_all_for_doc = True
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
if delete_all_for_doc:
PillowError.objects.filter(doc_id=error_doc.doc_id).delete()
else:
error_doc.delete()
def get_by_pillow_name(cls, pillow_name):
try:
pillow = get_pillow_by_name(pillow_name)
except PillowNotFoundError:
# Could not find the pillow
return None
if not pillow:
return None
try:
store = cls.objects.get(checkpoint_id=pillow.checkpoint.checkpoint_id)
except cls.DoesNotExist:
return None
return store
def get_by_pillow_name(cls, pillow_name):
try:
pillow = get_pillow_by_name(pillow_name)
except PillowNotFoundError:
# Could not find the pillow
return None
if not pillow:
return None
try:
store = cls.objects.get(checkpoint_id=pillow.get_checkpoint()['_id'])
except cls.DoesNotExist:
return None
return store
def handle(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
if list_all:
print("\nPillows registered in system:")
for config in get_all_pillow_configs():
print(u'{}: {}'.format(config.section, config.name))
print("\n\tRun with --pillow-name <name> to run a pillow")
print(
"\n\tRun with --pillow-key <key> to run a group of pillows together\n"
)
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, settings.PILLOWTOPS.keys()))
sys.exit()
else:
pillows_to_run = [
get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]
]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(pillow_name)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print(pillow.checkpoint.checkpoint_id)
sys.exit()
else:
print(
"\nNo command set, please see --help for runtime instructions")
sys.exit()
start_pillows(pillows=[
pillow_config.get_instance() for pillow_config in pillows_to_run
])
def handle(self, *args, **options):
pillow = get_pillow_by_name(args[0])
if not options['noinput']:
confirm = raw_input("""
You have requested to wipe %s table
Type 'yes' to continue, or 'no' to cancel:
""" % pillow.pillow_id)
if confirm != 'yes':
print "\tWipe cancelled."
return
processor = FluffPillowProcessor(pillow.indicator_class)
engine = processor.get_sql_engine()
table = pillow.indicator_class().table
engine.execute(table.delete())
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is ptop_reindexer_fluff %s' % self.args)
fluff_configs = {
config.name: config
for config in get_fluff_pillow_configs()
}
pillow_name = args[0]
if pillow_name not in fluff_configs:
raise CommandError(
'Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format(
pillow_name, '\n\t'.join(fluff_configs)))
pillow_getter = get_pillow_by_name(pillow_name, instantiate=False)
pillow = pillow_getter(delete_filtered=True)
if len(args) == 1:
domains = pillow.domains
else:
domains = args[1:]
domains_not_in_pillow = set(domains) - set(pillow.domains)
if domains_not_in_pillow:
bad_domains = ', '.join(domains_not_in_pillow)
available_domains = ', '.join(pillow.domains)
raise CommandError(
"The following domains aren't for this pillow: {}.\nAvailable domains are: {}"
.format(bad_domains, available_domains))
if pillow.kafka_topic in (topics.CASE, topics.FORM):
couch_db = couch_config.get_db(None)
elif pillow.kafka_topic == topics.COMMCARE_USER:
couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB)
else:
raise CommandError('Reindexer not configured for topic: {}'.format(
pillow.kafka_topic))
change_provider = CouchDomainDocTypeChangeProvider(
couch_db=couch_db,
domains=domains,
doc_types=[pillow.doc_type],
event_handler=ReindexEventHandler(pillow_name),
)
PillowChangeProviderReindexer(pillow, change_provider).reindex()
def handle(self, *args, **options):
pillow_class = get_pillow_by_name(args[0], instantiate=False)
if not options['noinput']:
confirm = raw_input(
"""
You have requested to wipe %s table
Type 'yes' to continue, or 'no' to cancel:
""" % pillow_class.__name__
)
if confirm != 'yes':
print "\tWipe cancelled."
return
engine = pillow_class.get_sql_engine()
table = pillow_class.indicator_class().table
engine.execute(table.delete())
def handle(self, pillow_name, **options):
pillow = get_pillow_by_name(pillow_name)
if not options['noinput']:
confirm = input("""
You have requested to wipe %s table
Type 'yes' to continue, or 'no' to cancel:
""" % pillow.pillow_id)
if confirm != 'yes':
print("\tWipe cancelled.")
return
for processor in pillow.processors:
engine = processor.get_sql_engine()
table = processor.indicator_class().table
engine.execute(table.delete())
def handle(self, *args, **options):
pillow = get_pillow_by_name(args[0])
if not options['noinput']:
confirm = raw_input(
"""
You have requested to wipe %s table
Type 'yes' to continue, or 'no' to cancel:
""" % pillow.pillow_id
)
if confirm != 'yes':
print "\tWipe cancelled."
return
processor = FluffPillowProcessor(pillow.indicator_class)
engine = processor.get_sql_engine()
table = pillow.indicator_class().table
engine.execute(table.delete())
def handle(self, pillow_name, **options):
pillow = get_pillow_by_name(pillow_name)
if not options['noinput']:
confirm = input(
"""
You have requested to wipe %s table
Type 'yes' to continue, or 'no' to cancel:
""" % pillow.pillow_id
)
if confirm != 'yes':
print("\tWipe cancelled.")
return
for processor in pillow.processors:
engine = processor.get_sql_engine()
table = processor.indicator_class().table
engine.execute(table.delete())
def handle_noargs(self, **options):
run_all = options['run_all']
list_all = options['list_all']
list_checkpoints = options['list_checkpoints']
pillow_name = options['pillow_name']
pillow_key = options['pillow_key']
if list_all:
print "\nPillows registered in system:"
for config in get_all_pillow_configs():
print u'{}: {}'.format(config.section, config.name)
print "\n\tRun with --pillow-name <name> to run a pillow"
print "\n\tRun with --pillow-key <key> to run a group of pillows together\n"
sys.exit()
if run_all:
pillows_to_run = get_all_pillow_configs()
elif not run_all and not pillow_name and pillow_key:
# get pillows from key
if pillow_key not in settings.PILLOWTOPS:
print "\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \
(pillow_key, settings.PILLOWTOPS.keys())
sys.exit()
else:
pillows_to_run = [get_pillow_config_from_setting(pillow_key, config)
for config in settings.PILLOWTOPS[pillow_key]]
elif not run_all and not pillow_key and pillow_name:
pillow = get_pillow_by_name(pillow_name)
start_pillow(pillow)
sys.exit()
elif list_checkpoints:
for pillow in get_all_pillow_instances():
print pillow.checkpoint.checkpoint_id
sys.exit()
else:
print "\nNo command set, please see --help for runtime instructions"
sys.exit()
start_pillows(pillows=[pillow_config.get_instance() for pillow_config in pillows_to_run])
def handle(self, pillow_name, **options):
confirm("Are you sure you want to reset the checkpoint for the '{}' pillow?".format(pillow_name))
confirm("Have you stopped the pillow?")
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError("No pillow found with name: {}".format(pillow_name))
checkpoint = pillow.checkpoint
store = HistoricalPillowCheckpoint.get_historical_max(checkpoint.checkpoint_id)
if not store:
print("No new sequence exists for that pillow. You'll have to do it manually.")
exit()
old_seq = pillow.get_last_checkpoint_sequence()
new_seq = store.seq
confirm("\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format(
pillow_name, old_seq, new_seq
))
checkpoint.update_to(new_seq)
print("Checkpoint updated")
def process_pillow_retry(error_doc_id):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into.") % pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
return
change = error_doc.change_object
try:
change_metadata = change.metadata
if change_metadata:
document_store = get_document_store(
data_source_type=change_metadata.data_source_type,
data_source_name=change_metadata.data_source_name,
domain=change_metadata.domain)
change.document_store = document_store
pillow.process_change(change)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.save()
else:
error_doc.delete()
def handle(self, *args, **options):
pillow_name = args[0]
confirm("Are you sure you want to reset the checkpoint for the '{}' pillow?".format(pillow_name))
confirm("Have you stopped the pillow?")
pillow = get_pillow_by_name(pillow_name)
if not pillow:
raise CommandError("No pillow found with name: {}".format(pillow_name))
checkpoint = pillow.get_checkpoint()
try:
seq = PillowCheckpointSeqStore.objects.get(checkpoint_id=checkpoint.checkpoint_id)
except PillowCheckpointSeqStore.DoesNotExist:
print "No new sequence exists for that pillow. You'll have to do it manually."
exit()
old_seq = checkpoint.sequence
new_seq = seq.seq
confirm("\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format(pillow_name, old_seq, new_seq))
pillow.checkpoint.update_to(new_seq)
print "Checkpoint updated"
def process_pillow_retry(error_doc_id):
# Redis error logged in get_redis_client
try:
client = cache_core.get_redis_client()
except cache_core.RedisClientError:
return
# Prevent more than one task from processing this error, just in case
# it got enqueued twice.
lock = client.lock(
"pillow-retry-processing-%s" % error_doc_id,
timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60
)
if lock.acquire(blocking=False):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
return
pillow_class = error_doc.pillow
try:
pillow = get_pillow_instance(pillow_class)
except ValueError:
# all fluff pillows have module path of 'fluff' so can't be imported directly
_, pillow_class_name = pillow_class.rsplit('.', 1)
try:
pillow = get_pillow_by_name(pillow_class_name)
except PillowNotFoundError:
pillow = None
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
release_lock(lock, True)
return
change = error_doc.change_object
if pillow.include_docs:
try:
change.set_document(pillow.get_couch_db().open_doc(change.id))
except ResourceNotFound:
change.deleted = True
try:
try:
from corehq.apps.userreports.pillow import ConfigurableIndicatorPillow
if isinstance(pillow, ConfigurableIndicatorPillow):
raise Exception('this is temporarily not supported!')
except ImportError:
pass
pillow.process_change(change, is_retry_attempt=True)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.queued = False
error_doc.save()
else:
error_doc.delete()
finally:
release_lock(lock, True)
def pillows(self):
return [get_pillow_by_name(name) for name in self.pillow_names]
def _get_pillow(pillow_name_or_class):
return get_pillow_by_name(pillow_name_or_class)
def pillows(self):
return [get_pillow_by_name(name) for name in self.pillow_names]
def process_pillow_retry(error_doc_id):
# Redis error logged in get_redis_client
try:
client = cache_core.get_redis_client()
except cache_core.RedisClientError:
return
# Prevent more than one task from processing this error, just in case
# it got enqueued twice.
lock = client.lock(
"pillow-retry-processing-%s" % error_doc_id,
timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60
)
if lock.acquire(blocking=False):
try:
error_doc = PillowError.objects.get(id=error_doc_id)
except PillowError.DoesNotExist:
release_lock(lock, True)
return
pillow_name_or_class = error_doc.pillow
try:
pillow = get_pillow_by_name(pillow_name_or_class)
except PillowNotFoundError:
if not settings.UNIT_TESTING:
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_assert(False, 'Pillow retry {} is still using legacy class {}'.format(
error_doc.pk, pillow_name_or_class
))
pillow = _try_legacy_import(pillow_name_or_class)
if not pillow:
notify_error((
"Could not find pillowtop class '%s' while attempting a retry. "
"If this pillow was recently deleted then this will be automatically cleaned up eventually. "
"If not, then this should be looked into."
) % pillow_name_or_class)
try:
error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1
error_doc.save()
finally:
release_lock(lock, True)
return
change = error_doc.change_object
if getattr(pillow, 'include_docs', False):
try:
change.set_document(pillow.get_couch_db().open_doc(change.id))
except ResourceNotFound:
change.deleted = True
try:
try:
from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow
if isinstance(pillow, ConfigurableReportKafkaPillow):
raise Exception('this is temporarily not supported!')
except ImportError:
pass
pillow.process_change(change, is_retry_attempt=True)
except Exception:
ex_type, ex_value, ex_tb = sys.exc_info()
error_doc.add_attempt(ex_value, ex_tb)
error_doc.queued = False
error_doc.save()
else:
error_doc.delete()
finally:
release_lock(lock, True)
