def test_checkpoint_creation(self, reindex_id, pillow_name): # checks that checkpoipnts are set to the latest checkpoints after reindexing with real_pillow_settings(): pillow = get_pillow_by_name(pillow_name) # set the offets to something obviously wrong current_offsets = pillow.checkpoint.get_current_sequence_as_dict() bad_offsets = { tp: (offset + 38014) for tp, offset in current_offsets.items() } pillow.checkpoint.update_to(bad_offsets) self.assertNotEqual(current_offsets, pillow.checkpoint.get_current_sequence_as_dict()) self.assertEqual(bad_offsets, pillow.checkpoint.get_current_sequence_as_dict()) reindex_and_clean(reindex_id) pillow = get_pillow_by_name(pillow_name) self.assertNotEqual(bad_offsets, pillow.checkpoint.get_current_sequence_as_dict()) self.assertEqual( pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(), pillow.checkpoint.get_or_create_wrapped().wrapped_sequence, ) self.assertEqual( pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(), pillow.checkpoint.get_current_sequence_as_dict(), )
def test_no_checkpoint_creation(self, reindex_id, pillow_name): # these pillows should not touch checkpoints since they are run with other # reindexers with real_pillow_settings(): pillow = get_pillow_by_name(pillow_name) # set these to something obviously wrong current_offsets = pillow.checkpoint.get_current_sequence_as_dict() bad_offsets = {tp: (offset + 38014) for tp, offset in current_offsets.items()} pillow.checkpoint.update_to(bad_offsets) self.assertNotEqual(current_offsets, pillow.checkpoint.get_current_sequence_as_dict()) self.assertEqual(bad_offsets, pillow.checkpoint.get_current_sequence_as_dict()) reindex_and_clean(reindex_id) # make sure they are still bad pillow = get_pillow_by_name(pillow_name) self.assertNotEqual( current_offsets, pillow.checkpoint.get_current_sequence_as_dict(), ) self.assertEqual( bad_offsets, pillow.checkpoint.get_current_sequence_as_dict(), ) self.assertNotEqual( pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(), pillow.checkpoint.get_or_create_wrapped().wrapped_sequence, ) self.assertNotEqual( pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(), pillow.checkpoint.get_current_sequence_as_dict(), )
def test_checkpoint_creation(self, reindex_id, pillow_name): with real_pillow_settings(): pillow = get_pillow_by_name(pillow_name) random_seq = uuid.uuid4().hex pillow.checkpoint.update_to(random_seq) self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id()) call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True) pillow = get_pillow_by_name(pillow_name) self.assertNotEqual(random_seq, pillow.checkpoint.get_current_sequence_id()) self.assertEqual( pillow.get_change_feed().get_latest_offsets_as_checkpoint_value(), pillow.checkpoint.get_or_create_wrapped().wrapped_sequence, )
def test_checkpoint_creation(self, reindex_id, pillow_name): with real_pillow_settings(): pillow = get_pillow_by_name(pillow_name) random_seq = uuid.uuid4().hex pillow.checkpoint.update_to(random_seq) self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id()) call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True) pillow = get_pillow_by_name(pillow_name) self.assertNotEqual(random_seq, pillow.checkpoint.get_current_sequence_id()) self.assertEqual( str(pillow.get_change_feed().get_checkpoint_value()), pillow.checkpoint.get_current_sequence_id(), )
def pillow_to_checkpoint_id_mapping(reorg_mapping): checkpoint_mapping = {} for new_pillow_name, old_pillows in reorg_mapping.items(): new_pillow = get_pillow_by_name(new_pillow_name) checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = [] checkpoints = [] for pillow_name in old_pillows: old_pillow = get_pillow_by_name(pillow_name) checkpoints.append(old_pillow.checkpoint.checkpoint_id) checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = checkpoints, list(new_pillow.topics) return checkpoint_mapping
def pillow_to_checkpoint_id_mapping(): checkpoint_mapping = {} for new_pillow_name, old_pillows in six.iteritems(PILLOW_REORG_MAPPING): new_pillow = get_pillow_by_name(new_pillow_name) checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = [] checkpoints = [] for pillow_name in old_pillows: old_pillow = get_pillow_by_name(pillow_name) checkpoints.append(old_pillow.checkpoint.checkpoint_id) checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = checkpoints, list(new_pillow.topics) return checkpoint_mapping
def test_no_checkpoint_creation(self, reindex_id, pillow_name): # these pillows should not touch checkpoints since they are run with other # reindexers with real_pillow_settings(): pillow = get_pillow_by_name(pillow_name) random_seq = uuid.uuid4().hex pillow.checkpoint.update_to(random_seq) self.assertEqual(random_seq, pillow.checkpoint.get_current_sequence_id()) call_command('ptop_reindexer_v2', reindex_id, cleanup=True, noinput=True) pillow = get_pillow_by_name(pillow_name) self.assertEqual( random_seq, pillow.checkpoint.get_current_sequence_id(), )
def pillow_to_checkpoint_id_mapping(): checkpoint_mapping = {} for new_pillow_name, old_pillows in six.iteritems(PILLOW_REORG_MAPPING): new_pillow = get_pillow_by_name(new_pillow_name) checkpoint_mapping[new_pillow.checkpoint.checkpoint_id] = [] checkpoints = [] for pillow_name in old_pillows: old_pillow = get_pillow_by_name(pillow_name) checkpoints.append(old_pillow.checkpoint.checkpoint_id) checkpoint_mapping[ new_pillow.checkpoint.checkpoint_id] = checkpoints, list( new_pillow.topics) return checkpoint_mapping
def handle(self, pillow_name, **options): confirm( "Are you sure you want to reset the checkpoint for the '{}' pillow?" .format(pillow_name)) confirm("Have you stopped the pillow?") pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError( "No pillow found with name: {}".format(pillow_name)) checkpoint = pillow.checkpoint store = HistoricalPillowCheckpoint.get_historical_max( checkpoint.checkpoint_id) if not store: print( "No new sequence exists for that pillow. You'll have to do it manually." ) exit() old_seq = pillow.get_last_checkpoint_sequence() new_seq = store.seq confirm( "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n". format(pillow_name, old_seq, new_seq)) checkpoint.update_to(new_seq) print("Checkpoint updated")
def handle(self, pillow_name, **options): fluff_configs = {config.name: config for config in get_fluff_pillow_configs()} if pillow_name not in fluff_configs: raise CommandError('Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format( pillow_name, '\n\t'.join(fluff_configs))) pillow_getter = get_pillow_by_name(pillow_name, instantiate=False) pillow = pillow_getter(delete_filtered=True) domains = options.get('domain') or pillow.domains domains_not_in_pillow = set(domains) - set(pillow.domains) if domains_not_in_pillow: bad_domains = ', '.join(domains_not_in_pillow) available_domains = ', '.join(pillow.domains) raise CommandError( "The following domains aren't for this pillow: {}.\nAvailable domains are: {}".format( bad_domains, available_domains )) if pillow.kafka_topic in (topics.CASE, topics.FORM): couch_db = couch_config.get_db(None) elif pillow.kafka_topic == topics.COMMCARE_USER: couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB) else: raise CommandError('Reindexer not configured for topic: {}'.format(pillow.kafka_topic)) change_provider = CouchDomainDocTypeChangeProvider( couch_db=couch_db, domains=domains, doc_types=[pillow.doc_type], event_handler=ReindexEventHandler(pillow_name), ) PillowChangeProviderReindexer(pillow, change_provider).reindex()
def handle(self, *args, **options): pillow_name = args[0] confirm( "Are you sure you want to reset the checkpoint for the '{}' pillow?" .format(pillow_name)) confirm("Have you stopped the pillow?") pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError( "No pillow found with name: {}".format(pillow_name)) checkpoint = pillow.get_checkpoint() try: seq = PillowCheckpointSeqStore.objects.get( checkpoint_id=checkpoint.checkpoint_id) except PillowCheckpointSeqStore.DoesNotExist: print( "No new sequence exists for that pillow. You'll have to do it manually." ) exit() old_seq = checkpoint.sequence new_seq = seq.seq confirm( "\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n". format(pillow_name, old_seq, new_seq)) pillow.checkpoint.update_to(new_seq) print("Checkpoint updated")
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain ) change.document_store = document_store pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock("pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT * 60) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff( ) + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object try: try: from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow if isinstance(pillow, ConfigurableReportKafkaPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def handle(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] num_processes = options['num_processes'] process_number = options['process_number'] processor_chunk_size = options['processor_chunk_size'] dedicated_migration_process = options['dedicated_migration_process'] assert 0 <= process_number < num_processes assert processor_chunk_size if list_all: print("\nPillows registered in system:") for config in get_all_pillow_configs(): print('{}: {}'.format(config.section, config.name)) print("\n\tRun with --pillow-name <name> to run a pillow") print( "\n\tRun with --pillow-key <key> to run a group of pillows together\n" ) sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, list(settings.PILLOWTOPS))) sys.exit() else: pillows_to_run = [ get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key] ] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name( pillow_name, num_processes=num_processes, process_num=process_number, processor_chunk_size=processor_chunk_size, dedicated_migration_process=dedicated_migration_process) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print(pillow.checkpoint.checkpoint_id) sys.exit() else: print( "\nNo command set, please see --help for runtime instructions") sys.exit() start_pillows(pillows=[ pillow_config.get_instance() for pillow_config in pillows_to_run ])
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type( change_metadata.document_type, data_source_type ), change_metadata ) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def _try_legacy_import(pillow_name_or_class): try: return get_pillow_instance(pillow_name_or_class) except ValueError: # all fluff pillows have module path of 'fluff' so can't be imported directly if '.' in pillow_name_or_class: _, pillow_name_or_class = pillow_name_or_class.rsplit('.', 1) try: return get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: return None
def get_latest_for_pillow(cls, pillow_name): try: pillow = get_pillow_by_name(pillow_name) except PillowNotFoundError: # Could not find the pillow return None if not pillow: return None return cls.get_latest(pillow.checkpoint.checkpoint_id)
def get_active_pillows(pillows): # return active pillows based on the heuristic # that have their checkpoints updated in last 30 days active_pillows = [] for pillow in pillows: pillow = get_pillow_by_name(pillow) last_modified = KafkaCheckpoint.objects.filter( checkpoint_id=pillow.checkpoint.checkpoint_id).order_by( '-last_modified')[0].last_modified if last_modified > datetime.today() - timedelta(days=30): active_pillows.append(pillow) return active_pillows
def process_pillow_retry(error_doc): pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = const.PILLOW_RETRY_MULTI_ATTEMPTS_CUTOFF + 1 error_doc.save() finally: return change = error_doc.change_object delete_all_for_doc = False try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain, load_source="pillow_retry", ) change.document_store = document_store if isinstance(pillow.get_change_feed(), CouchChangeFeed): pillow.process_change(change) else: if change_metadata.data_source_type in ('couch', 'sql'): data_source_type = change_metadata.data_source_type else: # legacy metadata will have other values for non-sql # can remove this once all legacy errors have been processed data_source_type = 'sql' producer.send_change( get_topic_for_doc_type(change_metadata.document_type, data_source_type), change_metadata) delete_all_for_doc = True except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: if delete_all_for_doc: PillowError.objects.filter(doc_id=error_doc.doc_id).delete() else: error_doc.delete()
def get_by_pillow_name(cls, pillow_name): try: pillow = get_pillow_by_name(pillow_name) except PillowNotFoundError: # Could not find the pillow return None if not pillow: return None try: store = cls.objects.get(checkpoint_id=pillow.checkpoint.checkpoint_id) except cls.DoesNotExist: return None return store
def get_by_pillow_name(cls, pillow_name): try: pillow = get_pillow_by_name(pillow_name) except PillowNotFoundError: # Could not find the pillow return None if not pillow: return None try: store = cls.objects.get(checkpoint_id=pillow.get_checkpoint()['_id']) except cls.DoesNotExist: return None return store
def handle(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] if list_all: print("\nPillows registered in system:") for config in get_all_pillow_configs(): print(u'{}: {}'.format(config.section, config.name)) print("\n\tRun with --pillow-name <name> to run a pillow") print( "\n\tRun with --pillow-key <key> to run a group of pillows together\n" ) sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print("\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, settings.PILLOWTOPS.keys())) sys.exit() else: pillows_to_run = [ get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key] ] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name(pillow_name) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print(pillow.checkpoint.checkpoint_id) sys.exit() else: print( "\nNo command set, please see --help for runtime instructions") sys.exit() start_pillows(pillows=[ pillow_config.get_instance() for pillow_config in pillows_to_run ])
def handle(self, *args, **options): pillow = get_pillow_by_name(args[0]) if not options['noinput']: confirm = raw_input(""" You have requested to wipe %s table Type 'yes' to continue, or 'no' to cancel: """ % pillow.pillow_id) if confirm != 'yes': print "\tWipe cancelled." return processor = FluffPillowProcessor(pillow.indicator_class) engine = processor.get_sql_engine() table = pillow.indicator_class().table engine.execute(table.delete())
def handle(self, *args, **options): if len(args) < 1: raise CommandError('Usage is ptop_reindexer_fluff %s' % self.args) fluff_configs = { config.name: config for config in get_fluff_pillow_configs() } pillow_name = args[0] if pillow_name not in fluff_configs: raise CommandError( 'Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format( pillow_name, '\n\t'.join(fluff_configs))) pillow_getter = get_pillow_by_name(pillow_name, instantiate=False) pillow = pillow_getter(delete_filtered=True) if len(args) == 1: domains = pillow.domains else: domains = args[1:] domains_not_in_pillow = set(domains) - set(pillow.domains) if domains_not_in_pillow: bad_domains = ', '.join(domains_not_in_pillow) available_domains = ', '.join(pillow.domains) raise CommandError( "The following domains aren't for this pillow: {}.\nAvailable domains are: {}" .format(bad_domains, available_domains)) if pillow.kafka_topic in (topics.CASE, topics.FORM): couch_db = couch_config.get_db(None) elif pillow.kafka_topic == topics.COMMCARE_USER: couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB) else: raise CommandError('Reindexer not configured for topic: {}'.format( pillow.kafka_topic)) change_provider = CouchDomainDocTypeChangeProvider( couch_db=couch_db, domains=domains, doc_types=[pillow.doc_type], event_handler=ReindexEventHandler(pillow_name), ) PillowChangeProviderReindexer(pillow, change_provider).reindex()
def handle(self, *args, **options): pillow_class = get_pillow_by_name(args[0], instantiate=False) if not options['noinput']: confirm = raw_input( """ You have requested to wipe %s table Type 'yes' to continue, or 'no' to cancel: """ % pillow_class.__name__ ) if confirm != 'yes': print "\tWipe cancelled." return engine = pillow_class.get_sql_engine() table = pillow_class.indicator_class().table engine.execute(table.delete())
def handle(self, pillow_name, **options): pillow = get_pillow_by_name(pillow_name) if not options['noinput']: confirm = input(""" You have requested to wipe %s table Type 'yes' to continue, or 'no' to cancel: """ % pillow.pillow_id) if confirm != 'yes': print("\tWipe cancelled.") return for processor in pillow.processors: engine = processor.get_sql_engine() table = processor.indicator_class().table engine.execute(table.delete())
def handle(self, *args, **options): pillow = get_pillow_by_name(args[0]) if not options['noinput']: confirm = raw_input( """ You have requested to wipe %s table Type 'yes' to continue, or 'no' to cancel: """ % pillow.pillow_id ) if confirm != 'yes': print "\tWipe cancelled." return processor = FluffPillowProcessor(pillow.indicator_class) engine = processor.get_sql_engine() table = pillow.indicator_class().table engine.execute(table.delete())
def handle(self, pillow_name, **options): pillow = get_pillow_by_name(pillow_name) if not options['noinput']: confirm = input( """ You have requested to wipe %s table Type 'yes' to continue, or 'no' to cancel: """ % pillow.pillow_id ) if confirm != 'yes': print("\tWipe cancelled.") return for processor in pillow.processors: engine = processor.get_sql_engine() table = processor.indicator_class().table engine.execute(table.delete())
def handle_noargs(self, **options): run_all = options['run_all'] list_all = options['list_all'] list_checkpoints = options['list_checkpoints'] pillow_name = options['pillow_name'] pillow_key = options['pillow_key'] if list_all: print "\nPillows registered in system:" for config in get_all_pillow_configs(): print u'{}: {}'.format(config.section, config.name) print "\n\tRun with --pillow-name <name> to run a pillow" print "\n\tRun with --pillow-key <key> to run a group of pillows together\n" sys.exit() if run_all: pillows_to_run = get_all_pillow_configs() elif not run_all and not pillow_name and pillow_key: # get pillows from key if pillow_key not in settings.PILLOWTOPS: print "\n\tError, key %s is not in settings.PILLOWTOPS, legal keys are: %s" % \ (pillow_key, settings.PILLOWTOPS.keys()) sys.exit() else: pillows_to_run = [get_pillow_config_from_setting(pillow_key, config) for config in settings.PILLOWTOPS[pillow_key]] elif not run_all and not pillow_key and pillow_name: pillow = get_pillow_by_name(pillow_name) start_pillow(pillow) sys.exit() elif list_checkpoints: for pillow in get_all_pillow_instances(): print pillow.checkpoint.checkpoint_id sys.exit() else: print "\nNo command set, please see --help for runtime instructions" sys.exit() start_pillows(pillows=[pillow_config.get_instance() for pillow_config in pillows_to_run])
def handle(self, pillow_name, **options): confirm("Are you sure you want to reset the checkpoint for the '{}' pillow?".format(pillow_name)) confirm("Have you stopped the pillow?") pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError("No pillow found with name: {}".format(pillow_name)) checkpoint = pillow.checkpoint store = HistoricalPillowCheckpoint.get_historical_max(checkpoint.checkpoint_id) if not store: print("No new sequence exists for that pillow. You'll have to do it manually.") exit() old_seq = pillow.get_last_checkpoint_sequence() new_seq = store.seq confirm("\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format( pillow_name, old_seq, new_seq )) checkpoint.update_to(new_seq) print("Checkpoint updated")
def process_pillow_retry(error_doc_id): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into.") % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: return change = error_doc.change_object try: change_metadata = change.metadata if change_metadata: document_store = get_document_store( data_source_type=change_metadata.data_source_type, data_source_name=change_metadata.data_source_name, domain=change_metadata.domain) change.document_store = document_store pillow.process_change(change) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.save() else: error_doc.delete()
def handle(self, *args, **options): pillow_name = args[0] confirm("Are you sure you want to reset the checkpoint for the '{}' pillow?".format(pillow_name)) confirm("Have you stopped the pillow?") pillow = get_pillow_by_name(pillow_name) if not pillow: raise CommandError("No pillow found with name: {}".format(pillow_name)) checkpoint = pillow.get_checkpoint() try: seq = PillowCheckpointSeqStore.objects.get(checkpoint_id=checkpoint.checkpoint_id) except PillowCheckpointSeqStore.DoesNotExist: print "No new sequence exists for that pillow. You'll have to do it manually." exit() old_seq = checkpoint.sequence new_seq = seq.seq confirm("\nReset checkpoint for '{}' pillow from:\n\n{}\n\nto\n\n{}\n\n".format(pillow_name, old_seq, new_seq)) pillow.checkpoint.update_to(new_seq) print "Checkpoint updated"
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: return pillow_class = error_doc.pillow try: pillow = get_pillow_instance(pillow_class) except ValueError: # all fluff pillows have module path of 'fluff' so can't be imported directly _, pillow_class_name = pillow_class.rsplit('.', 1) try: pillow = get_pillow_by_name(pillow_class_name) except PillowNotFoundError: pillow = None if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object if pillow.include_docs: try: change.set_document(pillow.get_couch_db().open_doc(change.id)) except ResourceNotFound: change.deleted = True try: try: from corehq.apps.userreports.pillow import ConfigurableIndicatorPillow if isinstance(pillow, ConfigurableIndicatorPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change, is_retry_attempt=True) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)
def pillows(self): return [get_pillow_by_name(name) for name in self.pillow_names]
def _get_pillow(pillow_name_or_class): return get_pillow_by_name(pillow_name_or_class)
def process_pillow_retry(error_doc_id): # Redis error logged in get_redis_client try: client = cache_core.get_redis_client() except cache_core.RedisClientError: return # Prevent more than one task from processing this error, just in case # it got enqueued twice. lock = client.lock( "pillow-retry-processing-%s" % error_doc_id, timeout=settings.PILLOW_RETRY_PROCESSING_LOCK_TIMEOUT*60 ) if lock.acquire(blocking=False): try: error_doc = PillowError.objects.get(id=error_doc_id) except PillowError.DoesNotExist: release_lock(lock, True) return pillow_name_or_class = error_doc.pillow try: pillow = get_pillow_by_name(pillow_name_or_class) except PillowNotFoundError: if not settings.UNIT_TESTING: _assert = soft_assert(to='@'.join(['czue', 'dimagi.com'])) _assert(False, 'Pillow retry {} is still using legacy class {}'.format( error_doc.pk, pillow_name_or_class )) pillow = _try_legacy_import(pillow_name_or_class) if not pillow: notify_error(( "Could not find pillowtop class '%s' while attempting a retry. " "If this pillow was recently deleted then this will be automatically cleaned up eventually. " "If not, then this should be looked into." ) % pillow_name_or_class) try: error_doc.total_attempts = PillowError.multi_attempts_cutoff() + 1 error_doc.save() finally: release_lock(lock, True) return change = error_doc.change_object if getattr(pillow, 'include_docs', False): try: change.set_document(pillow.get_couch_db().open_doc(change.id)) except ResourceNotFound: change.deleted = True try: try: from corehq.apps.userreports.pillow import ConfigurableReportKafkaPillow if isinstance(pillow, ConfigurableReportKafkaPillow): raise Exception('this is temporarily not supported!') except ImportError: pass pillow.process_change(change, is_retry_attempt=True) except Exception: ex_type, ex_value, ex_tb = sys.exc_info() error_doc.add_attempt(ex_value, ex_tb) error_doc.queued = False error_doc.save() else: error_doc.delete() finally: release_lock(lock, True)