def pillow_seq_store(): for pillow in get_all_pillow_instances(): checkpoint = pillow.get_checkpoint() store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint['_id']) if not created and force_seq_int(checkpoint['seq']) < force_seq_int(store.seq) - EPSILON: notify_exception( None, message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format(store.checkpoint_id), details={ 'pillow checkpoint seq': checkpoint['seq'], 'stored seq': store.seq }) else: store.seq = checkpoint['seq'] store.save()
def pillow_seq_store(): for pillow in get_couch_pillow_instances(): checkpoint = pillow.checkpoint store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint.checkpoint_id) db_seq = checkpoint.get_current_sequence_id() store_seq = force_seq_int(store.seq) or 0 if not created and force_seq_int(db_seq) < store_seq - EPSILON: notify_exception( None, message="Found seq number lower than previous for {}. " "This could mean we are in a rewind state".format(store.checkpoint_id), details={"pillow checkpoint seq": db_seq, "stored seq": store.seq}, ) else: store.seq = db_seq store.save()
def check_for_rewind(checkpoint): historical_checkpoint = HistoricalPillowCheckpoint.get_historical_max(checkpoint.checkpoint_id) if not historical_checkpoint: return False db_seq = checkpoint.get_current_sequence_id() store_seq = historical_checkpoint.seq_int has_rewound = force_seq_int(db_seq) < store_seq - EPSILON return has_rewound, historical_checkpoint.seq
def pillow_seq_store(): for pillow in get_couch_pillow_instances(): checkpoint = pillow.checkpoint store, created = PillowCheckpointSeqStore.objects.get_or_create( checkpoint_id=checkpoint.checkpoint_id) db_seq = checkpoint.get_current_sequence_id() store_seq = force_seq_int(store.seq) or 0 if not created and force_seq_int(db_seq) < store_seq - EPSILON: notify_exception( None, message='Found seq number lower than previous for {}. ' 'This could mean we are in a rewind state'.format( store.checkpoint_id), details={ 'pillow checkpoint seq': db_seq, 'stored seq': store.seq }) else: store.seq = db_seq store.save()
def _normalize_sequence(self, sequence): from pillowtop.feed.couch import CouchChangeFeed change_feed = self.get_change_feed() if not isinstance(sequence, dict): if isinstance(change_feed, CouchChangeFeed): topic = change_feed.couch_db else: return {} sequence = {topic: force_seq_int(sequence)} return sequence
def _normalize_sequence(self, sequence): from pillowtop.feed.couch import CouchChangeFeed from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed change_feed = self.get_change_feed() if not isinstance(sequence, dict): if isinstance(change_feed, KafkaChangeFeed): topics = change_feed.topics assert len(topics) == 1 topic = topics[0] elif isinstance(change_feed, CouchChangeFeed): topic = change_feed.couch_db else: return {} sequence = {topic: force_seq_int(sequence)} return sequence
def get_latest_offsets(self): return { self._couch_db.dbname: force_seq_int(get_current_seq(self._couch_db)) }
def get_processed_offsets(self): return {self._couch_db.dbname: force_seq_int(self._last_processed_seq)}
def get_current_offsets(self): return {self._couch_db.dbname: force_seq_int(self.get_latest_change_id())}
def get_latest_offsets(self): return {self._couch_db.dbname: force_seq_int(get_current_seq(self._couch_db))}