def pillow_seq_store():
for pillow in get_all_pillow_instances():
checkpoint = pillow.get_checkpoint()
store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint['_id'])
if not created and force_seq_int(checkpoint['seq']) < force_seq_int(store.seq) - EPSILON:
notify_exception(
None,
message='Found seq number lower than previous for {}. '
'This could mean we are in a rewind state'.format(store.checkpoint_id),
details={
'pillow checkpoint seq': checkpoint['seq'],
'stored seq': store.seq
})
else:
store.seq = checkpoint['seq']
store.save()
def pillow_seq_store():
for pillow in get_couch_pillow_instances():
checkpoint = pillow.checkpoint
store, created = PillowCheckpointSeqStore.objects.get_or_create(checkpoint_id=checkpoint.checkpoint_id)
db_seq = checkpoint.get_current_sequence_id()
store_seq = force_seq_int(store.seq) or 0
if not created and force_seq_int(db_seq) < store_seq - EPSILON:
notify_exception(
None,
message="Found seq number lower than previous for {}. "
"This could mean we are in a rewind state".format(store.checkpoint_id),
details={"pillow checkpoint seq": db_seq, "stored seq": store.seq},
)
else:
store.seq = db_seq
store.save()
def check_for_rewind(checkpoint):
historical_checkpoint = HistoricalPillowCheckpoint.get_historical_max(checkpoint.checkpoint_id)
if not historical_checkpoint:
return False
db_seq = checkpoint.get_current_sequence_id()
store_seq = historical_checkpoint.seq_int
has_rewound = force_seq_int(db_seq) < store_seq - EPSILON
return has_rewound, historical_checkpoint.seq
def check_for_rewind(checkpoint):
historical_checkpoint = HistoricalPillowCheckpoint.get_historical_max(checkpoint.checkpoint_id)
if not historical_checkpoint:
return False
db_seq = checkpoint.get_current_sequence_id()
store_seq = historical_checkpoint.seq_int
has_rewound = force_seq_int(db_seq) < store_seq - EPSILON
return has_rewound, historical_checkpoint.seq
def pillow_seq_store():
for pillow in get_couch_pillow_instances():
checkpoint = pillow.checkpoint
store, created = PillowCheckpointSeqStore.objects.get_or_create(
checkpoint_id=checkpoint.checkpoint_id)
db_seq = checkpoint.get_current_sequence_id()
store_seq = force_seq_int(store.seq) or 0
if not created and force_seq_int(db_seq) < store_seq - EPSILON:
notify_exception(
None,
message='Found seq number lower than previous for {}. '
'This could mean we are in a rewind state'.format(
store.checkpoint_id),
details={
'pillow checkpoint seq': db_seq,
'stored seq': store.seq
})
else:
store.seq = db_seq
store.save()
def _normalize_sequence(self, sequence):
from pillowtop.feed.couch import CouchChangeFeed
change_feed = self.get_change_feed()
if not isinstance(sequence, dict):
if isinstance(change_feed, CouchChangeFeed):
topic = change_feed.couch_db
else:
return {}
sequence = {topic: force_seq_int(sequence)}
return sequence
def _normalize_sequence(self, sequence):
from pillowtop.feed.couch import CouchChangeFeed
change_feed = self.get_change_feed()
if not isinstance(sequence, dict):
if isinstance(change_feed, CouchChangeFeed):
topic = change_feed.couch_db
else:
return {}
sequence = {topic: force_seq_int(sequence)}
return sequence
def _normalize_sequence(self, sequence):
from pillowtop.feed.couch import CouchChangeFeed
from corehq.apps.change_feed.consumer.feed import KafkaChangeFeed
change_feed = self.get_change_feed()
if not isinstance(sequence, dict):
if isinstance(change_feed, KafkaChangeFeed):
topics = change_feed.topics
assert len(topics) == 1
topic = topics[0]
elif isinstance(change_feed, CouchChangeFeed):
topic = change_feed.couch_db
else:
return {}
sequence = {topic: force_seq_int(sequence)}
return sequence
def get_latest_offsets(self):
return {
self._couch_db.dbname:
force_seq_int(get_current_seq(self._couch_db))
}
def get_processed_offsets(self):
return {self._couch_db.dbname: force_seq_int(self._last_processed_seq)}
def get_current_offsets(self):
return {self._couch_db.dbname: force_seq_int(self.get_latest_change_id())}
def get_latest_offsets(self):
return {self._couch_db.dbname: force_seq_int(get_current_seq(self._couch_db))}
def get_processed_offsets(self):
return {self._couch_db.dbname: force_seq_int(self._last_processed_seq)}
