def migrate_kafka_sequence(change_feed, checkpoint): int_seq = None try: # optimistically try convert to int int_seq = int(checkpoint.sequence) except ValueError: pass if checkpoint.sequence_format == 'text' or int_seq is not None: topics = change_feed.topics assert len(topics) == 1, topics return kafka_seq_to_str({(topics[0], 0): int_seq}) elif checkpoint.sequence_format == 'json': sequence = json.loads(checkpoint.sequence) if not sequence: # if sequence is an empty dict just return it return sequence change_feed_topics = set(change_feed.topics) assert change_feed_topics <= set(sequence) return kafka_seq_to_str({(topic, 0): offset for topic, offset in sequence.items() if topic in change_feed_topics}) else: raise ValueError("Unknown checkpoint format: {}".format( checkpoint.sequence_format))
def update_to(self, seq): kafka_seq = None if isinstance(seq, dict): assert self.sequence_format == 'json' kafka_seq = seq seq = kafka_seq_to_str(seq) elif isinstance(seq, int): seq = str(seq) pillow_logging.info( "(%s) setting checkpoint: %s" % (self.checkpoint_id, seq) ) with transaction.atomic(): if kafka_seq: for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=self.checkpoint_id, topic=topic_partition[0], partition=topic_partition[1], defaults={'offset': offset} ) checkpoint = self.get_or_create_wrapped(verify_unchanged=True) checkpoint.sequence = seq checkpoint.timestamp = datetime.utcnow() checkpoint.save() self._last_checkpoint = checkpoint
def update_to(self, seq, change=None): kafka_seq = None if isinstance(seq, dict): assert self.sequence_format == 'json' kafka_seq = seq seq = kafka_seq_to_str(seq) elif isinstance(seq, int): seq = str(seq) pillow_logging.info( "(%s) setting checkpoint: %s" % (self.checkpoint_id, seq) ) doc_modification_time = change.metadata.publish_timestamp if change else None with transaction.atomic(): if kafka_seq: for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=self.checkpoint_id, topic=topic_partition[0], partition=topic_partition[1], defaults={'offset': offset, 'doc_modification_time': doc_modification_time} ) checkpoint = self.get_or_create_wrapped(verify_unchanged=True) checkpoint.sequence = seq checkpoint.timestamp = datetime.utcnow() checkpoint.save() self._last_checkpoint = checkpoint
def update_to(self, seq): kafka_seq = seq seq = kafka_seq_to_str(seq) pillow_logging.info( "(%s) setting checkpoint: %s" % (self.checkpoint_id, seq) ) with transaction.atomic(): if kafka_seq: for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=self.checkpoint_id, topic=topic_partition[0], partition=topic_partition[1], defaults={'offset': offset} )
def handle(self, topic, num_partitions, **options): stop_pillows = raw_input("did you stop pillows? [y/n]") if stop_pillows not in ['y', 'yes']: print("then stop them") kafka_command = ( "./kafka-topics.sh --alter --zookeeper <zk IP>:2181 --partitions={} --topic={}" .format(num_partitions, topic)) added_partition = raw_input( "have you run {} ? [y/n]".format(kafka_command)) if added_partition not in ['y', 'yes']: print("then run it") for checkpoint in DjangoPillowCheckpoint.objects.filter( sequence_format='json'): try: kafka_seq = str_to_kafka_seq(checkpoint.sequence) except ValueError: print("unable to parse {}", checkpoint.checkpoint_id) continue topics = [tp.topic for tp in kafka_seq] if topic not in topics: print("topic does not exist in {}", checkpoint.checkpoint_id) continue changed = False for partition in range(num_partitions): tp = TopicAndPartition(topic, partition) if tp in kafka_seq: continue else: changed = True kafka_seq[tp] = 0 if changed: checkpoint.old_sequence = checkpoint.sequence checkpoint.sequence = kafka_seq_to_str(kafka_seq) checkpoint.save() for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=checkpoint.checkpoint_id, topic=topic_partition.topic, partition=topic_partition.partition, defaults={'offset': offset}) print("please restart the pillows")
def update_to(self, seq, change=None): if isinstance(seq, six.string_types): kafka_seq = str_to_kafka_seq(seq) else: kafka_seq = seq seq = kafka_seq_to_str(seq) pillow_logging.info( "(%s) setting checkpoint: %s" % (self.checkpoint_id, seq) ) doc_modification_time = change.metadata.publish_timestamp if change else None with transaction.atomic(): if kafka_seq: for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=self.checkpoint_id, topic=topic_partition[0], partition=topic_partition[1], defaults={'offset': offset, 'doc_modification_time': doc_modification_time} )
def update_to(self, seq, change=None): if isinstance(seq, str): kafka_seq = str_to_kafka_seq(seq) else: kafka_seq = seq seq = kafka_seq_to_str(seq) pillow_logging.info( "(%s) setting checkpoint: %s" % (self.checkpoint_id, seq) ) doc_modification_time = change.metadata.publish_timestamp if change else None with transaction.atomic(): if kafka_seq: for topic_partition, offset in kafka_seq.items(): KafkaCheckpoint.objects.update_or_create( checkpoint_id=self.checkpoint_id, topic=topic_partition[0], partition=topic_partition[1], defaults={'offset': offset, 'doc_modification_time': doc_modification_time} )
def get_latest_offsets_json(self): return json.loads(kafka_seq_to_str(self.get_latest_offsets()))
def _seq_to_int(checkpoint, seq): from pillowtop.models import kafka_seq_to_str if _is_kafka(checkpoint): return json.loads(kafka_seq_to_str(seq)) else: return force_seq_int(seq)
def get_current_sequence_id(self): return kafka_seq_to_str(self.get_current_sequence_as_dict())
def get_current_sequence_id(self): return kafka_seq_to_str(self.get_current_sequence_as_dict())
def _seq_to_int(checkpoint, seq): from pillowtop.models import kafka_seq_to_str if checkpoint.sequence_format == 'json': return json.loads(kafka_seq_to_str(seq)) else: return force_seq_int(seq)
def get_latest_offsets_json(self): return json.loads(kafka_seq_to_str(self.get_latest_offsets()))
def _seq_to_int(checkpoint, seq): from pillowtop.models import kafka_seq_to_str if _is_kafka(checkpoint): return json.loads(kafka_seq_to_str(seq)) else: return force_seq_int(seq)