class SetConsumer(skytools.DBScript): last_local_wm_publish_time = 0 last_global_wm_publish_time = 0 main_worker = True reg_ok = False actual_dst_event_id = 0 batch_max_event_id = 0 seq_buffer = 10000 def __init__(self, service_name, args, node_db_name = 'node_db'): skytools.DBScript.__init__(self, service_name, args) self.node_db_name = node_db_name self.consumer_name = self.cf.get('consumer_name', self.job_name) def work(self): self.tick_id_cache = {} self.set_name = self.cf.get('set_name') dst_db = self.get_database(self.node_db_name) dst_curs = dst_db.cursor() dst_node = self.load_node_info(dst_db) if self.main_worker: self.consumer_name = dst_node.name if not dst_node.uptodate: self.tag_node_uptodate(dst_db) if dst_node.paused: return 0 if dst_node.need_action('global-wm-event'): self.publish_global_watermark(dst_db, dst_node.local_watermark) if not dst_node.need_action('process-batch'): return 0 # # batch processing follows # src_db = self.get_database('src_db', connstr = dst_node.provider_location) src_curs = src_db.cursor() src_node = self.load_node_info(src_db) # get batch src_queue = RawQueue(src_node.queue_name, self.consumer_name) self.src_queue = src_queue self.dst_queue = None if not self.main_worker and not self.reg_ok: self.register_consumer(src_curs) batch_id = src_queue.next_batch(src_curs) src_db.commit() if batch_id is None: return 0 self.log.debug("New batch: tick_id=%d / batch_id=%d" % (src_queue.cur_tick, batch_id)) if dst_node.need_action('wait-behind'): if dst_node.should_wait(src_queue.cur_tick): return 0 if dst_node.need_action('process-events'): # load and process batch data ev_list = src_queue.get_batch_events(src_curs) if dst_node.need_action('copy-events'): self.dst_queue = RawQueue(dst_node.get_target_queue(), self.consumer_name) self.process_set_batch(src_db, dst_db, ev_list) if self.dst_queue: self.dst_queue.finish_bulk_insert(dst_curs) self.copy_tick(dst_curs, src_queue, self.dst_queue) # COMBINED_BRANCH needs to sync with part sets if dst_node.need_action('sync-part-pos'): self.move_part_positions(dst_curs) if dst_node.need_action('update-event-seq'): self.update_event_seq(dst_curs) # we are done on target self.set_tick_complete(dst_curs, src_queue.cur_tick) dst_db.commit() # done on source src_queue.finish_batch(src_curs) src_db.commit() # occasinally send watermark upwards if dst_node.need_action('local-wm-publish'): self.send_local_watermark_upwards(src_db, dst_node) # got a batch so there can be more return 1 def process_set_batch(self, src_db, dst_db, ev_list): dst_curs = dst_db.cursor() max_id = 0 for ev in ev_list: self.process_set_event(dst_curs, ev) if self.dst_queue: self.dst_queue.bulk_insert(dst_curs, ev) if ev.id > max_id: max_id = ev.id self.batch_max_event_id = max_id self.stat_increase('count', len(ev_list)) def update_event_seq(self, dst_curs): qname = self.dst_queue.queue_name if self.actual_dst_event_id == 0: q = "select pgq.seq_getval(queue_event_seq) from pgq.queue where queue_name = %s" dst_curs.execute(q, [qname]) self.actual_dst_event_id = dst_curs.fetchone()[0] self.log.debug('got local event_id value = %d' % self.actual_dst_event_id) if self.batch_max_event_id + self.seq_buffer >= self.actual_dst_event_id: next_id = self.batch_max_event_id + 2 * self.seq_buffer q = "select pgq.seq_setval(queue_event_seq, %s) from pgq.queue where queue_name = %s" self.log.debug('set local event_id value = %d' % next_id) dst_curs.execute(q, [next_id, qname]) self.actual_dst_event_id = next_id def process_set_event(self, dst_curs, ev): if ev.type == 'set-tick': self.handle_set_tick(dst_curs, ev) elif ev.type == 'member-info': self.handle_member_info(dst_curs, ev) elif ev.type == 'global-watermark': self.handle_global_watermark(dst_curs, ev) else: raise Exception('bad event for set consumer') def handle_global_watermark(self, dst_curs, ev): set_name = ev.extra1 tick_id = ev.data if set_name == self.set_name: self.set_global_watermark(dst_curs, tick_id) def handle_set_tick(self, dst_curs, ev): data = skytools.db_urldecode(ev.data) set_name = data['set_name'] tick_id = data['tick_id'] self.tick_id_cache[set_name] = tick_id def move_part_positions(self, dst_curs): q = "select * from pgq_set.set_partition_watermark(%s, %s, %s)" for set_name, tick_id in self.tick_id_cache.items(): dst_curs.execute(q, [self.set_name, set_name, tick_id]) def handle_member_info(self, dst_curs, ev): node_name = ev.ev_data set_name = ev.ev_extra1 node_location = ev.ev_extra2 dead = ev.ev_extra3 # this can also be member for part set, ignore then if set_name != self.set_name: return q = "select * from pgq_set.add_member(%s, %s, %s, %s)" dst_curs.execute(q, [set_name, node_name, node_location, dead]) def send_local_watermark_upwards(self, src_db, node): # fixme - delay now = time.time() delay = now - self.last_local_wm_publish_time if delay < 1*60: return self.last_local_wm_publish_time = now self.log.debug("send_local_watermark_upwards") src_curs = src_db.cursor() q = "select pgq_set.set_subscriber_watermark(%s, %s, %s)" src_curs.execute(q, [self.set_name, node.name, node.local_watermark]) src_db.commit() def set_global_watermark(self, dst_curs, tick_id): self.log.debug("set_global_watermark: %s" % tick_id) q = "select pgq_set.set_global_watermark(%s, %s)" dst_curs.execute(q, [self.set_name, tick_id]) def publish_global_watermark(self, dst_db, watermark): now = time.time() delay = now - self.last_global_wm_publish_time if delay < 1*60: return self.last_global_wm_publish_time = now self.set_global_watermark(dst_db.cursor(), watermark) dst_db.commit() def load_node_info(self, db): curs = db.cursor() q = "select * from pgq_set.get_node_info(%s)" curs.execute(q, [self.set_name]) node_row = curs.dictfetchone() if not node_row: raise Exception('node not initialized') q = "select * from pgq_set.get_member_info(%s)" curs.execute(q, [self.set_name]) mbr_list = curs.dictfetchall() db.commit() return NodeInfo(self.set_name, node_row, self.main_worker) def tag_node_uptodate(self, dst_db): dst_curs = dst_db.cursor() q = "select * from pgq_set.set_node_uptodate(%s, true)" dst_curs.execute(q, [self.set_name]) dst_db.commit() def copy_tick(self, dst_curs, src_queue, dst_queue): q = "select * from pgq.ticker(%s, %s, %s)" dst_curs.execute(q, [dst_queue.queue_name, src_queue.cur_tick, src_queue.tick_time]) def set_tick_complete(self, dst_curs, tick_id): q = "select * from pgq_set.set_completed_tick(%s, %s, %s)" dst_curs.execute(q, [self.set_name, self.consumer_name, tick_id]) def register_consumer(self, src_curs): if self.main_worker: raise Exception('main set worker should not play with registrations') q = "select * from pgq.register_consumer(%s, %s)" src_curs.execute(q, [self.src_queue.queue_name, self.consumer_name]) def unregister_consumer(self, src_curs): if self.main_worker: raise Exception('main set worker should not play with registrations') q = "select * from pgq.unregister_consumer(%s, %s)" src_curs.execute(q, [self.src_queue.queue_name, self.consumer_name])
def work(self): self.tick_id_cache = {} self.set_name = self.cf.get('set_name') dst_db = self.get_database(self.node_db_name) dst_curs = dst_db.cursor() dst_node = self.load_node_info(dst_db) if self.main_worker: self.consumer_name = dst_node.name if not dst_node.uptodate: self.tag_node_uptodate(dst_db) if dst_node.paused: return 0 if dst_node.need_action('global-wm-event'): self.publish_global_watermark(dst_db, dst_node.local_watermark) if not dst_node.need_action('process-batch'): return 0 # # batch processing follows # src_db = self.get_database('src_db', connstr = dst_node.provider_location) src_curs = src_db.cursor() src_node = self.load_node_info(src_db) # get batch src_queue = RawQueue(src_node.queue_name, self.consumer_name) self.src_queue = src_queue self.dst_queue = None if not self.main_worker and not self.reg_ok: self.register_consumer(src_curs) batch_id = src_queue.next_batch(src_curs) src_db.commit() if batch_id is None: return 0 self.log.debug("New batch: tick_id=%d / batch_id=%d" % (src_queue.cur_tick, batch_id)) if dst_node.need_action('wait-behind'): if dst_node.should_wait(src_queue.cur_tick): return 0 if dst_node.need_action('process-events'): # load and process batch data ev_list = src_queue.get_batch_events(src_curs) if dst_node.need_action('copy-events'): self.dst_queue = RawQueue(dst_node.get_target_queue(), self.consumer_name) self.process_set_batch(src_db, dst_db, ev_list) if self.dst_queue: self.dst_queue.finish_bulk_insert(dst_curs) self.copy_tick(dst_curs, src_queue, self.dst_queue) # COMBINED_BRANCH needs to sync with part sets if dst_node.need_action('sync-part-pos'): self.move_part_positions(dst_curs) if dst_node.need_action('update-event-seq'): self.update_event_seq(dst_curs) # we are done on target self.set_tick_complete(dst_curs, src_queue.cur_tick) dst_db.commit() # done on source src_queue.finish_batch(src_curs) src_db.commit() # occasinally send watermark upwards if dst_node.need_action('local-wm-publish'): self.send_local_watermark_upwards(src_db, dst_node) # got a batch so there can be more return 1