def finish_remote_batch(self, src_db, dst_db, tick_id):
"""Worker-specific cleanup on target node.
"""
if self.main_worker:
st = self._worker_state
dst_curs = dst_db.cursor()
self.flush_events(dst_curs)
# send tick event into queue
if st.send_tick_event:
q = "select pgq.insert_event(%s, 'pgq.tick-id', %s, %s, null, null, null)"
dst_curs.execute(
q, [st.target_queue,
str(tick_id), self.pgq_queue_name])
if st.create_tick:
# create actual tick
tick_id = self._batch_info['tick_id']
tick_time = self._batch_info['batch_end']
q = "select pgq.ticker(%s, %s, %s, %s)"
dst_curs.execute(
q,
[self.pgq_queue_name, tick_id, tick_time, self.cur_max_id])
CascadedConsumer.finish_remote_batch(self, src_db, dst_db, tick_id)
def finish_remote_batch(self, src_db, dst_db, tick_id):
"""Worker-specific cleanup on target node.
"""
# merge-leaf on branch should not update tick pos
wst = self._worker_state
if wst.wait_behind:
dst_db.commit()
return
if self.main_worker:
st = self._worker_state
dst_curs = dst_db.cursor()
self.flush_events(dst_curs)
# send tick event into queue
if st.send_tick_event:
q = "select pgq.insert_event(%s, 'pgq.tick-id', %s, %s, null, null, null)"
dst_curs.execute(
q, [st.target_queue,
str(tick_id), self.pgq_queue_name])
CascadedConsumer.finish_remote_batch(self, src_db, dst_db, tick_id)
if self.main_worker:
if st.create_tick:
# create actual tick
tick_id = self.batch_info['tick_id']
tick_time = self.batch_info['batch_end']
self.create_branch_tick(dst_db, tick_id, tick_time)
def finish_remote_batch(self, src_db, dst_db, tick_id):
"""Worker-specific cleanup on target node.
"""
# merge-leaf on branch should not update tick pos
wst = self._worker_state
if wst.wait_behind:
dst_db.commit()
return
if self.main_worker:
st = self._worker_state
dst_curs = dst_db.cursor()
self.flush_events(dst_curs)
# send tick event into queue
if st.send_tick_event:
q = "select pgq.insert_event(%s, 'pgq.tick-id', %s, %s, null, null, null)"
dst_curs.execute(q, [st.target_queue, str(tick_id), self.pgq_queue_name])
CascadedConsumer.finish_remote_batch(self, src_db, dst_db, tick_id)
if self.main_worker:
if st.create_tick:
# create actual tick
tick_id = self.batch_info['tick_id']
tick_time = self.batch_info['batch_end']
self.create_branch_tick(dst_db, tick_id, tick_time)
def reload(self):
CascadedConsumer.reload(self)
self.global_wm_publish_period = self.cf.getfloat(
'global_wm_publish_period',
CascadedWorker.global_wm_publish_period)
self.local_wm_publish_period = self.cf.getfloat(
'local_wm_publish_period', CascadedWorker.local_wm_publish_period)
def __init__(self, service_name, db_name, args):
"""Initialize new consumer.
@param service_name: service_name for DBScript
@param db_name: target database name for get_database()
@param args: cmdline args for DBScript
"""
CascadedConsumer.__init__(self, service_name, db_name, args)
def __init__(self, service_name, db_name, args):
"""Initialize new consumer.
@param service_name: service_name for DBScript
@param db_name: target database name for get_database()
@param args: cmdline args for DBScript
"""
CascadedConsumer.__init__(self, service_name, db_name, args)
def process_remote_event(self, src_curs, dst_curs, ev):
"""Handle cascading events.
"""
if ev.retry:
raise Exception('CascadedWorker must not get retry events')
# non cascade events send to CascadedConsumer to error out
if ev.ev_type[:4] != 'pgq.':
CascadedConsumer.process_remote_event(self, src_curs, dst_curs, ev)
return
# ignore cascade events if not main worker
if not self.main_worker:
return
# check if for right queue
t = ev.ev_type
if ev.ev_extra1 != self.pgq_queue_name and t != "pgq.tick-id":
raise Exception("bad event in queue: " + str(ev))
self.log.debug("got cascade event: %s(%s)" % (t, ev.ev_data))
st = self._worker_state
if t == "pgq.location-info":
node = ev.ev_data
loc = ev.ev_extra2
dead = ev.ev_extra3
q = "select * from pgq_node.register_location(%s, %s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, node, loc, dead])
elif t == "pgq.unregister-location":
node = ev.ev_data
q = "select * from pgq_node.unregister_location(%s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, node])
elif t == "pgq.global-watermark":
if st.sync_watermark:
tick_id = int(ev.ev_data)
self.log.debug('Half-ignoring global watermark %d', tick_id)
self.real_global_wm = tick_id
elif st.process_global_wm:
tick_id = int(ev.ev_data)
q = "select * from pgq_node.set_global_watermark(%s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, tick_id])
elif t == "pgq.tick-id":
tick_id = int(ev.ev_data)
if ev.ev_extra1 == self.pgq_queue_name:
raise Exception('tick-id event for own queue?')
if st.process_tick_event:
q = "select * from pgq_node.set_partition_watermark(%s, %s, %s)"
dst_curs.execute(q,
[self.pgq_queue_name, ev.ev_extra1, tick_id])
else:
raise Exception("unknown cascade event: %s" % t)
def process_remote_event(self, src_curs, dst_curs, ev):
"""Handle cascading events.
"""
if ev.retry:
raise Exception('CascadedWorker must not get retry events')
# non cascade events send to CascadedConsumer to error out
if ev.ev_type[:4] != 'pgq.':
CascadedConsumer.process_remote_event(self, src_curs, dst_curs, ev)
return
# ignore cascade events if not main worker
if not self.main_worker:
return
# check if for right queue
t = ev.ev_type
if ev.ev_extra1 != self.pgq_queue_name and t != "pgq.tick-id":
raise Exception("bad event in queue: "+str(ev))
self.log.debug("got cascade event: %s(%s)" % (t, ev.ev_data))
st = self._worker_state
if t == "pgq.location-info":
node = ev.ev_data
loc = ev.ev_extra2
dead = ev.ev_extra3
q = "select * from pgq_node.register_location(%s, %s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, node, loc, dead])
elif t == "pgq.unregister-location":
node = ev.ev_data
q = "select * from pgq_node.unregister_location(%s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, node])
elif t == "pgq.global-watermark":
if st.sync_watermark:
tick_id = int(ev.ev_data)
self.log.debug('Half-ignoring global watermark %d', tick_id)
self.real_global_wm = tick_id
elif st.process_global_wm:
tick_id = int(ev.ev_data)
q = "select * from pgq_node.set_global_watermark(%s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, tick_id])
elif t == "pgq.tick-id":
tick_id = int(ev.ev_data)
if ev.ev_extra1 == self.pgq_queue_name:
raise Exception('tick-id event for own queue?')
if st.process_tick_event:
q = "select * from pgq_node.set_partition_watermark(%s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, ev.ev_extra1, tick_id])
else:
raise Exception("unknown cascade event: %s" % t)
def process_root_node(self, dst_db):
"""On root node send global watermark downstream.
"""
CascadedConsumer.process_root_node(self, dst_db)
t = time.time()
if t - self.global_wm_publish_time < self.global_wm_publish_period:
return
self.log.debug("Publishing global watermark")
dst_curs = dst_db.cursor()
q = "select * from pgq_node.set_global_watermark(%s, NULL)"
dst_curs.execute(q, [self.pgq_queue_name])
dst_db.commit()
self.global_wm_publish_time = t
def process_root_node(self, dst_db):
"""On root node send global watermark downstream.
"""
CascadedConsumer.process_root_node(self, dst_db)
t = time.time()
if t - self.global_wm_publish_time < self.global_wm_publish_period:
return
self.log.debug("Publishing global watermark")
dst_curs = dst_db.cursor()
q = "select * from pgq_node.set_global_watermark(%s, NULL)"
dst_curs.execute(q, [self.pgq_queue_name])
dst_db.commit()
self.global_wm_publish_time = t
def refresh_state(self, dst_db, full_logic = True):
"""Load also node state from target node.
"""
res = CascadedConsumer.refresh_state(self, dst_db, full_logic)
q = "select * from pgq_node.get_node_info(%s)"
st = self.exec_cmd(dst_db, q, [ self.pgq_queue_name ])
self._worker_state = WorkerState(self.pgq_queue_name, st[0])
return res
def refresh_state(self, dst_db, full_logic=True):
"""Load also node state from target node.
"""
res = CascadedConsumer.refresh_state(self, dst_db, full_logic)
q = "select * from pgq_node.get_node_info(%s)"
st = self.exec_cmd(dst_db, q, [self.pgq_queue_name])
self._worker_state = WorkerState(self.pgq_queue_name, st[0])
return res
def is_batch_done(self, state, batch_info):
wst = self._worker_state
# on combined-branch the target can get several batches ahead
if wst.wait_behind:
cur_tick = batch_info['tick_id']
dst_tick = state['completed_tick']
if cur_tick < dst_tick:
return True
return CascadedConsumer.is_batch_done(self, state, batch_info)
def process_remote_event(self, src_curs, dst_curs, ev):
"""Handle cascading events.
"""
# non cascade events send to CascadedConsumer to error out
if ev.ev_type[:4] != 'pgq.':
CascadedConsumer.process_remote_event(self, src_curs, dst_curs, ev)
return
# ignore cascade events if not main worker
if not self.main_worker:
ev.tag_done()
return
# check if for right queue
t = ev.ev_type
if ev.ev_extra1 != self.pgq_queue_name and t != "pgq.tick-id":
raise Exception("bad event in queue: "+str(ev))
self.log.info("got cascade event: %s" % t)
if t == "pgq.location-info":
node = ev.ev_data
loc = ev.ev_extra2
dead = ev.ev_extra3
q = "select * from pgq_node.register_location(%s, %s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, node, loc, dead])
elif t == "pgq.global-watermark":
tick_id = int(ev.ev_data)
q = "select * from pgq_node.set_global_watermark(%s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, tick_id])
elif t == "pgq.tick-id":
tick_id = int(ev.ev_data)
if ev.ev_extra1 == self.pgq_queue_name:
raise Exception('tick-id event for own queue?')
st = self._worker_state
if st.process_tick_event:
q = "select * from pgq_node.set_partition_watermark(%s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, ev.ev_extra1, tick_id])
else:
raise Exception("unknown cascade event: %s" % t)
ev.tag_done()
def finish_remote_batch(self, src_db, dst_db, tick_id):
"""Worker-specific cleanup on target node.
"""
if self.main_worker:
st = self._worker_state
dst_curs = dst_db.cursor()
self.flush_events(dst_curs)
# send tick event into queue
if st.send_tick_event:
q = "select pgq.insert_event(%s, 'pgq.tick-id', %s, %s, null, null, null)"
dst_curs.execute(q, [st.target_queue, str(tick_id), self.pgq_queue_name])
if st.create_tick:
# create actual tick
tick_id = self._batch_info['tick_id']
tick_time = self._batch_info['batch_end']
q = "select pgq.ticker(%s, %s, %s, %s)"
dst_curs.execute(q, [self.pgq_queue_name, tick_id, tick_time, self.cur_max_id])
CascadedConsumer.finish_remote_batch(self, src_db, dst_db, tick_id)
def is_batch_done(self, state, batch_info, dst_db):
wst = self._worker_state
# on combined-branch the target can get several batches ahead
if wst.wait_behind:
cur_tick = batch_info['tick_id']
dst_tick = state['completed_tick']
if cur_tick < dst_tick:
return True
# check if events have processed
done = CascadedConsumer.is_batch_done(self, state, batch_info, dst_db)
if not wst.create_tick:
return done
if not done:
return False
# check if tick is done - it happens in separate tx
# fetch last tick from target queue
q = "select t.tick_id from pgq.tick t, pgq.queue q"\
" where t.tick_queue = q.queue_id and q.queue_name = %s"\
" order by t.tick_queue desc, t.tick_id desc"\
" limit 1"
curs = dst_db.cursor()
curs.execute(q, [self.queue_name])
last_tick = curs.fetchone()['tick_id']
dst_db.commit()
# insert tick if missing
cur_tick = batch_info['tick_id']
if last_tick != cur_tick:
prev_tick = batch_info['prev_tick_id']
tick_time = batch_info['batch_end']
if last_tick != prev_tick:
raise Exception(
'is_batch_done: last branch tick = %d, expected %d or %d' %
(last_tick, prev_tick, cur_tick))
self.create_branch_tick(dst_db, cur_tick, tick_time)
return True
def is_batch_done(self, state, batch_info, dst_db):
wst = self._worker_state
# on combined-branch the target can get several batches ahead
if wst.wait_behind:
cur_tick = batch_info['tick_id']
dst_tick = state['completed_tick']
if cur_tick < dst_tick:
return True
# check if events have processed
done = CascadedConsumer.is_batch_done(self, state, batch_info, dst_db)
if not wst.create_tick:
return done
if not done:
return False
# check if tick is done - it happens in separate tx
# fetch last tick from target queue
q = "select t.tick_id from pgq.tick t, pgq.queue q"\
" where t.tick_queue = q.queue_id and q.queue_name = %s"\
" order by t.tick_queue desc, t.tick_id desc"\
" limit 1"
curs = dst_db.cursor()
curs.execute(q, [self.queue_name])
last_tick = curs.fetchone()['tick_id']
dst_db.commit()
# insert tick if missing
cur_tick = batch_info['tick_id']
if last_tick != cur_tick:
prev_tick = batch_info['prev_tick_id']
tick_time = batch_info['batch_end']
if last_tick != prev_tick:
raise Exception('is_batch_done: last branch tick = %d, expected %d or %d' % (
last_tick, prev_tick, cur_tick))
self.create_branch_tick(dst_db, cur_tick, tick_time)
return True
def reload(self):
CascadedConsumer.reload(self)
self.global_wm_publish_period = self.cf.getfloat('global_wm_publish_period', CascadedWorker.global_wm_publish_period)
self.local_wm_publish_period = self.cf.getfloat('local_wm_publish_period', CascadedWorker.local_wm_publish_period)