def _on_register(self, channel):
gid = self.data.balancer.get_next_registered()
while gid:
self.logger.info('Registring GID: {0}'.format(gid))
self.data.pubsub.broadcast_command(
S1.poller_channel_name(self.name), S1.msg_update(), gid)
gid = self.data.balancer.get_next_registered()
def _on_update(self, channel, items):
if not items:
self.logger.warning('[{0}] Empty items skipped...'.format(
self.name))
return
gid_set = [item.strip(',') for item in items.split(',')]
if not gid_set:
self.logger.warning('[{0}] Empty gid_set skipped...'.format(
self.name))
return
for gid in gid_set:
if self.dummy:
# sleep random to imitate web-service call
self.logger.info(
'Poller is dummy, not posting [{0}]'.format(gid))
time.sleep(random.randint(1000, 1500) / 1000.0)
else:
#### POLL GOOGLE for data, reschedule for expedited retry if poll fails
if not self.google_poll.poll(gid):
self.logger.warning('Retrying for {0} ...'.format(gid))
self.data.pubsub.broadcast_command(
S1.poller_channel_name('all'), S1.msg_update(), gid)
else:
# the gid will be picked up by poller master and decorated for the next poll
self.data.pubsub.broadcast_data(
S1.poller_channel_name('all-out'), gid)
def run(self, *args, **kwargs):
"""
Goes into infinite blocking listener() loop
"""
callback = {
S1.msg_update(): self._on_update,
S1.msg_validate(): self._on_validate,
S1.msg_register(): self._on_register,
}
self.listener(
[S1.poller_channel_name('all'),
S1.poller_channel_name(self.name)], callback)
def schedule_next_batch(self, allow_worker_start=False):
try:
self.logger.info('[{0}] wake up!'.format(self.name))
# get the gid set until all processed
while True:
at_time = time.time()
gid_set = self.data.balancer.get_next_poll_set(at_time +
self.period_s /
2.0)
gid_set_len = len(gid_set)
if not gid_set_len:
self.logger.warning('[{0}] Empty gid_set...'.format(
self.name))
return
elif allow_worker_start and gid_set_len > self.gid_set_threshold:
self.logger.warning(
'Gid set count [{0}] above threshold, starting worker...'
.format(gid_set_len))
self.start_worker()
self.logger.info(
'[{0}] Invoking poll for [{1}] items...'.format(
self.name, gid_set_len))
# clean orphaned gids
update_set = [
gid for gid in gid_set
if not self.data.check_orphan(gid, at_time)
]
# post each gid to poller
for gid in update_set:
# move next poll time for the gid to avoid duplicate polling
self.data.balancer.add_gid_set(gid,
at_time + self.gid_poll_s)
# post to pollers
self.broadcast_command(S1.poller_channel_name('all'),
S1.msg_update(), gid)
# update stats
self.update_stats(at_time, len(update_set))
except Exception as e:
self.logger.warning('Exception in poller driver: {0}'.format(e))
self.logger.exception(traceback.format_exc())
self.data.unregister_poller(self.name)