def _on_update(self, channel, items):
if not items:
self.logger.warning('[{0}] Empty items skipped...'.format(
self.name))
return
gid_set = [item.strip(',') for item in items.split(',')]
if not gid_set:
self.logger.warning('[{0}] Empty gid_set skipped...'.format(
self.name))
return
for gid in gid_set:
if self.dummy:
# sleep random to imitate web-service call
self.logger.info(
'Poller is dummy, not posting [{0}]'.format(gid))
time.sleep(random.randint(1000, 1500) / 1000.0)
else:
#### POLL GOOGLE for data, reschedule for expedited retry if poll fails
if not self.google_poll.poll(gid):
self.logger.warning('Retrying for {0} ...'.format(gid))
self.data.pubsub.broadcast_command(
S1.poller_channel_name('all'), S1.msg_update(), gid)
else:
# the gid will be picked up by poller master and decorated for the next poll
self.data.pubsub.broadcast_data(
S1.poller_channel_name('all-out'), gid)
def run(self, *args, **kwargs):
"""
Goes into infinite blocking listener() loop
"""
callback = {
S1.msg_update(): self._on_update,
S1.msg_validate(): self._on_validate,
S1.msg_register(): self._on_register,
}
self.listener(
[S1.poller_channel_name('all'),
S1.poller_channel_name(self.name)], callback)
def _on_register(self, channel):
gid = self.data.balancer.get_next_registered()
while gid:
self.logger.info('Registring GID: {0}'.format(gid))
self.data.pubsub.broadcast_command(
S1.poller_channel_name(self.name), S1.msg_update(), gid)
gid = self.data.balancer.get_next_registered()
def get_poller_stats_ex(self):
return dict(all_count=self.rc.zcard(S1.gid_set('all')),
pollers=self.get_poller_stats(),
poller_names=list(self.rc.smembers(S1.poller_set())),
register_set_len=self.rc.scard(S1.register_set()),
poll_list_len=self.rc.llen(S1.poller_channel_name('all')))
def run(self, *args, **kwargs):
self.kwargs = kwargs
cfg = config.load_config(kwargs['config_path'], 'poller.json')
self.gid_poll_s = cfg[
'gid_poll_s'] if 'gid_poll_s' in cfg else self.gid_poll_s
self.period_s = cfg['period_s'] if 'period_s' in cfg else self.period_s
self.workers_min = cfg[
'workers_min'] if 'workers_min' in cfg else self.workers_min
self.workers_max = cfg[
'workers_max'] if 'workers_max' in cfg else self.workers_max
self.logger.info(
'Poller v[{0}], name=[{1}], poll delay=[{2}]s, period=[{3}]s starting...'
.format(config.version, self.name, self.gid_poll_s, self.period_s))
# give pub sub some time... not using syncho notifications...
time.sleep(1)
# register self as poller
self.data.register_poller(self.name)
# start worker processes
for n in range(0, self.workers_min):
self.start_worker()
# drop message to self to do immediate poll round
self.broadcast_data(S1.poller_channel_name(self.name), '#')
# start listening
self.listener([
S1.poller_channel_name('all-out'),
S1.poller_channel_name(self.name)
],
None,
timeout=self.period_s)
self.logger.warning('Poller master listener exit!')
# un-register self
self.data.unregister_poller(self.name)
# force kill any remaining workers
while self.workers:
p = self.workers.popitem()
self.logger.warning('Terminating remaining poller {0}!'.format(
p[0]))
p[1].terminate()
self.logger.warning('Poller master process exit!')
def on_terminate(self, *args, **kwargs):
"""
Called by signal handlers from ServiceBase
WARNING: This can be called multiple times during process termination!
"""
self.logger.warning('Poller master is terminating...')
# stop workers
while self.stop_worker():
self.logger.warning('One worker stopped')
# stop self
self.send_exit(S1.poller_channel_name(self.name), self_target=True)
self.logger.warning('Poller master terminate sequence complete!')
def __init__(self,
logger,
name,
data,
providers,
config_path,
dummy=False):
"""
@type logger: Logger
@type data: Data
"""
super(Poller, self).__init__(logger, name, data, providers,
config_path)
self.kwargs = None
# default poller driver period
self.period_s = 2
# how many gids are allowed to expire in period_s before new worker is launched
self.gid_set_threshold = 100
# number of worker processes
self.workers_min = 3
# max number of worker process
self.workers_max = 4
# default gid poll period, 10 min
self.gid_poll_s = 600
# default no poll period, 30 min
self.gid_no_poll_s = 1800
self.started_at = time.time()
self.stats = {
'hour': (self.started_at, 0),
'day': (self.started_at, 0),
}
self.channel_handler = {
S1.poller_channel_name('all-out'): self.on_all_out,
S1.poller_channel_name(self.name): self.on_my_name
}
def schedule_next_batch(self, allow_worker_start=False):
try:
self.logger.info('[{0}] wake up!'.format(self.name))
# get the gid set until all processed
while True:
at_time = time.time()
gid_set = self.data.balancer.get_next_poll_set(at_time +
self.period_s /
2.0)
gid_set_len = len(gid_set)
if not gid_set_len:
self.logger.warning('[{0}] Empty gid_set...'.format(
self.name))
return
elif allow_worker_start and gid_set_len > self.gid_set_threshold:
self.logger.warning(
'Gid set count [{0}] above threshold, starting worker...'
.format(gid_set_len))
self.start_worker()
self.logger.info(
'[{0}] Invoking poll for [{1}] items...'.format(
self.name, gid_set_len))
# clean orphaned gids
update_set = [
gid for gid in gid_set
if not self.data.check_orphan(gid, at_time)
]
# post each gid to poller
for gid in update_set:
# move next poll time for the gid to avoid duplicate polling
self.data.balancer.add_gid_set(gid,
at_time + self.gid_poll_s)
# post to pollers
self.broadcast_command(S1.poller_channel_name('all'),
S1.msg_update(), gid)
# update stats
self.update_stats(at_time, len(update_set))
except Exception as e:
self.logger.warning('Exception in poller driver: {0}'.format(e))
self.logger.exception(traceback.format_exc())
self.data.unregister_poller(self.name)
def _stop_worker(self, p):
self.logger.info('Stopping worker: {0}...'.format(p[0]))
self.data.unregister_poller(p[0])
self.send_exit(S1.poller_channel_name(p[0]))
p[1].join()
def register_gid(self, gid):
# add to balance list
self.rc.sadd(S1.register_set(), gid)
# notify poller(s)
self.pubsub.broadcast_command_now(S1.poller_channel_name('all'),
S1.msg_register())