def _forward_worker(self):
LOG.debug('forward worker begun')
session = neutron_db_api.get_session()
while True:
try:
def work(k, v):
LOG.debug('forward worker updating etcd key %s' % k)
if self.do_etcd_update(k, v):
return True
else:
# something went bad; breathe, in case we end
# up in a tight loop
time.sleep(1)
return False
LOG.debug('forward worker reading journal')
while db.journal_read(session, work):
pass
LOG.debug('forward worker has emptied journal')
# work queue is now empty.
LOG.debug("ML2_VPP(%s): worker thread pausing" %
self.__class__.__name__)
# Wait to be kicked, or (in case of emergency) run every
# few seconds in case another thread or process dumped
# work and failed to process it
try:
with eventlet.Timeout(PARANOIA_TIME):
# Wait for kick
dummy = self.db_q_ev.wait()
# Clear the event - we will now process till
# we've run out of things in the backlog
# so any trigger lost in this gap is harmless
self.db_q_ev.reset()
LOG.debug("ML2_VPP(%s): worker thread kicked: %s" %
(self.__class__.__name__, str(dummy)))
except eventlet.Timeout:
LOG.debug("ML2_VPP(%s): worker thread suspicious of "
"a long pause" % self.__class__.__name__)
pass
LOG.debug("ML2_VPP(%s): worker thread active" %
self.__class__.__name__)
except Exception as e:
# TODO(ijw): log exception properly
LOG.error("problems in forward worker: %s", e)
LOG.error(traceback.format_exc())
# never quit
pass
def _forward_worker(self):
LOG.debug('forward worker begun')
session = neutron_db_api.get_session()
while True:
try:
def work(k, v):
LOG.debug('forward worker updating etcd key %s' % k)
if self.do_etcd_update(k, v):
return True
else:
os.sleep(1) # something went bad; breathe, in
# case we end up in a tight loop
return False
LOG.debug('forward worker reading journal')
while db.journal_read(session, work):
pass
LOG.debug('forward worker has emptied journal')
# work queue is now empty.
LOG.debug("ML2_VPP(%s): worker thread pausing" % self.__class__.__name__)
# Wait to be kicked, or (in case of emergency) run every
# few seconds in case another thread or process dumped
# work and failed to process it
try:
with eventlet.Timeout(PARANOIA_TIME) as t:
# Wait for kick
dummy = self.db_q_ev.wait()
# Clear the event - we will now process till
# we've run out of things in the backlog
# so any trigger lost in this gap is harmless
self.db_q_ev.reset()
LOG.debug("ML2_VPP(%s): worker thread kicked: %s" % (self.__class__.__name__, str(dummy)))
except eventlet.Timeout:
LOG.debug("ML2_VPP(%s): worker thread suspicious of a long pause" % self.__class__.__name__)
pass
LOG.debug("ML2_VPP(%s): worker thread active" % self.__class__.__name__)
except Exception, e:
# TODO(ijw): log exception properly
LOG.error("problems in forward worker: %s", e)
LOG.error(traceback.format_exc())
def _forward_worker(self):
LOG.debug('forward worker begun')
etcd_client = self.client_factory.client()
etcd_writer = etcdutils.SignedEtcdJSONWriter(etcd_client)
lease_time = cfg.CONF.ml2_vpp.forward_worker_master_lease_time
recovery_time = cfg.CONF.ml2_vpp.forward_worker_recovery_time
etcd_election = etcdutils.EtcdElection(etcd_client,
'forward_worker',
self.election_key_space,
work_time=lease_time,
recovery_time=recovery_time)
while True:
# Try indefinitely to regain the mastery of this thread pool. Most
# threads will be sitting here
etcd_election.wait_until_elected()
try:
# Master loop - as long as we are master and can
# maintain it, process incoming events.
# Every long running section is preceded by extending
# mastership of the thread pool and followed by
# confirmation that we still have mastership (usually
# by a further extension).
def work(k, v):
self.do_etcd_update(etcd_writer, k, v)
# We will try to empty the pending rows in the DB
while True:
etcd_election.extend_election(
cfg.CONF.ml2_vpp.db_query_time)
session = neutron_db_api.get_session()
maybe_more = db.journal_read(session, work)
if not maybe_more:
LOG.debug('forward worker has emptied journal')
etcd_election.extend_election(lease_time)
break
# work queue is now empty.
# Wait to be kicked, or (in case of emergency) run
# every few seconds in case another thread or process
# dumped work and failed to get notification to us to
# process it.
with eventlet.Timeout(lease_time + 1, False):
etcd_election.extend_election(lease_time)
try:
etcd_client.watch(self.journal_kick_key,
timeout=lease_time)
except etcd.EtcdException:
# Check the DB queue now, anyway
pass
except etcdutils.EtcdElectionLost:
# We are no longer master
pass
except Exception as e:
# TODO(ijw): log exception properly
LOG.warning(
"problems in forward worker - Error name is %s. "
"proceeding without quiting",
type(e).__name__)
# something went bad; breathe, in case we end
# up in a tight loop
time.sleep(1)
# never quit
pass
def _forward_worker(self, thread_id):
LOG.debug('forward worker begun')
session = neutron_db_api.get_session()
etcd_election = EtcdElection(self.etcd_client, 'forward_worker',
self.election_key_space, thread_id,
wait_until_elected=True,
recovery_time=3)
while True:
try:
etcd_election.wait_until_elected()
def work(k, v):
LOG.debug('forward worker updating etcd key %s', k)
if self.do_etcd_update(k, v):
return True
else:
# something went bad; breathe, in case we end
# up in a tight loop
time.sleep(1)
return False
LOG.debug('forward worker reading journal')
# TODO(najoy): Limit the journal read entries processed
# by a worker thread to a finite number, say 50.
# This will ensure that one thread does not run forever.
# The re-election process will wake up one of the sleeping
# threads after the specified recovery_time of 3 seconds
# and will get a chance to split the available work
while db.journal_read(session, work):
pass
LOG.debug('forward worker has emptied journal')
# work queue is now empty.
LOG.debug("ML2_VPP(%s): worker thread pausing"
% self.__class__.__name__)
# Wait to be kicked, or (in case of emergency) run every
# few seconds in case another thread or process dumped
# work and failed to process it
try:
with eventlet.Timeout(PARANOIA_TIME):
# Wait for kick
dummy = self.db_q_ev.wait()
# Clear the event - we will now process till
# we've run out of things in the backlog
# so any trigger lost in this gap is harmless
self.db_q_ev.reset()
LOG.debug("ML2_VPP(%s): worker thread kicked: %s"
% (self.__class__.__name__, str(dummy)))
except eventlet.Timeout:
LOG.debug("ML2_VPP(%s): worker thread suspicious of "
"a long pause"
% self.__class__.__name__)
pass
LOG.debug("ML2_VPP(%s): worker thread active"
% self.__class__.__name__)
except Exception as e:
# TODO(ijw): log exception properly
LOG.error("problems in forward worker: %s", e)
LOG.error(traceback.format_exc())
# never quit
pass