class MJPEGProxy:
def __init__(self, listen_address, connect_address):
self.log = logging.getLogger('MJPEGProxy')
self.connect_address = connect_address
self.listen_address = listen_address
self.clients = []
self.connection = None
self.header = None
self.sem = Semaphore(1)
def run(self):
self.log.info("Starting")
eventlet.spawn_n(self.proxy)
self.listen()
def listen(self):
server = eventlet.listen(self.listen_address)
while True:
connection, address = server.accept()
self.add_client(connection, address)
def proxy(self):
while True:
eventlet.sleep(0) # sem.release(); sem.acquire() doesn't yield?!
self.sem.acquire()
if len(self.clients) == 0:
if self.connection:
self.disconnect()
self.sem.release()
eventlet.sleep(0.1)
continue
self.sem.release()
data = ''
try:
data = self.connection.recv(1024)
except:
self.log.info("Timed out reading data from source.")
if (len(data) == 0):
self.log.info("No data recieved from source, forcing reconnect.");
self.disconnect()
data = self.connect()
for client in self.clients:
try:
client.send(data)
except socket.error, err:
self.clients.remove(client)
client.close()
self.log.info("Client %s disconnected: %s [clients: %s]", client, err, len(self.clients))
class EntrypointWaiter(DependencyProvider):
"""Helper for `entrypoint_waiter`
DependencyProvider to be manually (and temporarily) added to an existing
container. Takes an entrypoint name, and exposes a `wait` method, which
will return once the entrypoint has fired.
"""
def __init__(self, entrypoint):
self.attr_name = '_entrypoint_waiter_{}'.format(entrypoint)
self.entrypoint = entrypoint
self.done = Semaphore(value=0)
def worker_teardown(self, worker_ctx):
entrypoint = worker_ctx.entrypoint
if entrypoint.method_name == self.entrypoint:
self.done.release()
def wait(self):
self.done.acquire()
class EntrypointWaiter(InjectionProvider):
"""Helper for `entrypoint_waiter`
Injection to be manually (and temporarily) added to an existing container.
Takes an entrypoint name, and exposes a `wait` method, which will return
once the entrypoint has fired.
"""
def __init__(self, entrypoint):
self.name = '_entrypoint_waiter_{}'.format(entrypoint)
self.entrypoint = entrypoint
self.done = Semaphore(value=0)
def worker_teardown(self, worker_ctx):
provider = worker_ctx.provider
if provider.name == self.entrypoint:
self.done.release()
def wait(self):
self.done.acquire()
class EntrypointWaiter(DependencyProvider):
"""Helper for `entrypoint_waiter`
DependencyProvider to be manually (and temporarily) added to an existing
container. Takes an entrypoint name, and exposes a `wait` method, which
will return once the entrypoint has fired.
"""
class Timeout(Exception):
pass
def __init__(self, entrypoint):
self.attr_name = '_entrypoint_waiter_{}'.format(entrypoint)
self.entrypoint = entrypoint
self.done = Semaphore(value=0)
def worker_teardown(self, worker_ctx):
entrypoint = worker_ctx.entrypoint
if entrypoint.method_name == self.entrypoint:
self.done.release()
def wait(self):
self.done.acquire()
class ServiceEngine(Node):
def __init__(self, **kwargs):
self.sessions = []
self.garbageLoop = hub.spawn_after(1, self._garbageCollector)
super(ServiceEngine, self).__init__(**kwargs)
self.type = 'se'
self.handover = None
self.rsttcp = None
self.lock = Semaphore()
def __str__(self):
return 'Service Engine node. HTTP engine on {}:{:d}'.format(self.ip, self.port) + \
'. Attached to Access Switch {} port id {:d}'.format(self.datapath_id, self.port_id) if self.datapath_id else ''
def __eq__(self, other):
return isinstance(other, ServiceEngine) and \
self.name == other.name and \
self.ip == other.ip and \
self.port == other.port
def setHandoverCallback(self, fn):
self.handover = fn
def setRSTCallback(self, fn):
self.rsttcp = fn
def _performHandover(self, sess):
self.handover(sess)
def _garbageCollector(self):
self.lock.acquire()
for sess in self.sessions[:]: # type: TCPSesssion
if sess.state in [
TCPSesssion.STATE_CLOSED, TCPSesssion.STATE_TIMEOUT,
TCPSesssion.STATE_CLOSED_RESET,
TCPSesssion.STATE_HANDOVERED
]:
self.logger.info('Removing finished session ' + str(sess))
self.sessions.remove(sess)
self.lock.release()
self.garbageLoop = hub.spawn_after(1, self._garbageCollector)
def handlePacket(self, pkt, eth, ip, ptcp):
"""
Handles packet and returns the packet. Packet might change
:param pkt:
:param eth:
:type eth: ethernet.ethernet
:param ip:
:type ip: ipv4.ipv4
:param ptcp:
:type ptcp: tcp.tcp
:return:
"""
self.lock.acquire()
for sess in self.sessions: #type: TCPSesssion
if sess.ip.src == ip.src and \
sess.ip.dst == ip.dst and \
sess.ptcp.src_port == ptcp.src_port and \
sess.ptcp.dst_port == ptcp.dst_port:
pkt = sess.handlePacket(pkt, eth, ip, ptcp)
if sess.handoverReady and not sess.handovered:
self.logger.debug(
'Handover is ready on SE too. Requesting CNT to do the dirty stuff'
)
self._performHandover(sess)
sess.handovered = True
self.lock.release()
return pkt, sess
self.lock.release()
return pkt, None
if sess.ip.dst == ip.src and \
sess.ip.src == ip.dst and \
sess.ptcp.src_port == ptcp.dst_port and \
sess.ptcp.dst_port == ptcp.src_port:
pkt = sess.handlePacket(pkt, eth, ip, ptcp)
self.lock.release()
return pkt, None
# Create a new TCP session if the existin session is not found
if ptcp.bits & tcp.TCP_SYN:
sess = TCPSesssion(pkt, eth, ip, ptcp, self)
self.sessions.append(sess)
self.lock.release()
return pkt, None
else:
self.logger.error(
'Unexpected non SYN packet arrived to processing')
self.logger.error(
"Packet went through pipeline without match in SE {}:{}<->{}:{}".
format(ip.src, ptcp.src_port, ip.dst, ptcp.dst_port))
self.lock.release()
return pkt, None
class Coroutine(object):
"""
This class simulates a coroutine, which is ironic, as greenlet actually
*is* a coroutine. But trying to use greenlet here gives nasty results
since eventlet thoroughly monkey-patches things, making it difficult
to run greenlet on its own.
Essentially think of this as a wrapper for eventlet's threads which has a
run and sleep function similar to old school coroutines, meaning it won't
start until told and when asked to sleep it won't wake back up without
permission.
"""
ALL = []
def __init__(self, func, *args, **kwargs):
self.my_sem = Semaphore(0) # This is held by the thread as it runs.
self.caller_sem = None
self.dead = False
started = Event()
self.id = 5
self.ALL.append(self)
def go():
self.id = eventlet.corolocal.get_ident()
started.send(True)
self.my_sem.acquire(blocking=True, timeout=None)
try:
func(*args, **kwargs)
# except Exception as e:
# print("Exception in coroutine! %s" % e)
finally:
self.dead = True
self.caller_sem.release() # Relinquish control back to caller.
for i in range(len(self.ALL)):
if self.ALL[i].id == self.id:
del self.ALL[i]
break
true_spawn(go)
started.wait()
@classmethod
def get_current(cls):
"""Finds the coroutine associated with the thread which calls it."""
return cls.get_by_id(eventlet.corolocal.get_ident())
@classmethod
def get_by_id(cls, id):
for cr in cls.ALL:
if cr.id == id:
return cr
raise RuntimeError("Coroutine with id %s not found!" % id)
def sleep(self):
"""Puts the coroutine to sleep until run is called again.
This should only be called by the thread which owns this object.
"""
# Only call this from its own thread.
assert eventlet.corolocal.get_ident() == self.id
self.caller_sem.release() # Relinquish control back to caller.
self.my_sem.acquire(blocking=True, timeout=None)
def run(self):
"""Starts up the thread. Should be called from a different thread."""
# Don't call this from the thread which it represents.
assert eventlet.corolocal.get_ident() != self.id
self.caller_sem = Semaphore(0)
self.my_sem.release()
self.caller_sem.acquire() # Wait for it to finish.
class Pool(object):
def __init__(self, min_size=0, max_size=4, track_events=False):
if min_size > max_size:
raise ValueError('min_size cannot be bigger than max_size')
self.max_size = max_size
self.sem = Semaphore(max_size)
self.procs = proc.RunningProcSet()
if track_events:
self.results = coros.queue()
else:
self.results = None
def resize(self, new_max_size):
""" Change the :attr:`max_size` of the pool.
If the pool gets resized when there are more than *new_max_size*
coroutines checked out, when they are returned to the pool they will be
discarded. The return value of :meth:`free` will be negative in this
situation.
"""
max_size_delta = new_max_size - self.max_size
self.sem.counter += max_size_delta
self.max_size = new_max_size
@property
def current_size(self):
""" The number of coroutines that are currently executing jobs. """
return len(self.procs)
def free(self):
""" Returns the number of coroutines that are available for doing
work."""
return self.sem.counter
def execute(self, func, *args, **kwargs):
"""Execute func in one of the coroutines maintained
by the pool, when one is free.
Immediately returns a :class:`~eventlet.proc.Proc` object which can be
queried for the func's result.
>>> pool = Pool()
>>> task = pool.execute(lambda a: ('foo', a), 1)
>>> task.wait()
('foo', 1)
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
if self.sem.locked() and api.getcurrent() in self.procs:
p = proc.spawn(func, *args, **kwargs)
try:
p.wait()
except:
pass
else:
self.sem.acquire()
p = self.procs.spawn(func, *args, **kwargs)
# assuming the above line cannot raise
p.link(lambda p: self.sem.release())
if self.results is not None:
p.link(self.results)
return p
execute_async = execute
def _execute(self, evt, func, args, kw):
p = self.execute(func, *args, **kw)
p.link(evt)
return p
def waitall(self):
""" Calling this function blocks until every coroutine
completes its work (i.e. there are 0 running coroutines)."""
return self.procs.waitall()
wait_all = waitall
def wait(self):
"""Wait for the next execute in the pool to complete,
and return the result."""
return self.results.wait()
def waiting(self):
"""Return the number of coroutines waiting to execute.
"""
if self.sem.balance < 0:
return -self.sem.balance
else:
return 0
def killall(self):
""" Kill every running coroutine as immediately as possible."""
return self.procs.killall()
def launch_all(self, function, iterable):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Discard values returned by ``function()``. You should call
``wait_all()`` to wait for all coroutines, newly-launched plus any
previously-submitted :meth:`execute` or :meth:`execute_async` calls, to
complete.
>>> pool = Pool()
>>> def saw(x):
... print "I saw %s!" % x
...
>>> pool.launch_all(saw, "ABC")
>>> pool.wait_all()
I saw A!
I saw B!
I saw C!
"""
for tup in iterable:
self.execute(function, *tup)
def process_all(self, function, iterable):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Discard values returned by ``function()``. Don't return until all
coroutines, newly-launched plus any previously-submitted :meth:`execute()`
or :meth:`execute_async` calls, have completed.
>>> from eventlet import coros
>>> pool = coros.CoroutinePool()
>>> def saw(x): print "I saw %s!" % x
...
>>> pool.process_all(saw, "DEF")
I saw D!
I saw E!
I saw F!
"""
self.launch_all(function, iterable)
self.wait_all()
def generate_results(self, function, iterable, qsize=None):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Yield each of the values returned by ``function()``, in the order
they're completed rather than the order the coroutines were launched.
Iteration stops when we've yielded results for each arguments tuple in
*iterable*. Unlike :meth:`wait_all` and :meth:`process_all`, this
function does not wait for any previously-submitted :meth:`execute` or
:meth:`execute_async` calls.
Results are temporarily buffered in a queue. If you pass *qsize=*, this
value is used to limit the max size of the queue: an attempt to buffer
too many results will suspend the completed :class:`CoroutinePool`
coroutine until the requesting coroutine (the caller of
:meth:`generate_results`) has retrieved one or more results by calling
this generator-iterator's ``next()``.
If any coroutine raises an uncaught exception, that exception will
propagate to the requesting coroutine via the corresponding ``next()``
call.
What I particularly want these tests to illustrate is that using this
generator function::
for result in generate_results(function, iterable):
# ... do something with result ...
pass
executes coroutines at least as aggressively as the classic eventlet
idiom::
events = [pool.execute(function, *args) for args in iterable]
for event in events:
result = event.wait()
# ... do something with result ...
even without a distinct event object for every arg tuple in *iterable*,
and despite the funny flow control from interleaving launches of new
coroutines with yields of completed coroutines' results.
(The use case that makes this function preferable to the classic idiom
above is when the *iterable*, which may itself be a generator, produces
millions of items.)
>>> from eventlet import coros
>>> import string
>>> pool = coros.CoroutinePool(max_size=5)
>>> pausers = [coros.Event() for x in xrange(2)]
>>> def longtask(evt, desc):
... print "%s woke up with %s" % (desc, evt.wait())
...
>>> pool.launch_all(longtask, zip(pausers, "AB"))
>>> def quicktask(desc):
... print "returning %s" % desc
... return desc
...
(Instead of using a ``for`` loop, step through :meth:`generate_results`
items individually to illustrate timing)
>>> step = iter(pool.generate_results(quicktask, string.ascii_lowercase))
>>> print step.next()
returning a
returning b
returning c
a
>>> print step.next()
b
>>> print step.next()
c
>>> print step.next()
returning d
returning e
returning f
d
>>> pausers[0].send("A")
>>> print step.next()
e
>>> print step.next()
f
>>> print step.next()
A woke up with A
returning g
returning h
returning i
g
>>> print "".join([step.next() for x in xrange(3)])
returning j
returning k
returning l
returning m
hij
>>> pausers[1].send("B")
>>> print "".join([step.next() for x in xrange(4)])
B woke up with B
returning n
returning o
returning p
returning q
klmn
"""
# Get an iterator because of our funny nested loop below. Wrap the
# iterable in enumerate() so we count items that come through.
tuples = iter(enumerate(iterable))
# If the iterable is empty, this whole function is a no-op, and we can
# save ourselves some grief by just quitting out. In particular, once
# we enter the outer loop below, we're going to wait on the queue --
# but if we launched no coroutines with that queue as the destination,
# we could end up waiting a very long time.
try:
index, args = tuples.next()
except StopIteration:
return
# From this point forward, 'args' is the current arguments tuple and
# 'index+1' counts how many such tuples we've seen.
# This implementation relies on the fact that _execute() accepts an
# event-like object, and -- unless it's None -- the completed
# coroutine calls send(result). We slyly pass a queue rather than an
# event -- the same queue instance for all coroutines. This is why our
# queue interface intentionally resembles the event interface.
q = coros.queue(max_size=qsize)
# How many results have we yielded so far?
finished = 0
# This first loop is only until we've launched all the coroutines. Its
# complexity is because if iterable contains more args tuples than the
# size of our pool, attempting to _execute() the (poolsize+1)th
# coroutine would suspend until something completes and send()s its
# result to our queue. But to keep down queue overhead and to maximize
# responsiveness to our caller, we'd rather suspend on reading the
# queue. So we stuff the pool as full as we can, then wait for
# something to finish, then stuff more coroutines into the pool.
try:
while True:
# Before each yield, start as many new coroutines as we can fit.
# (The self.free() test isn't 100% accurate: if we happen to be
# executing in one of the pool's coroutines, we could _execute()
# without waiting even if self.free() reports 0. See _execute().)
# The point is that we don't want to wait in the _execute() call,
# we want to wait in the q.wait() call.
# IMPORTANT: at start, and whenever we've caught up with all
# coroutines we've launched so far, we MUST iterate this inner
# loop at least once, regardless of self.free() -- otherwise the
# q.wait() call below will deadlock!
# Recall that index is the index of the NEXT args tuple that we
# haven't yet launched. Therefore it counts how many args tuples
# we've launched so far.
while self.free() > 0 or finished == index:
# Just like the implementation of execute_async(), save that
# we're passing our queue instead of None as the "event" to
# which to send() the result.
self._execute(q, function, args, {})
# We've consumed that args tuple, advance to next.
index, args = tuples.next()
# Okay, we've filled up the pool again, yield a result -- which
# will probably wait for a coroutine to complete. Although we do
# have q.ready(), so we could iterate without waiting, we avoid
# that because every yield could involve considerable real time.
# We don't know how long it takes to return from yield, so every
# time we do, take the opportunity to stuff more requests into the
# pool before yielding again.
yield q.wait()
# Be sure to count results so we know when to stop!
finished += 1
except StopIteration:
pass
# Here we've exhausted the input iterable. index+1 is the total number
# of coroutines we've launched. We probably haven't yielded that many
# results yet. Wait for the rest of the results, yielding them as they
# arrive.
while finished < index + 1:
yield q.wait()
finished += 1
class Pool(object):
def __init__(self, min_size=0, max_size=4, track_events=False):
if min_size > max_size:
raise ValueError('min_size cannot be bigger than max_size')
self.max_size = max_size
self.sem = Semaphore(max_size)
self.procs = proc.RunningProcSet()
if track_events:
self.results = coros.queue()
else:
self.results = None
def resize(self, new_max_size):
""" Change the :attr:`max_size` of the pool.
If the pool gets resized when there are more than *new_max_size*
coroutines checked out, when they are returned to the pool they will be
discarded. The return value of :meth:`free` will be negative in this
situation.
"""
max_size_delta = new_max_size - self.max_size
self.sem.counter += max_size_delta
self.max_size = new_max_size
@property
def current_size(self):
""" The number of coroutines that are currently executing jobs. """
return len(self.procs)
def free(self):
""" Returns the number of coroutines that are available for doing
work."""
return self.sem.counter
def execute(self, func, *args, **kwargs):
"""Execute func in one of the coroutines maintained
by the pool, when one is free.
Immediately returns a :class:`~eventlet.proc.Proc` object which can be
queried for the func's result.
>>> pool = Pool()
>>> task = pool.execute(lambda a: ('foo', a), 1)
>>> task.wait()
('foo', 1)
"""
# if reentering an empty pool, don't try to wait on a coroutine freeing
# itself -- instead, just execute in the current coroutine
if self.sem.locked() and api.getcurrent() in self.procs:
p = proc.spawn(func, *args, **kwargs)
try:
p.wait()
except:
pass
else:
self.sem.acquire()
p = self.procs.spawn(func, *args, **kwargs)
# assuming the above line cannot raise
p.link(lambda p: self.sem.release())
if self.results is not None:
p.link(self.results)
return p
execute_async = execute
def _execute(self, evt, func, args, kw):
p = self.execute(func, *args, **kw)
p.link(evt)
return p
def waitall(self):
""" Calling this function blocks until every coroutine
completes its work (i.e. there are 0 running coroutines)."""
return self.procs.waitall()
wait_all = waitall
def wait(self):
"""Wait for the next execute in the pool to complete,
and return the result."""
return self.results.wait()
def waiting(self):
"""Return the number of coroutines waiting to execute.
"""
if self.sem.balance < 0:
return -self.sem.balance
else:
return 0
def killall(self):
""" Kill every running coroutine as immediately as possible."""
return self.procs.killall()
def launch_all(self, function, iterable):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Discard values returned by ``function()``. You should call
``wait_all()`` to wait for all coroutines, newly-launched plus any
previously-submitted :meth:`execute` or :meth:`execute_async` calls, to
complete.
>>> pool = Pool()
>>> def saw(x):
... print("I saw %s!" % x)
...
>>> pool.launch_all(saw, "ABC")
>>> pool.wait_all()
I saw A!
I saw B!
I saw C!
"""
for tup in iterable:
self.execute(function, *tup)
def process_all(self, function, iterable):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Discard values returned by ``function()``. Don't return until all
coroutines, newly-launched plus any previously-submitted :meth:`execute()`
or :meth:`execute_async` calls, have completed.
>>> from eventlet import coros
>>> pool = coros.CoroutinePool()
>>> def saw(x): print("I saw %s!" % x)
...
>>> pool.process_all(saw, "DEF")
I saw D!
I saw E!
I saw F!
"""
self.launch_all(function, iterable)
self.wait_all()
def generate_results(self, function, iterable, qsize=None):
"""For each tuple (sequence) in *iterable*, launch ``function(*tuple)``
in its own coroutine -- like ``itertools.starmap()``, but in parallel.
Yield each of the values returned by ``function()``, in the order
they're completed rather than the order the coroutines were launched.
Iteration stops when we've yielded results for each arguments tuple in
*iterable*. Unlike :meth:`wait_all` and :meth:`process_all`, this
function does not wait for any previously-submitted :meth:`execute` or
:meth:`execute_async` calls.
Results are temporarily buffered in a queue. If you pass *qsize=*, this
value is used to limit the max size of the queue: an attempt to buffer
too many results will suspend the completed :class:`CoroutinePool`
coroutine until the requesting coroutine (the caller of
:meth:`generate_results`) has retrieved one or more results by calling
this generator-iterator's ``next()``.
If any coroutine raises an uncaught exception, that exception will
propagate to the requesting coroutine via the corresponding ``next()``
call.
What I particularly want these tests to illustrate is that using this
generator function::
for result in generate_results(function, iterable):
# ... do something with result ...
pass
executes coroutines at least as aggressively as the classic eventlet
idiom::
events = [pool.execute(function, *args) for args in iterable]
for event in events:
result = event.wait()
# ... do something with result ...
even without a distinct event object for every arg tuple in *iterable*,
and despite the funny flow control from interleaving launches of new
coroutines with yields of completed coroutines' results.
(The use case that makes this function preferable to the classic idiom
above is when the *iterable*, which may itself be a generator, produces
millions of items.)
>>> from eventlet import coros
>>> import string
>>> pool = coros.CoroutinePool(max_size=5)
>>> pausers = [coros.Event() for x in range(2)]
>>> def longtask(evt, desc):
... print("%s woke up with %s" % (desc, evt.wait()))
...
>>> pool.launch_all(longtask, zip(pausers, "AB"))
>>> def quicktask(desc):
... print("returning %s" % desc)
... return desc
...
(Instead of using a ``for`` loop, step through :meth:`generate_results`
items individually to illustrate timing)
>>> step = iter(pool.generate_results(quicktask, string.ascii_lowercase))
>>> print(step.next())
returning a
returning b
returning c
a
>>> print(step.next())
b
>>> print(step.next())
c
>>> print(step.next())
returning d
returning e
returning f
d
>>> pausers[0].send("A")
>>> print(step.next())
e
>>> print(step.next())
f
>>> print(step.next())
A woke up with A
returning g
returning h
returning i
g
>>> print("".join([step.next() for x in range(3)]))
returning j
returning k
returning l
returning m
hij
>>> pausers[1].send("B")
>>> print("".join([step.next() for x in range(4)]))
B woke up with B
returning n
returning o
returning p
returning q
klmn
"""
# Get an iterator because of our funny nested loop below. Wrap the
# iterable in enumerate() so we count items that come through.
tuples = iter(enumerate(iterable))
# If the iterable is empty, this whole function is a no-op, and we can
# save ourselves some grief by just quitting out. In particular, once
# we enter the outer loop below, we're going to wait on the queue --
# but if we launched no coroutines with that queue as the destination,
# we could end up waiting a very long time.
try:
index, args = tuples.next()
except StopIteration:
return
# From this point forward, 'args' is the current arguments tuple and
# 'index+1' counts how many such tuples we've seen.
# This implementation relies on the fact that _execute() accepts an
# event-like object, and -- unless it's None -- the completed
# coroutine calls send(result). We slyly pass a queue rather than an
# event -- the same queue instance for all coroutines. This is why our
# queue interface intentionally resembles the event interface.
q = coros.queue(max_size=qsize)
# How many results have we yielded so far?
finished = 0
# This first loop is only until we've launched all the coroutines. Its
# complexity is because if iterable contains more args tuples than the
# size of our pool, attempting to _execute() the (poolsize+1)th
# coroutine would suspend until something completes and send()s its
# result to our queue. But to keep down queue overhead and to maximize
# responsiveness to our caller, we'd rather suspend on reading the
# queue. So we stuff the pool as full as we can, then wait for
# something to finish, then stuff more coroutines into the pool.
try:
while True:
# Before each yield, start as many new coroutines as we can fit.
# (The self.free() test isn't 100% accurate: if we happen to be
# executing in one of the pool's coroutines, we could _execute()
# without waiting even if self.free() reports 0. See _execute().)
# The point is that we don't want to wait in the _execute() call,
# we want to wait in the q.wait() call.
# IMPORTANT: at start, and whenever we've caught up with all
# coroutines we've launched so far, we MUST iterate this inner
# loop at least once, regardless of self.free() -- otherwise the
# q.wait() call below will deadlock!
# Recall that index is the index of the NEXT args tuple that we
# haven't yet launched. Therefore it counts how many args tuples
# we've launched so far.
while self.free() > 0 or finished == index:
# Just like the implementation of execute_async(), save that
# we're passing our queue instead of None as the "event" to
# which to send() the result.
self._execute(q, function, args, {})
# We've consumed that args tuple, advance to next.
index, args = tuples.next()
# Okay, we've filled up the pool again, yield a result -- which
# will probably wait for a coroutine to complete. Although we do
# have q.ready(), so we could iterate without waiting, we avoid
# that because every yield could involve considerable real time.
# We don't know how long it takes to return from yield, so every
# time we do, take the opportunity to stuff more requests into the
# pool before yielding again.
yield q.wait()
# Be sure to count results so we know when to stop!
finished += 1
except StopIteration:
pass
# Here we've exhausted the input iterable. index+1 is the total number
# of coroutines we've launched. We probably haven't yielded that many
# results yet. Wait for the rest of the results, yielding them as they
# arrive.
while finished < index + 1:
yield q.wait()
finished += 1
class Coroutine(object):
"""
This class simulates a coroutine, which is ironic, as greenlet actually
*is* a coroutine. But trying to use greenlet here gives nasty results
since eventlet thoroughly monkey-patches things, making it difficult
to run greenlet on its own.
Essentially think of this as a wrapper for eventlet's threads which has a
run and sleep function similar to old school coroutines, meaning it won't
start until told and when asked to sleep it won't wake back up without
permission.
"""
ALL = []
def __init__(self, func, *args, **kwargs):
self.my_sem = Semaphore(0) # This is held by the thread as it runs.
self.caller_sem = None
self.dead = False
started = Event()
self.id = 5
self.ALL.append(self)
def go():
self.id = eventlet.corolocal.get_ident()
started.send(True)
self.my_sem.acquire(blocking=True, timeout=None)
try:
func(*args, **kwargs)
# except Exception as e:
# print("Exception in coroutine! %s" % e)
finally:
self.dead = True
self.caller_sem.release() # Relinquish control back to caller.
for i in range(len(self.ALL)):
if self.ALL[i].id == self.id:
del self.ALL[i]
break
true_spawn(go)
started.wait()
@classmethod
def get_current(cls):
"""Finds the coroutine associated with the thread which calls it."""
return cls.get_by_id(eventlet.corolocal.get_ident())
@classmethod
def get_by_id(cls, id):
for cr in cls.ALL:
if cr.id == id:
return cr
raise RuntimeError("Coroutine with id %s not found!" % id)
def sleep(self):
"""Puts the coroutine to sleep until run is called again.
This should only be called by the thread which owns this object.
"""
# Only call this from its own thread.
assert eventlet.corolocal.get_ident() == self.id
self.caller_sem.release() # Relinquish control back to caller.
self.my_sem.acquire(blocking=True, timeout=None)
def run(self):
"""Starts up the thread. Should be called from a different thread."""
# Don't call this from the thread which it represents.
assert eventlet.corolocal.get_ident() != self.id
self.caller_sem = Semaphore(0)
self.my_sem.release()
self.caller_sem.acquire() # Wait for it to finish.
class TCPSession():
STATE_CLOSED = "CLOSED"
STATE_LISTEN = "LISTEN"
STATE_ESTABLISHED = "ESTABLISHED"
STATE_SYN_SENT = "SYN-SENT"
STATE_SYN_RECEIVED = "SYN-RECEIVED"
STATE_TIME_WAIT = "TIME_WAIT"
STATE_CLOSE_WAIT = "CLOSE_WAIT"
STATE_LAST_ACK = "LAST_ACK"
DIRECTION_INBOUND = "INBOUND"
DIRECTION_OUTBOUND = "OUTBOUND"
EVENT_RETRANSMISSION = "retransmission"
EVENT_TIMEOUT = "timeout"
EVENT_KEEPALIVE = "keepalive"
RETRANSMISSION_TIMER = 1
RETRANSMISSION_TIMER_MULTIPLIER = 1.5 # Use 2 in prod
RETRANSMISSION_RETRIES = 5 # USE 15 in prod
QUIET_TIMER = 5 # Notused
KEEPALIVE_TIMER = 10 # default 60
IDLE_TIMER = 5 # Notused default 30
TIMEOUT_TIMER = 5 # USE 120 in prod
KEEPALIVE_RETRIES = 5
KEEPALIVE_INTERVAL = 3
def __init__(self, datapath, src_ip, dst_ip, src_port, dst_port, seq, direction, in_port=None, src_mac=None, dst_mac=None, tcp_opts=None, pkt=None):
self.datapath = datapath
self.timers = {
'retransmission': None,
'timeout': None,
'keepalive': None,
'keepalive_interval': None,
}
self.src_mac = src_mac
self.dst_mac = dst_mac
self.src_ip = src_ip
self.src_port = src_port
self.dst_ip = dst_ip
self.dst_port = dst_port
self.last_sent_chunk_size = 0
self.sent_acked = False
self.received_acked = False
self.lastRetransmission = self.RETRANSMISSION_TIMER
self.retransmissionRetries = 0
self.inEvent = Semaphore()
self.event = None
self.keepalive_sent = False
self.keepalive_count = 0
if direction == self.DIRECTION_INBOUND:
self.in_port = in_port
self.direction = self.DIRECTION_INBOUND
self.state = self.STATE_LISTEN
self.source_seq = seq
self.dst_seq = self._generate_seq()
self.last_received_seq = seq
self.last_sent_seq = self.dst_seq
self.tcp_opts = tcp_opts
self.initial_pkt = pkt
def __str__(self):
return "%s:%s:%d:%s:%d" % (self.datapath.id, self.src_ip, self.src_port, self.dst_ip, self.dst_port)
def __repr__(self):
return "%s Connection from %s:%d to %s:%d on datapath %d" % (self.direction, self.src_ip, self.src_port, self.dst_ip, self.dst_port, self.datapath.id)
def __eq__(self, other):
return self.__str__() == other.__str__()
def generateSYNACK(self):
pkt = packet.Packet()
t = tcp.tcp(src_port=self.dst_port, dst_port=self.src_port, seq=self.dst_seq, ack=self.source_seq+1, offset=0, bits=(tcp.TCP_SYN | tcp.TCP_ACK), window_size=28960, csum=0, urgent=False,
option=[x for x in self.tcp_opts if type(x) is not tcp.TCPOptionTimestamps])
ip = ipv4.ipv4(version=4, header_length=5, tos=0, total_length=0, identification=0, flags=0, offset=0, ttl=255, proto=6, csum=0, src=self.dst_ip, dst=self.src_ip, option=None)
e = ethernet.ethernet(dst=self.src_mac, src=self.dst_mac, ethertype=ether_types.ETH_TYPE_IP)
pkt.add_protocol(e)
pkt.add_protocol(ip)
pkt.add_protocol(t)
pkt.serialize()
actions = [self.datapath.ofproto_parser.OFPActionOutput(self.in_port, 0)]
ofp = self.datapath.ofproto
ofp_parser = self.datapath.ofproto_parser
res = ofp_parser.OFPPacketOut(datapath=self.datapath, buffer_id=ofp.OFP_NO_BUFFER,
in_port=self.datapath.ofproto.OFPP_CONTROLLER, actions=actions, data=pkt.data)
self.datapath.send_msg(res)
print 'We have sent TCP SYN ACK'
def generateKeepalive(self):
pkt = packet.Packet()
t = tcp.tcp(src_port=self.dst_port, dst_port=self.src_port, seq=self.last_sent_seq + self.last_sent_chunk_size, ack=self.last_received_seq, offset=0,
bits=tcp.TCP_ACK, window_size=28960, csum=0, urgent=False)
ip = ipv4.ipv4(version=4, header_length=5, tos=0, total_length=0, identification=0, flags=0, offset=0, ttl=255,
proto=6, csum=0, src=self.dst_ip, dst=self.src_ip, option=None)
e = ethernet.ethernet(dst=self.src_mac, src=self.dst_mac, ethertype=ether_types.ETH_TYPE_IP)
pkt.add_protocol(e)
pkt.add_protocol(ip)
pkt.add_protocol(t)
pkt.serialize()
actions = [self.datapath.ofproto_parser.OFPActionOutput(self.in_port, 0)]
ofp = self.datapath.ofproto
ofp_parser = self.datapath.ofproto_parser
res = ofp_parser.OFPPacketOut(datapath=self.datapath, buffer_id=ofp.OFP_NO_BUFFER,
in_port=self.datapath.ofproto.OFPP_CONTROLLER, actions=actions, data=pkt.data)
self.datapath.send_msg(res)
print 'We have sent keepalive'
def terminate(self):
# SEND FIN ACK
pkt = packet.Packet()
t = tcp.tcp(src_port=self.dst_port, dst_port=self.src_port, seq=self.last_sent_seq + self.last_sent_chunk_size + 1, ack=self.last_received_seq + 1,
offset=0, bits=(tcp.TCP_ACK|tcp.TCP_FIN), window_size=28960, csum=0, urgent=False)
ip = ipv4.ipv4(version=4, header_length=5, tos=0, total_length=0, identification=0, flags=0, offset=0, ttl=255, proto=6, csum=0, src=self.dst_ip, dst=self.src_ip, option=None)
e = ethernet.ethernet(dst=self.src_mac, src=self.dst_mac, ethertype=ether_types.ETH_TYPE_IP)
pkt.add_protocol(e)
pkt.add_protocol(ip)
pkt.add_protocol(t)
pkt.serialize()
actions = [self.datapath.ofproto_parser.OFPActionOutput(self.in_port, 0)]
ofp = self.datapath.ofproto
ofp_parser = self.datapath.ofproto_parser
res = ofp_parser.OFPPacketOut(datapath=self.datapath, buffer_id=ofp.OFP_NO_BUFFER,
in_port=self.datapath.ofproto.OFPP_CONTROLLER, actions=actions, data=pkt.data)
self.datapath.send_msg(res)
print 'We have sent TCP FIN, ACK'
def reset(self):
# SEND FIN ACK
pkt = packet.Packet()
t = tcp.tcp(src_port=self.dst_port, dst_port=self.src_port, seq=self.last_sent_seq + self.last_sent_chunk_size + 1, ack=self.last_received_seq + 1,
offset=0, bits=(tcp.TCP_RST), window_size=28960, csum=0, urgent=False)
ip = ipv4.ipv4(version=4, header_length=5, tos=0, total_length=0, identification=0, flags=0, offset=0, ttl=255, proto=6, csum=0, src=self.dst_ip, dst=self.src_ip, option=None)
e = ethernet.ethernet(dst=self.src_mac, src=self.dst_mac, ethertype=ether_types.ETH_TYPE_IP)
pkt.add_protocol(e)
pkt.add_protocol(ip)
pkt.add_protocol(t)
pkt.serialize()
actions = [self.datapath.ofproto_parser.OFPActionOutput(self.in_port, 0)]
ofp = self.datapath.ofproto
ofp_parser = self.datapath.ofproto_parser
res = ofp_parser.OFPPacketOut(datapath=self.datapath, buffer_id=ofp.OFP_NO_BUFFER,
in_port=self.datapath.ofproto.OFPP_CONTROLLER, actions=actions, data=pkt.data)
self.datapath.send_msg(res)
print 'We have sent TCP RST to an inconsistent TCP state'
def ack(self):
if self.keepalive_sent:
self.keepalive_sent = False
self.keepalive_count = 0
def handleRetransmission(self):
print 'Retranmsission occured'
if self.inEvent.acquire(timeout=1):
self.event = self.EVENT_RETRANSMISSION
self.retransmissionRetries += 1
self.lastRetransmission *= self.RETRANSMISSION_TIMER_MULTIPLIER
if self.retransmissionRetries > self.RETRANSMISSION_RETRIES:
print 'Reached maximum level of retransmissions, closing TCP connection'
self.setState(self.STATE_CLOSED)
else:
if self.state == self.STATE_SYN_RECEIVED:
self.generateSYNACK()
self.setState(self.state)
self.event = None
self.inEvent.release()
else:
print 'failed to handle retransmission event, this should not happen'
def handleKeepalive(self):
print 'Keepalive occured'
if self.inEvent.acquire(timeout=1):
self.event = self.EVENT_KEEPALIVE
self.generateKeepalive()
self.keepalive_sent = True
self.keepalive_count += 1
if self.keepalive_count > self.KEEPALIVE_RETRIES:
self.setState(self.STATE_CLOSED)
else:
self.setState(self.state)
self.event = None
self.inEvent.release()
else:
print 'Failed to handle keepalive event, this should not happen'
def handleTimeout(self):
print 'Timeout occured, closing connection'
if self.inEvent.acquire(timeout=1):
self.event = self.EVENT_TIMEOUT
self.setState(self.STATE_CLOSED)
self.event = None
self.inEvent.release()
else:
print 'Failed to handle timeout event, this should not happen'
def clearTimers(self):
for key, thr in self.timers.iteritems():
if self.event is not key:
if thr is not None:
self.timers[key].kill()
self.timers[key] = None
def setTimers(self):
if self.direction == self.DIRECTION_INBOUND:
if self.state == self.STATE_SYN_RECEIVED:
thr = eventlet.spawn_after(self.lastRetransmission, self.handleRetransmission)
self.timers['retransmission'] = thr
print 'Setting Retransmission timer'
if self.timers['timeout'] is None:
# if not none, timer is already running
thr = eventlet.spawn_after(self.TIMEOUT_TIMER, self.handleTimeout)
self.timers['timeout'] = thr
print 'Setting Timeout timer'
elif self.state == self.STATE_ESTABLISHED:
if self.keepalive_sent:
thr = eventlet.spawn_after(self.KEEPALIVE_INTERVAL, self.handleKeepalive)
self.timers['keepalive_interval'] = thr
print 'Setting Keepalive interval timer'
else:
self.clearTimers()
thr = eventlet.spawn_after(self.KEEPALIVE_TIMER, self.handleKeepalive)
self.timers['keepalive'] = thr
print 'Setting Keepalive timer'
elif self.state == self.STATE_CLOSED:
self.clearTimers()
def setState(self, state):
print self.__repr__() + 'State transition ' + self.state + ' -> ' + state
self.state = state
self.setTimers()
def handlePacket(self, pkt):
protocol = pkt.get_protocol(tcp.tcp)
if self.direction == self.DIRECTION_INBOUND:
if self.state == self.STATE_LISTEN:
if (protocol.bits & tcp.TCP_SYN):
# S1
# Going to SYN received state
self.generateSYNACK()
self.received_acked = True
self.setState(self.STATE_SYN_RECEIVED)
return
else:
self.reset()
print 'Received packet without active session, ignoring, should return RST'
if self.state == self.STATE_SYN_RECEIVED:
if (protocol.bits & tcp.TCP_FIN):
self.last_received_seq = protocol.seq
self.setState(self.STATE_CLOSE_WAIT)
self.terminate()
elif (protocol.bits & tcp.TCP_RST):
pass
elif (protocol.bits & tcp.TCP_ACK):
# S6
# Going to established state
if protocol.ack == self.last_sent_seq + 1 and protocol.seq == self.source_seq + 1:
self.last_received_seq = protocol.seq
self.sent_acked = True
self.setState(self.STATE_ESTABLISHED)
return
if self.state == self.STATE_ESTABLISHED:
if (protocol.bits & tcp.TCP_FIN):
self.last_received_seq = protocol.seq
self.received_acked = False
self.setState(self.STATE_CLOSE_WAIT)
self.terminate()
self.sent_acked = False
self.setState(self.STATE_LAST_ACK)
elif (protocol.bits & tcp.TCP_RST):
# TODO handle RST
pass
elif (protocol.bits & tcp.TCP_ACK):
print 'Was Acked, probably keepalive'
self.last_received_seq = protocol.seq
self.received_acked = False
self.ack()
self.received_acked = True
self.setState(self.state)
# TODO standard data transfer, might need for the HTTP GET parsing if GET is long
pass
return
if self.state == self.STATE_LAST_ACK and (protocol.bits & tcp.TCP_ACK):
self.clearTimers()
self.setState(self.STATE_CLOSED)
return
@staticmethod
def _generate_seq():
uint32_t_max = np.iinfo(np.uint32)
return random.randint(0, uint32_t_max.max)
class DTestQueue(object):
"""
DTestQueue
==========
The DTestQueue class maintains a queue of tests waiting to be run.
The constructor initializes the queue to an empty state and stores
a maximum simultaneous thread count ``maxth`` (None means
unlimited); a ``skip`` evaluation routine (defaults to testing the
``skip`` attribute of the test); and an instance of DTestOutput.
The list of all tests in the queue is maintained in the ``tests``
attribute; tests may be added to a queue with add_test() (for a
single test) or add_tests() (for a sequence of tests). The tests
in the queue may be run by invoking the run() method.
"""
def __init__(self, maxth=None, skip=lambda dt: dt.skip,
output=DTestOutput()):
"""
Initialize a DTestQueue. The ``maxth`` argument must be
either None or an integer specifying the maximum number of
simultaneous threads permitted. The ``skip`` arguments is
function references; it should take a test and return True if
the test should be skipped. The ``output`` argument should be
an instance of DTestOutput containing a notify() method, which
takes a test and the state to which it is transitioning, and
may use that information to emit a test result. Note that the
notify() method will receive state transitions to the RUNNING
state, as well as state transitions for test fixtures; callers
may find the DTestBase.istest() method useful for
differentiating between regular tests and test fixtures for
reporting purposes.
"""
# Save our maximum thread count
if maxth is None:
self.sem = None
else:
self.sem = Semaphore(maxth)
# Need to remember the skip routine
self.skip = skip
# Also remember the output
self.output = output
# Initialize the lists of tests
self.tests = set()
self.waiting = None
self.runlist = set()
# No initial resource manager...
self.res_mgr = resource.ResourceManager()
# Need locks for the waiting and runlist lists
self.waitlock = Semaphore()
self.runlock = Semaphore()
# Set up some statistics...
self.th_count = 0
self.th_event = Event()
self.th_simul = 0
self.th_max = 0
# Place to keep any exceptions we encounter within dtest
# itself
self.caught = []
# We're not yet running
self.running = False
def add_test(self, tst):
"""
Add a test ``tst`` to the queue. Tests can be added multiple
times, but the test will only be run once.
"""
# Can't add a test if the queue is running
if self.running:
raise DTestException("Cannot add tests to a running queue.")
# First we need to get the test object
dt = test._gettest(tst)
# Add it to the set of tests
self.tests.add(dt)
def add_tests(self, tests):
"""
Add a sequence of tests ``tests`` to the queue. Tests can be
added multiple times, but the test will only be run once.
"""
# Can't add a test if the queue is running
if self.running:
raise DTestException("Cannot add tests to a running queue.")
# Run add_test() in a loop
for tst in tests:
self.add_test(tst)
def dot(self, grname='testdeps'):
"""
Constructs a GraphViz-compatible dependency graph with the
given name (``testdeps``, by default). Returns the graph as a
string. The graph can be fed to the ``dot`` tool to generate
a visualization of the dependency graph. Note that red nodes
in the graph indicate test fixtures, and red dashed edges
indicate dependencies associated with test fixtures. If the
node outline is dotted, that indicates that the test was
skipped in the most recent test run.
"""
# Helper to generate node and edge options
def mkopts(opts):
# If there are no options, return an empty string
if not opts:
return ''
# OK, let's do this...
return ' [' + ','.join(['%s="%s"' % (k, opts[k])
for k in opts]) + ']'
# Now, create the graph
nodes = []
edges = []
for dt in sorted(self.tests, key=lambda dt: str(dt)):
# Get the real test function
tfunc = dt.test
# Make the node
opts = dict(label=r'%s\n%s:%d' %
(dt, tfunc.func_code.co_filename,
tfunc.func_code.co_firstlineno))
if dt.state:
opts['label'] += r'\n(Result: %s)' % dt.state
if (dt.state == FAIL or dt.state == XFAIL or dt.state == ERROR or
dt.state == DEPFAIL):
opts['color'] = 'red'
elif isinstance(dt, test.DTestFixture):
opts['color'] = 'blue'
if dt.state == SKIPPED:
opts['style'] = 'dotted'
elif dt.state == DEPFAIL:
opts['style'] = 'dashed'
nodes.append('"%s"%s;' % (dt, mkopts(opts)))
# Make all the edges
for dep in sorted(dt.dependencies, key=lambda dt: str(dt)):
opts = {}
if (isinstance(dt, test.DTestFixture) or
isinstance(dep, test.DTestFixture)):
opts.update(dict(color='blue', style='dashed'))
if dt._partner is not None and dep == dt._partner:
opts['style'] = 'dotted'
edges.append('"%s" -> "%s"%s;' % (dt, dep, mkopts(opts)))
# Return a graph
return (('strict digraph "%s" {\n\t' % grname) +
'\n\t'.join(nodes) + '\n\n\t' + '\n\t'.join(edges) + '\n}')
def run(self, debug=False):
"""
Runs all tests that have been queued up. Does not return
until all tests have been run. Causes test results and
summary data to be emitted using the ``output`` object
registered when the queue was initialized.
"""
# Can't run an already running queue
if self.running:
raise DTestException("Queue is already running.")
# OK, put ourselves into the running state
self.running = True
# Must begin by ensuring we're monkey-patched
monkey_patch()
# OK, let's prepare all the tests...
for dt in self.tests:
dt._prepare()
# Second pass--determine which tests are being skipped
waiting = []
for dt in self.tests:
# Do we skip this one?
willskip = self.skip(dt)
# If not, check if it's a fixture with no dependencies...
if not willskip and not dt.istest():
if dt._partner is None:
if len(dt._revdeps) == 0:
willskip = True
else:
if len(dt._revdeps) == 1:
willskip = True
# OK, mark it skipped if we're skipping
if willskip:
dt._skipped(self.output)
else:
waiting.append(dt)
# OK, last pass: generate list of waiting tests; have to
# filter out SKIPPED tests
self.waiting = set([dt for dt in self.tests if dt.state != SKIPPED])
# Install the capture proxies...
if not debug:
capture.install()
# Spawn waiting tests
self._spawn(self.waiting)
# Wait for all tests to finish
if self.th_count > 0:
self.th_event.wait()
# OK, uninstall the capture proxies
if not debug:
capture.uninstall()
# Now we go through and clean up all left-over resources
self.res_mgr.release_all()
# Walk through the tests and output the results
cnt = {
OK: 0,
UOK: 0,
SKIPPED: 0,
FAIL: 0,
XFAIL: 0,
ERROR: 0,
DEPFAIL: 0,
'total': 0,
'threads': self.th_max,
}
for t in self.tests:
# Get the result object
r = t.result
# Update the counts
cnt[r.state] += int(r.test)
cnt['total'] += int(r.test)
# Special case update for unexpected OKs and expected failures
if r.state == UOK:
cnt[OK] += int(r.test)
elif r.state == XFAIL:
cnt[FAIL] += int(r.test)
try:
# Emit the result messages
self.output.result(r, debug)
except TypeError:
# Maybe the output object is written to the older
# standard?
self.output.result(r)
# Emit summary data
self.output.summary(cnt)
# If there were resource tearDown exceptions, emit data about
# them
msgs = self.res_mgr.messages
if msgs:
self.output.resources(msgs)
# If we saw exceptions, emit data about them
if self.caught:
self.output.caught(self.caught)
# We're done running; re-running should be legal
self.running = False
# Return False if there were any unexpected OKs, unexpected
# failures, errors, or dependency failures
if (cnt[UOK] > 0 or
(cnt[FAIL] - cnt[XFAIL]) > 0 or
cnt[ERROR] > 0 or cnt[DEPFAIL] > 0 or
len(self.res_mgr.messages) > 0):
return False
# All tests passed!
return True
def _spawn(self, tests):
"""
Selects all ready tests from the set or list specified in
``tests`` and spawns threads to execute them. Note that the
maximum thread count restriction is implemented by having the
thread wait on the ``sem`` Semaphore after being spawned.
"""
# Work with a copy of the tests
tests = list(tests)
# Loop through the list
while tests:
# Pop off a test to consider
dt = tests.pop(0)
with self.waitlock:
# Is test waiting?
if dt not in self.waiting:
continue
# OK, check dependencies
elif dt._depcheck(self.output):
# No longer waiting
self.waiting.remove(dt)
# Place test on the run list
with self.runlock:
self.runlist.add(dt)
# Spawn the test
self.th_count += 1
spawn_n(self._run_test, dt)
# Dependencies failed; check if state changed and add
# its dependents if so
elif dt.state is not None:
# No longer waiting
self.waiting.remove(dt)
# Check all its dependents. Note--not trying to
# remove duplicates, because some formerly
# unrunnable tests may now be runnable because of
# the state change
tests.extend(list(dt.dependents))
def _run_test(self, dt):
"""
Execute ``dt``. This method is meant to be run in a new
thread.
Once a test is complete, the thread's dependents will be
passed back to the spawn() method, in order to pick up and
execute any tests that are now ready for execution.
"""
# Acquire the thread semaphore
if self.sem is not None:
self.sem.acquire()
# Increment the simultaneous thread count
self.th_simul += 1
if self.th_simul > self.th_max:
self.th_max = self.th_simul
# Save the output and test relative to this thread, for the
# status stream
status.setup(self.output, dt)
# Execute the test
try:
dt._run(self.output, self.res_mgr)
except:
# Add the exception to the caught list
self.caught.append(sys.exc_info())
# Manually transition the test to the ERROR state
dt._result._transition(ERROR, output=self.output)
# OK, done running the test; take it off the run list
with self.runlock:
self.runlist.remove(dt)
# Now, walk through its dependents and check readiness
self._spawn(dt.dependents)
# All right, we're done; release the semaphore
if self.sem is not None:
self.sem.release()
# Decrement the thread count
self.th_simul -= 1
self.th_count -= 1
# If thread count is now 0, signal the event
with self.waitlock:
if len(self.waiting) == 0 and self.th_count == 0:
self.th_event.send()
return
# If the run list is empty, that means we have a cycle
with self.runlock:
if len(self.runlist) == 0:
for dt2 in list(self.waiting):
# Manually transition to DEPFAIL
dt2._result._transition(DEPFAIL, output=self.output)
# Emit an error message to let the user know what
# happened
self.output.info("A dependency cycle was discovered. "
"Please examine the dependency graph "
"and correct the cycle. The --dot "
"option may be useful here.")
# Now, let's signal our event
self.th_event.send()