def test_NoSpaceInBoxExceptionsReachThread(self): """If a threadedcomponent outbox is linked to a size restricted inbox, then the thread can send at most inbox_size+internal_queue_size messages before it receives a noSpaceInBox exception.""" QSIZE = 20 BSIZE = 10 sched = scheduler() t = ThreadedSender(QSIZE, slow=0.05) d = DoesNothingComponent().activate(Scheduler=sched) d.inboxes['inbox'].setSize(BSIZE) d.link((t, "outbox"), (d, "inbox")) s = sched.main() for _ in range(0, 10): next(s) t.activate(Scheduler=sched) try: t.howManyToSend.put(QSIZE + BSIZE + 10) while t.feedback.qsize() == 0: next(s) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == QSIZE + BSIZE) except: t.howManyToSend.put("STOP") raise
def test_addOutbox(self): """addOutbox - adds a new outbox with the specified name. Component can then send to that inbox.""" class T(threadedadaptivecommscomponent): def __init__(self, dst): super(T, self).__init__() self.toTestCase = queue.Queue() self.fromTestCase = queue.Queue() self.dst = dst def main(self): try: boxname = self.addOutbox("newbox") self.link((self, boxname), self.dst) msg = self.fromTestCase.get() self.send(msg, boxname) self.toTestCase.put((False, msg)) # except Exception, e: except Exception: e = sys.exc_info()[1] self.toTestCase.put( (True, str(e.__clas__.__name__) + str(e.args))) return class Recv(component): def __init__(self): super(Recv, self).__init__() self.rec = [] def main(self): while 1: yield 1 if self.dataReady("inbox"): self.rec.append(self.recv("inbox")) sched = scheduler() r = Recv().activate(Scheduler=sched) t = T((r, "inbox")).activate(Scheduler=sched) t.fromTestCase.put("hello") while not t.toTestCase.qsize(): next(t) next(t) (err, msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:" + str(msg)) try: next(t) except StopIteration: pass try: next(r) next(r) except StopIteration: pass self.assert_( r.rec == ["hello"], "Data send through outbox corrupted; r.rec = " + str(r.rec))
def test_NoSpaceInBoxExceptionsReachThread(self): """If a threadedcomponent outbox is linked to a size restricted inbox, then the thread can send at most inbox_size+internal_queue_size messages before it receives a noSpaceInBox exception.""" QSIZE=20 BSIZE=10 sched=scheduler() t=ThreadedSender(QSIZE,slow=0.05) d=DoesNothingComponent().activate(Scheduler=sched) d.inboxes['inbox'].setSize(BSIZE) d.link((t,"outbox"),(d,"inbox")) s=sched.main() for _ in range(0,10): next(s) t.activate(Scheduler=sched) try: t.howManyToSend.put(QSIZE+BSIZE+10) while t.feedback.qsize() == 0: next(s) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == QSIZE+BSIZE) except: t.howManyToSend.put("STOP") raise
def verifyDelivery(self, execute, src, target, chainlength): waitcycles=10 visited = [] (comp,boxname) = src if boxname in comp.inboxes: # skip if an inbox (can't send from an inbox!) boxtype="inbox" elif boxname in comp.outboxes: boxtype="outbox" data = object() comp.send(data, boxname) try: for _ in range(0,waitcycles): next(execute) except StopIteration: self.fail("Scheduler terminated unexpectedly") (tcomp,tboxname) = target self.failUnless( tboxname in tcomp.inboxes, "ERROR IN TEST - destination box not an inbox" ) self.assert_( tcomp.dataReady(tboxname), "something delivered from "+comp.name+" '"+boxname+"' outbox to "+tcomp.name+" '"+tboxname+"' inbox" ) self.assert_( data == tcomp.recv(tboxname), "correct data delivered from "+comp.name+" '"+boxname+"' outbox to "+tcomp.name+" '"+tboxname+"' inbox" ) self.failIf( (comp,boxname,boxtype) in visited, "Loop detected in linkage chain!") visited.append((comp,boxname,boxtype)) else: self.fail("ERROR IN TEST - invalid boxname")
def test_flow_out(self): """main() - can send data to the component's outbox(es) using the standard send() method.""" class ThreadedSender(threadedcomponent): def __init__(self, msg): super(ThreadedSender, self).__init__() self.msg = msg def main(self): self.send(self.msg) msg = "hello there!" sched = scheduler() t = ThreadedSender(msg).activate(Scheduler=sched) r = RecvFrom((t, "outbox")).activate(Scheduler=sched) for i in range(10): time.sleep(0.1) try: next(t) except StopIteration: pass try: next(r) except StopIteration: pass self.assert_(r.rec == [msg])
def nextLine(lines,filterFuncs=(comment)): """Return next line, filtering out undesirable lines, eg. comments""" data = next(lines) if data: while applyFuncs(data,filterFuncs): data = next(lines) if not data: break return data
def nextLine(lines, filterFuncs=(comment)): """Return next line, filtering out undesirable lines, eg. comments""" data = next(lines) if data: while applyFuncs(data, filterFuncs): data = next(lines) if not data: break return data
def test_main_closedowntest(self): """main - This ensures that the closeDownComponent method is called at the end of the loop. It also repeats the above test.""" t=TestMainLoopComponentClosedown() m=t.main() self.failUnless(next(m)==1) for x in vrange(1,1000): self.failUnless(next(m)==x, "Failed when x = " + str(x)) self.failUnlessRaises(closeDownCompTestException , lambda: next(m))#Ensures that the closeDownComponent method has been called. self.failUnlessRaises(StopIteration, lambda: next(m))#, "Checks the generator has finished.")
def test_main_smokeTest(self): """main - Returns a generator that implements the documented behaviour of a highly simplistic approach component statemachine. First value returned is always 1 then the return values are those from the component's main method unitil it returns a False value.""" t=TestMainLoopComponent() m=t.main() self.failUnless(next(m)==1) for x in vrange(1,1000): self.failUnless(next(m)==x, "Failed when x = " + str(x)) self.failUnless(next(m)==1,"After the main method returns a false value the result of closeDownComponent is returned. Stub of 1 assumed.") self.failUnlessRaises(StopIteration, lambda : next(m))#, "Checks the generator has finished.")
def test_main_closedowntest(self): """main - This ensures that the closeDownComponent method is called at the end of the loop. It also repeats the above test.""" t = TestMainLoopComponentClosedown() m = t.main() self.failUnless(next(m) == 1) for x in vrange(1, 1000): self.failUnless(next(m) == x, "Failed when x = " + str(x)) self.failUnlessRaises(closeDownCompTestException, lambda: next( m)) #Ensures that the closeDownComponent method has been called. self.failUnlessRaises( StopIteration, lambda: next(m)) #, "Checks the generator has finished.")
def test_threadisseparate(self): """main() -runs in a separate thread of execution""" class Test(threadedcomponent): def __init__(self): super(Test,self).__init__() self.threadid = queue.Queue() def main(self): self.threadid.put( thread.get_ident() ) sched=scheduler() t=Test().activate(Scheduler=sched) next(t) # get the thread started self.assert_(t.threadid.get() != thread.get_ident(), "main() returns a different value for thread.get_ident()")
def test_addOutbox(self): """addOutbox - adds a new outbox with the specified name. Component can then send to that inbox.""" class T(threadedadaptivecommscomponent): def __init__(self,dst): super(T,self).__init__() self.toTestCase = queue.Queue() self.fromTestCase = queue.Queue() self.dst = dst def main(self): try: boxname=self.addOutbox("newbox") self.link( (self,boxname), self.dst ) msg = self.fromTestCase.get() self.send(msg,boxname) self.toTestCase.put( (False, msg) ) # except Exception, e: except Exception: e = sys.exc_info()[1] self.toTestCase.put( (True, str(e.__clas__.__name__) + str(e.args)) ) return class Recv(component): def __init__(self): super(Recv,self).__init__() self.rec = [] def main(self): while 1: yield 1 if self.dataReady("inbox"): self.rec.append(self.recv("inbox")) sched=scheduler() r=Recv().activate(Scheduler=sched) t=T( (r,"inbox") ).activate(Scheduler=sched) t.fromTestCase.put("hello") while not t.toTestCase.qsize(): next(t) next(t) (err,msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:"+str(msg)) try: next(t) except StopIteration: pass try: next(r) next(r) except StopIteration: pass self.assert_(r.rec == ["hello"], "Data send through outbox corrupted; r.rec = "+str(r.rec))
def test_localprocessterminatesOnlyIfOutqueueFlushed(self): """threadedcomponent ensures that if the thread terminates, any messages still pending in outqueues (waiting to be sent out of outboxes) get sent, even if it is held up for a while by noSpaceInBox exceptions""" class Test(threadedcomponent): def __init__(self): super(Test, self).__init__(queuelengths=5) def main(self): self.count = 0 while 1: try: self.send(object(), "outbox") self.count = self.count + 1 except noSpaceInBox: # outqueue is clearly full now, so lets terminate quick! return sched = scheduler() t = Test() r = DoesNothingComponent() r.link((t, "outbox"), (r, "inbox")) r.inboxes["inbox"].setSize(1) r.activate(Scheduler=sched) t.activate(Scheduler=sched) s = sched.main() for n in range(0, 50): time.sleep(0.05) next(s) self.assert_(not t._isStopped(), "Thread component should not have finished yet") self.assert_(r.dataReady("inbox"), "Should be data waiting at the receiver's inbox") # now relax the inbox size restriction and start receiving items r.inboxes["inbox"].setSize(999) r.recv("inbox") count = 1 for _ in range(0, 50): time.sleep(0.05) next(s) # should expect to have received all t.count items sent while r.dataReady("inbox"): count = count + 1 r.recv("inbox") self.assert_(count == t.count) self.assert_(t._isStopped(), "Thread component should have finished by now")
def test_main_smokeTest(self): """main - Returns a generator that implements the documented behaviour of a highly simplistic approach component statemachine. First value returned is always 1 then the return values are those from the component's main method unitil it returns a False value.""" t = TestMainLoopComponent() m = t.main() self.failUnless(next(m) == 1) for x in vrange(1, 1000): self.failUnless(next(m) == x, "Failed when x = " + str(x)) self.failUnless( next(m) == 1, "After the main method returns a false value the result of closeDownComponent is returned. Stub of 1 assumed." ) self.failUnlessRaises( StopIteration, lambda: next(m)) #, "Checks the generator has finished.")
def test_threadisseparate(self): """main() -runs in a separate thread of execution""" class Test(threadedcomponent): def __init__(self): super(Test, self).__init__() self.threadid = queue.Queue() def main(self): self.threadid.put(thread.get_ident()) sched = scheduler() t = Test().activate(Scheduler=sched) next(t) # get the thread started self.assert_( t.threadid.get() != thread.get_ident(), "main() returns a different value for thread.get_ident()")
def _microprocessGenerator(self,someobject, mainmethod="main"): """\ This contains the mainloop for a microprocess, returning a generator object. Creates the thread of control by calling the class's main method, then in a loop repeatedly calls the resulting generator's next method providing the object with time slices. After each time slice, the _microprocessGenerator yields control back to its caller. Keyword arguments: - someobject -- the object containing the main method (usually 'self') - mainmethod -- *name* of the method that is the generator to be run as the thread. """ pc = someobject.__getattribute__(mainmethod)() while(1): # Continually try to run the code, and then release control if someobject._isStopped(): # Microprocess has stopped yield None return else: # v = pc.next() # python 2 v = next(pc) yield v # Yield control back - making us into a generator function
def _microprocessGenerator(self, someobject, mainmethod="main"): """\ This contains the mainloop for a microprocess, returning a generator object. Creates the thread of control by calling the class's main method, then in a loop repeatedly calls the resulting generator's next method providing the object with time slices. After each time slice, the _microprocessGenerator yields control back to its caller. Keyword arguments: - someobject -- the object containing the main method (usually 'self') - mainmethod -- *name* of the method that is the generator to be run as the thread. """ pc = someobject.__getattribute__(mainmethod)() while (1): # Continually try to run the code, and then release control if someobject._isStopped(): # Microprocess has stopped yield None return else: # v = pc.next() # python 2 v = next(pc) yield v # Yield control back - making us into a generator function
def test_localprocessterminatesOnlyIfOutqueueFlushed(self): """threadedcomponent ensures that if the thread terminates, any messages still pending in outqueues (waiting to be sent out of outboxes) get sent, even if it is held up for a while by noSpaceInBox exceptions""" class Test(threadedcomponent): def __init__(self): super(Test,self).__init__(queuelengths=5) def main(self): self.count=0 while 1: try: self.send(object(),"outbox") self.count=self.count+1 except noSpaceInBox: # outqueue is clearly full now, so lets terminate quick! return sched=scheduler() t=Test() r=DoesNothingComponent() r.link((t,"outbox"),(r,"inbox")) r.inboxes["inbox"].setSize(1) r.activate(Scheduler=sched) t.activate(Scheduler=sched) s=sched.main() for n in range(0,50): time.sleep(0.05) next(s) self.assert_(not t._isStopped(), "Thread component should not have finished yet") self.assert_(r.dataReady("inbox"), "Should be data waiting at the receiver's inbox") # now relax the inbox size restriction and start receiving items r.inboxes["inbox"].setSize(999) r.recv("inbox") count=1 for _ in range(0,50): time.sleep(0.05) next(s) # should expect to have received all t.count items sent while r.dataReady("inbox"): count=count+1 r.recv("inbox") self.assert_(count==t.count) self.assert_(t._isStopped(), "Thread component should have finished by now")
def _newOutboxName(self, name="outbox"): """\ Allocates a new outbox name *based on* the name provided. If this name is available it will be returned unchanged. Otherwise the name will be returned with a number appended """ while name in self.outboxes: name =name+str(next(idGen.idGen())) return name
def _newOutboxName(self, name="outbox"): """\ Allocates a new outbox name *based on* the name provided. If this name is available it will be returned unchanged. Otherwise the name will be returned with a number appended """ while name in self.outboxes: name = name + str(next(idGen.idGen())) return name
def next(self): """\ Calls next() of the internal generator - lets you drop a microprocess in somewhere where you'd ordinarily stick a generator. Internally this calls self.__thread.next() to pass the timeslice down to the actual generator """ # return self.__thread.next() return next(self.__thread) # Python 3 idiom (via helper in python2)
def test_RestrictedInboxSize(self): """Setting the inbox size means at most inbox_size+internal_queue_size messages can queue up before the sender receives a noSpaceInBox exception""" QSIZE = 20 BSIZE = 10 sched = scheduler() t = DoesNothingThread(QSIZE).activate(Scheduler=sched) t.inboxes['inbox'].setSize(BSIZE) d = DoesNothingComponent().activate(Scheduler=sched) d.link((d, "outbox"), (t, "inbox")) s = sched.main() for _ in range(0, 10): next(s) for _ in range(QSIZE + BSIZE): d.send(object(), "outbox") for __ in range(0, 10): next(s) self.failUnlessRaises(noSpaceInBox, d.send, object(), "outbox")
def test_RestrictedInboxSize(self): """Setting the inbox size means at most inbox_size+internal_queue_size messages can queue up before the sender receives a noSpaceInBox exception""" QSIZE=20 BSIZE=10 sched=scheduler() t=DoesNothingThread(QSIZE).activate(Scheduler=sched) t.inboxes['inbox'].setSize(BSIZE) d=DoesNothingComponent().activate(Scheduler=sched) d.link((d,"outbox"),(t,"inbox")) s=sched.main() for _ in range(0,10): next(s) for _ in range(QSIZE+BSIZE): d.send(object(),"outbox") for __ in range(0,10): next(s) self.failUnlessRaises(noSpaceInBox, d.send, object(), "outbox")
def test_addInbox(self): """addInbox - adds a new inbox with the specified name. Component can then receive from that inbox.""" class T(threadedadaptivecommscomponent): def __init__(self): super(T, self).__init__() self.toTestCase = queue.Queue() self.fromTestCase = queue.Queue() def main(self): try: boxname = self.addInbox("newbox") self.toTestCase.put((False, boxname)) self.fromTestCase.get() if not self.dataReady(boxname): self.toTestCase.put( (True, "Data should have been ready at the new inbox")) return self.toTestCase.put((False, self.recv(boxname))) # except Exception, e: except Exception: e = sys.exc_info()[1] self.toTestCase.put( (True, str(e.__clas__.__name__) + str(e.args))) return sched = scheduler() t = T().activate(Scheduler=sched) timeout = 10 next(t) while t.toTestCase.empty(): next(t) timeout = timeout - 1 time.sleep(0.05) self.assert_(timeout, "timed out") (err, msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:" + str(msg)) boxname = msg t._deliver("hello", boxname) try: next(t) next(t) except StopIteration: pass t.fromTestCase.put(1) (err, msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:" + str(msg)) self.assert_(msg == "hello", "Data send through inbox corrupted, received:" + str(msg))
def test_ThereIsADefaultOutgoingQueueSize(self): """There is a default limit on the number of messages that can queue up waiting to be sent out by the main thread.""" sched = scheduler() t = ThreadedSender().activate(Scheduler=sched) try: s = sched.main() for _ in range(0, 10): next(s) t.howManyToSend.put(99999) while t.feedback.qsize() == 0: time.sleep(0.1) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result > 0) except: t.howManyToSend.put("STOP") raise
def test_ThereIsADefaultOutgoingQueueSize(self): """There is a default limit on the number of messages that can queue up waiting to be sent out by the main thread.""" sched=scheduler() t=ThreadedSender().activate(Scheduler=sched) try: s=sched.main() for _ in range(0,10): next(s) t.howManyToSend.put(99999) while t.feedback.qsize() == 0: time.sleep(0.1) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result > 0) except: t.howManyToSend.put("STOP") raise
def test_CanSetOutgoingQueueSize(self): """Setting the queue size in the initializer limits the number of messages that can queue up waiting to be sent out by the main thread.""" QSIZE = 20 sched = scheduler() t = ThreadedSender(QSIZE).activate(Scheduler=sched) try: s = sched.main() for _ in range(0, 10): next(s) t.howManyToSend.put(99999) while t.feedback.qsize() == 0: time.sleep(0.1) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == QSIZE) except: t.howManyToSend.put("STOP") raise
def test_CanSetOutgoingQueueSize(self): """Setting the queue size in the initializer limits the number of messages that can queue up waiting to be sent out by the main thread.""" QSIZE=20 sched=scheduler() t=ThreadedSender(QSIZE).activate(Scheduler=sched) try: s=sched.main() for _ in range(0,10): next(s) t.howManyToSend.put(99999) while t.feedback.qsize() == 0: time.sleep(0.1) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result==QSIZE) except: t.howManyToSend.put("STOP") raise
def test_flow_out(self): """main() - can send data to the component's outbox(es) using the standard send() method.""" class ThreadedSender(threadedcomponent): def __init__(self,msg): super(ThreadedSender,self).__init__() self.msg=msg def main(self): self.send(self.msg) msg="hello there!" sched = scheduler() t = ThreadedSender(msg).activate(Scheduler=sched) r = RecvFrom( (t,"outbox") ).activate(Scheduler=sched) for i in range(10): time.sleep(0.1) try: next(t) except StopIteration: pass try: next(r) except StopIteration: pass self.assert_(r.rec == [msg])
def test_addInbox(self): """addInbox - adds a new inbox with the specified name. Component can then receive from that inbox.""" class T(threadedadaptivecommscomponent): def __init__(self): super(T,self).__init__() self.toTestCase = queue.Queue() self.fromTestCase = queue.Queue() def main(self): try: boxname=self.addInbox("newbox") self.toTestCase.put( (False,boxname) ) self.fromTestCase.get() if not self.dataReady(boxname): self.toTestCase.put( (True,"Data should have been ready at the new inbox") ) return self.toTestCase.put( (False,self.recv(boxname)) ) # except Exception, e: except Exception: e = sys.exc_info()[1] self.toTestCase.put( (True, str(e.__clas__.__name__) + str(e.args)) ) return sched=scheduler() t=T().activate(Scheduler=sched) timeout=10 next(t) while t.toTestCase.empty(): next(t) timeout=timeout-1 time.sleep(0.05) self.assert_(timeout,"timed out") (err,msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:"+str(msg)) boxname=msg t._deliver("hello",boxname) try: next(t) next(t) except StopIteration: pass t.fromTestCase.put(1) (err,msg) = t.toTestCase.get() self.assert_(not err, "Error in thread:"+str(msg)) self.assert_(msg=="hello", "Data send through inbox corrupted, received:"+str(msg))
def test_flow_in(self): """main() - can receive data sent to the component's inbox(es) using the standard dataReady() and recv() methods.""" class ThreadedReceiver(threadedcomponent): def __init__(self): super(ThreadedReceiver,self).__init__() self.rec = [] def main(self): while 1: if self.dataReady("inbox"): self.rec.append(self.recv("inbox")) sched=scheduler() r = ThreadedReceiver().activate(Scheduler=sched) msg = "hello!" o=OneShotTo( (r,"inbox"), msg).activate(Scheduler=sched) next(r) next(o) next(r) time.sleep(0.1) next(r) self.assert_(r.rec==[msg])
def test_flow_in(self): """main() - can receive data sent to the component's inbox(es) using the standard dataReady() and recv() methods.""" class ThreadedReceiver(threadedcomponent): def __init__(self): super(ThreadedReceiver, self).__init__() self.rec = [] def main(self): while 1: if self.dataReady("inbox"): self.rec.append(self.recv("inbox")) sched = scheduler() r = ThreadedReceiver().activate(Scheduler=sched) msg = "hello!" o = OneShotTo((r, "inbox"), msg).activate(Scheduler=sched) next(r) next(o) next(r) time.sleep(0.1) next(r) self.assert_(r.rec == [msg])
def test_linksafe(self): """link() unlink() - thread safe when called. The postoffice link() and unlink() methods are not expected to be capable of re-entrant use.""" class ThreadedLinker(threadedcomponent): def main(self): for i in range(10): linkage = self.link( (self,"outbox"),(self,"inbox") ) self.unlink(thelinkage=linkage) sched=scheduler() t=ThreadedLinker().activate(Scheduler=sched) oldlink = t.postoffice.link oldunlink = t.postoffice.unlink safetycheck = threading.RLock() # re-entrancy permitting mutex failures = queue.Queue() def link_mock(*argL,**argD): # wrapper for postoffice.link() method if not safetycheck.acquire(False): # returns False if should block (meaning its not thread safe!) failures.put(".link()") return False else: result = oldlink(*argL,**argD) time.sleep(0.05) safetycheck.release() return result def unlink_mock(*argL,**argD): if not safetycheck.acquire(False): # returns False if should block (meaning its not thread safe!) failures.put(".unlink()") return False else: result = oldunlink(*argL,**argD) time.sleep(0.05) safetycheck.release() return result t.postoffice.link = link_mock t.postoffice.unlink = unlink_mock done=False for i in range(10): try: next(t) except StopIteration: done=True linkage = t.link( (t,"signal"),(t,"control") ) t.unlink(thelinkage=linkage) while not done: try: next(t) except StopIteration: done=True if failures.qsize(): failed = {} while failures.qsize(): failed[failures.get()] = 1 conj="" errmsg="threadedcomponent,postoffice" for method in failed.keys(): errmsg=errmsg+conj+method conj=" and " errmsg=errmsg+" should not be entered by more than one thread at once." self.fail(errmsg)
def runForAWhile(self, cycles=100): for _ in range(0,cycles): next(self.schedthread)
def main(self, slowmo=0, canblock=False): """\ main([slowmo][,canblock]) - Scheduler main loop generator Each cycle through this generator does two things: * one pass through all active microprocesses, giving executing them. * processing of wake/sleep requests You can optionally slow down execution to aid debugging. You can also allow the scheduler to block if there are no active, awake microprocesses. Keyword arguments: - slowmo -- slow down execution by waiting this number of seconds each cycle (default=0) - canblock -- if True, then will block (waiting for wake requests) if all microprocesses are sleeping (default=False) slowmo specifies a delay (in seconds) before the main loop is run. slowmo defaults to 0. If canblock is True, this generator will briefly) block if there are no active microprocesses, otherwise it will return immediately (default). This generator terminates when there are no microprocesses left (either sleeping or awake) because they've all terminated. (or because there were none to begin with!) """ nextrunqueue = [] running = True exception_caught = self.exception_caught while running: # slowmo now = time.time() until = now + slowmo if canblock: time.sleep(until - now) else: while now < until: yield 1 now = time.time() self.time = now # set "time" attribute for benefit for microprocesses runqueue = nextrunqueue nextrunqueue = [] # run microprocesses in the runqueue # if self.debuggingon: # print("-->", [ x.name for x in self.threads], [ x.name for x in runqueue]) for mprocess in runqueue: # if self.debuggingon: # print("Before Run", mprocess) yield 1 if self.threads[mprocess] == _ACTIVE: try: # result = mprocess.next() result = next(mprocess) if isinstance(result, newComponent): for c in result.components(): c.activate() if isinstance(result, WaitComplete): tag = result.argd.get("tag", "") if tag == "": tag = "__" + mprocess.name[ -10:] # So we have a clue of parentage(!) newThread = microprocess(result.args[0], reactivate(mprocess), tag=tag) newThread.activate() del self.threads[mprocess] mprocess = None # if self.debuggingon: # print("After Run", mprocess) if mprocess: nextrunqueue.append(mprocess) except exception_caught: del self.threads[mprocess] mprocess.stop() knockon = mprocess._closeDownMicroprocess() self.handleMicroprocessShutdownKnockon(knockon) else: # state is _GOINGTOSLEEP or _SLEEPING # so should *not* execute this one and leave it out of the # next run queue self.threads[mprocess] = _SLEEPING # make sure, even if there weren't any micprocesses active, we yield # control at least once yield 1 # process pause requests first - to prevent deadlock, we do # wakeup requests second - safer to leave a thread awake than asleep while not self.pauseRequests.empty(): mprocess = self.pauseRequests.get() # only sleep if we're actually in the set of threads(!) # otherwise it inadvertently gets added! # if self.threads.has_key(mprocess): if mprocess in self.threads: self.threads[mprocess] = _GOINGTOSLEEP # marked as going to sleep, rather than asleep since mprocess # is still in runqueue (more efficient to leave it to be # removed when we iterate through the runqueue) allsleeping = len(self.threads) > 0 and len(nextrunqueue) == 0 while (allsleeping and canblock) or not self.wakeRequests.empty(): # process requests to wake threads try: # wait for wakeup requests, blocks but with a # modest timeout so we still regularly yield (in case this # is a microprocess running in another scheduler) mprocess, canActivate = self.wakeRequests.get(True, 0.01) self.extra = 0 # Fix for race hazard regarding wait_for_one, esp problem with threaded components try: currentstate = self.threads[mprocess] if currentstate == _SLEEPING: nextrunqueue.append(mprocess) allsleeping = False self.threads[mprocess] = _ACTIVE except KeyError: # not activated, can we? if canActivate: nextrunqueue.append(mprocess) self.threads[mprocess] = _ACTIVE allsleeping = False except queue.Empty: # catch timeout pass if not self.stopRequests.empty(): # print("Do we get here? 1") break if not self.stopRequests.empty(): # print("Do we get here? 2") break # print("len(self.threads), wakeRequests" , len(self.threads), self.wakeRequests) running = len(self.threads) + self.extra if not self.stopRequests.empty(): # print("WE GOT HERE! :-)") for X in self.threads: # print("We now call .stop() on ", X.name, type(X)) X.stop()
def test_linksafe(self): """link() unlink() - thread safe when called. The postoffice link() and unlink() methods are not expected to be capable of re-entrant use.""" class ThreadedLinker(threadedcomponent): def main(self): for i in range(10): linkage = self.link((self, "outbox"), (self, "inbox")) self.unlink(thelinkage=linkage) sched = scheduler() t = ThreadedLinker().activate(Scheduler=sched) oldlink = t.postoffice.link oldunlink = t.postoffice.unlink safetycheck = threading.RLock() # re-entrancy permitting mutex failures = queue.Queue() def link_mock(*argL, **argD): # wrapper for postoffice.link() method if not safetycheck.acquire( False ): # returns False if should block (meaning its not thread safe!) failures.put(".link()") return False else: result = oldlink(*argL, **argD) time.sleep(0.05) safetycheck.release() return result def unlink_mock(*argL, **argD): if not safetycheck.acquire( False ): # returns False if should block (meaning its not thread safe!) failures.put(".unlink()") return False else: result = oldunlink(*argL, **argD) time.sleep(0.05) safetycheck.release() return result t.postoffice.link = link_mock t.postoffice.unlink = unlink_mock done = False for i in range(10): try: next(t) except StopIteration: done = True linkage = t.link((t, "signal"), (t, "control")) t.unlink(thelinkage=linkage) while not done: try: next(t) except StopIteration: done = True if failures.qsize(): failed = {} while failures.qsize(): failed[failures.get()] = 1 conj = "" errmsg = "threadedcomponent,postoffice" for method in failed.keys(): errmsg = errmsg + conj + method conj = " and " errmsg = errmsg + " should not be entered by more than one thread at once." self.fail(errmsg)
def test_TakingFromDestinationAllowsMoreToBeDelivered(self): """""" QSIZE=20 BSIZE=10 self.assert_(QSIZE > BSIZE) sched=scheduler() t=ThreadedSender(QSIZE) d=DoesNothingComponent().activate(Scheduler=sched) d.inboxes['inbox'].setSize(BSIZE) d.link((t,"outbox"),(d,"inbox")) s=sched.main() for _ in range(10): next(s) t.activate(Scheduler=sched) try: for _ in range(10): next(s) t.howManyToSend.put(QSIZE+BSIZE+10) # wait for a response, verify it filled its in queue result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == QSIZE) # flush them through to the inbox queue of the destination for _ in range(BSIZE*5): next(s) # let the thread fill the newly free slots t.howManyToSend.put(QSIZE+BSIZE+10) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == BSIZE) # collect messages NUM_COLLECT = 0 while NUM_COLLECT < BSIZE/2: while not d.dataReady("inbox"): next(s) if d.dataReady("inbox"): d.recv("inbox") NUM_COLLECT += 1 # let the main thread flush some message through from the thread for _ in range(50): next(s) t.howManyToSend.put(QSIZE+BSIZE+10) while t.feedback.qsize() == 0: next(s) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == NUM_COLLECT) except: t.howManyToSend.put("STOP") raise
def main(self,slowmo=0,canblock=False): """\ main([slowmo][,canblock]) - Scheduler main loop generator Each cycle through this generator does two things: * one pass through all active microprocesses, giving executing them. * processing of wake/sleep requests You can optionally slow down execution to aid debugging. You can also allow the scheduler to block if there are no active, awake microprocesses. Keyword arguments: - slowmo -- slow down execution by waiting this number of seconds each cycle (default=0) - canblock -- if True, then will block (waiting for wake requests) if all microprocesses are sleeping (default=False) slowmo specifies a delay (in seconds) before the main loop is run. slowmo defaults to 0. If canblock is True, this generator will briefly) block if there are no active microprocesses, otherwise it will return immediately (default). This generator terminates when there are no microprocesses left (either sleeping or awake) because they've all terminated. (or because there were none to begin with!) """ nextrunqueue = [] running = True exception_caught = self.exception_caught while running: # slowmo now = time.time() until = now + slowmo if canblock: time.sleep(until-now) else: while now < until: yield 1 now = time.time() self.time = now # set "time" attribute for benefit for microprocesses runqueue = nextrunqueue nextrunqueue = [] # run microprocesses in the runqueue # if self.debuggingon: # print("-->", [ x.name for x in self.threads], [ x.name for x in runqueue]) for mprocess in runqueue: # if self.debuggingon: # print("Before Run", mprocess) yield 1 if self.threads[mprocess] == _ACTIVE: try: # result = mprocess.next() result = next(mprocess) if isinstance(result, newComponent): for c in result.components(): c.activate() if isinstance(result, WaitComplete): tag = result.argd.get("tag","") if tag == "": tag = "__" + mprocess.name[-10:] # So we have a clue of parentage(!) newThread = microprocess(result.args[0], reactivate(mprocess), tag = tag ) newThread.activate() del self.threads[mprocess] mprocess = None # if self.debuggingon: # print("After Run", mprocess) if mprocess: nextrunqueue.append(mprocess) except exception_caught: del self.threads[mprocess] mprocess.stop() knockon = mprocess._closeDownMicroprocess() self.handleMicroprocessShutdownKnockon(knockon) else: # state is _GOINGTOSLEEP or _SLEEPING # so should *not* execute this one and leave it out of the # next run queue self.threads[mprocess] = _SLEEPING # make sure, even if there weren't any micprocesses active, we yield # control at least once yield 1 # process pause requests first - to prevent deadlock, we do # wakeup requests second - safer to leave a thread awake than asleep while not self.pauseRequests.empty(): mprocess = self.pauseRequests.get() # only sleep if we're actually in the set of threads(!) # otherwise it inadvertently gets added! # if self.threads.has_key(mprocess): if mprocess in self.threads: self.threads[mprocess] = _GOINGTOSLEEP # marked as going to sleep, rather than asleep since mprocess # is still in runqueue (more efficient to leave it to be # removed when we iterate through the runqueue) allsleeping = len(self.threads) > 0 and len(nextrunqueue) == 0 while (allsleeping and canblock) or not self.wakeRequests.empty(): # process requests to wake threads try: # wait for wakeup requests, blocks but with a # modest timeout so we still regularly yield (in case this # is a microprocess running in another scheduler) mprocess, canActivate = self.wakeRequests.get(True,0.01) self.extra = 0 # Fix for race hazard regarding wait_for_one, esp problem with threaded components try: currentstate = self.threads[mprocess] if currentstate == _SLEEPING: nextrunqueue.append(mprocess) allsleeping = False self.threads[mprocess] = _ACTIVE except KeyError: # not activated, can we? if canActivate: nextrunqueue.append(mprocess) self.threads[mprocess] = _ACTIVE allsleeping = False except queue.Empty: # catch timeout pass if not self.stopRequests.empty(): # print("Do we get here? 1") break if not self.stopRequests.empty(): # print("Do we get here? 2") break # print("len(self.threads), wakeRequests" , len(self.threads), self.wakeRequests) running = len(self.threads) + self.extra if not self.stopRequests.empty(): # print("WE GOT HERE! :-)") for X in self.threads: # print("We now call .stop() on ", X.name, type(X)) X.stop()
def runForAWhile(self, cycles=100): for _ in range(0, cycles): next(self.schedthread)
def test_TakingFromDestinationAllowsMoreToBeDelivered(self): """""" QSIZE = 20 BSIZE = 10 self.assert_(QSIZE > BSIZE) sched = scheduler() t = ThreadedSender(QSIZE) d = DoesNothingComponent().activate(Scheduler=sched) d.inboxes['inbox'].setSize(BSIZE) d.link((t, "outbox"), (d, "inbox")) s = sched.main() for _ in range(10): next(s) t.activate(Scheduler=sched) try: for _ in range(10): next(s) t.howManyToSend.put(QSIZE + BSIZE + 10) # wait for a response, verify it filled its in queue result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == QSIZE) # flush them through to the inbox queue of the destination for _ in range(BSIZE * 5): next(s) # let the thread fill the newly free slots t.howManyToSend.put(QSIZE + BSIZE + 10) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == BSIZE) # collect messages NUM_COLLECT = 0 while NUM_COLLECT < BSIZE / 2: while not d.dataReady("inbox"): next(s) if d.dataReady("inbox"): d.recv("inbox") NUM_COLLECT += 1 # let the main thread flush some message through from the thread for _ in range(50): next(s) t.howManyToSend.put(QSIZE + BSIZE + 10) while t.feedback.qsize() == 0: next(s) result = t.feedback.get() self.assert_(result != "ALL SENT") self.assert_(result == NUM_COLLECT) except: t.howManyToSend.put("STOP") raise