def testLock(self): lock = Lock() self.assertEquals(True, lock.acquire()) self.assertEquals(True, lock.is_locked()) self.assertEquals(None, lock.release()) xs = [] def t(x): try: with lock: Tasklet.sleep(1.0) xs.append(x) return x except TimeoutError: pass start = time.time() for i in range(5): Tasklet.new(t)(i) join_result = Tasklet.join_children() self.assertEquals(5, len(join_result)) self.assertEquals(10, sum(xs)) end = time.time() self.assertAlmostEqual(5.0, end - start, places = 1)
def testMultiTask(self): local = TaskLocal() def t(): local.piet = [] for i in range(10): local.piet.append(i) Tasklet.yield_() self.assertEquals(range(10), local.piet) t1 = Tasklet.new(t)() t2 = Tasklet.new(t)() Tasklet.join_all([t1,t2]) self.assertEquals(2, len(local._d.keys())) #the 2 tasks are sill around, so local keeps their values #check that values are gone from dict #when tasks are gone del t1 del t2 #we need to yield, because our 2 tasks were suspended by the join #yield will run the scheduler again, so our tasks can properly finish #the only strange thing is we need 2 yields for python, stackless requires just 1 Tasklet.yield_() Tasklet.yield_() self.assertEquals([], local._d.keys())
def testRecursive(self): #non-recursive local = TaskLocal() local.piet = 20 def t(): try: local.piet self.fail('expected attr error') except AttributeError: pass Tasklet.join(Tasklet.new(t)()) #recursive local = TaskLocal(True) local.piet = 30 def t(): self.assertEquals(30, local.piet) Tasklet.join(Tasklet.new(t)())
def main(): try: dispatcher = Dispatcher() host = Host(dispatcher, "/dev/ttyUSB0") dispatcher.add_host(host) Tasklet.new(dispatcher.loop)() print "Calibration: %s" % host.calibrate_baudrate() print "Available: %s" % host.available() dev = RadioDevice(dispatcher, 2) print "Protocol version: %s" % dev.protocol_version # Running HTTP server application = django.core.handlers.wsgi.WSGIHandler() def app_wrapper(*args, **kwargs): try: return application(*args, **kwargs) except Exception as e: print e server = WSGIServer(application) server.serve(('localhost', 8080)) while True: print "ADC data: %s, supply voltage: %.2f" % (dev.adc_data(), dev.supply_voltage()) Tasklet.sleep(1) os._exit(0) except Exception as e: logging.exception(e) os._exit(1)
def testDeadlocks(self): def process(cnn, cur, val): try: cur.execute("begin") cur.execute("insert into tbltest (test_id) values (1)") cur.execute("select sleep(2)") cur.execute("update tbltest set test_id=%d" % val) cur.execute("select sleep(2)") cur.execute("commit") return False except dbapi.Error as e: return "deadlock" in str(e).lower() cnn1 = dbapi.connect(host = DB_HOST, user = DB_USER, passwd = DB_PASSWD, db = DB_DB) cur1 = cnn1.cursor() cnn2 = dbapi.connect(host = DB_HOST, user = DB_USER, passwd = DB_PASSWD, db = DB_DB) cur2 = cnn2.cursor() t1 = Tasklet.new(process)(cnn1, cur1, 2) t2 = Tasklet.new(process)(cnn2, cur2, 3) res = Tasklet.join_all([t1, t2]) self.assertTrue(res[0] or res[1], 'At least one of the queries expected to fail due to deadlock (innodb must be used)') # Both connections must survive after error cur1.execute("select 1") cur2.execute("select 2") cur1.close() cnn1.close() cur2.close() cnn2.close()
def __receive(self): # Called in the receiving tasklet reader = self.__stream.reader try: buffer = "" processData = True while True: line = reader.read_line() if line <> '---': buffer += line + "\n" else: try: data = yaml.load(buffer) processData = True if data == 'ping': processData = False self.sendData('pong') except yaml.parser.ParserError: q.logger.log("[SocketTaskHandler] Received bad formatted data: " + str(buffer), 3) else: if processData: q.logger.log("[SocketTaskHandler] Received data: " + str(data), 5) Tasklet.new(self.__messageHandler)(data, self) buffer = "" except EOFError: q.logger.log("[SocketTaskHandler] Client disconnected.", 3) MSG_SOCKET_CLOSE.send(self.__sending_tasklet)() self.__stream.close()
def testMultiClientMultiServer(self): N = 40 * 100 keys = ['test%d' % i for i in range(N)] mc = Memcache() mc.set_servers([((MEMCACHE_IP, 11212), 100), ((MEMCACHE_IP, 11213), 100), ((MEMCACHE_IP, 11214), 100), ((MEMCACHE_IP, 11215), 100)]) with unittest.timer() as tmr: for i in range(N): self.assertEquals(MemcacheResult.STORED, mc.set(keys[i], 'hello world %d' % i)) print 'single client multi server single set keys/sec', tmr.sec(N) stride = 40 def fetcher(): for i in range(0, N, stride): result, values = mc.get_multi(keys[i:i+stride]) self.assertEquals(MemcacheResult.OK, result) self.assertEquals(stride, len(values)) for nr_clients in [2,4,8,16]:#,32,64,128]: with unittest.timer() as tmr: for i in range(nr_clients): Tasklet.new(fetcher)() Tasklet.join_children() print 'multi client (%d), multi server multi get (%d) keys/sec' % (nr_clients, stride), tmr.sec(N * nr_clients)
def main(): disp = Dispatcher() Tasklet.new(disp.loop)() host = Host(disp, "/dev/ttySP1") disp.add_host(host) while True: host.send([0x45])
def main(): options = parse_options() options.dbargs = {'host': options.host, 'port': options.port, 'user': options.user, 'passwd': options.passwd, 'db': options.db} if options.use_pool: options.pool = Pool(client, options.dbargs, options.use_pool) for i in range(options.sessions): session = Session(options) Tasklet.new(session.run)() try: query_count = options.query_count start = time.time() query_count_done = sum(Tasklet.join_children()) end = time.time() print end - start, query_count_done, query_count_done / (end - start) except Exception: logging.exception("must be an error in one of the sessions") quit()
def connect(self, addr): assert self._stream is None, "must not be disconneted before connecting" self._stream = BufferedStream(Socket.connect(addr, Timeout.current())) self._command_queue = Deque() self._response_queue = Deque() self._command_writer_task = Tasklet.new(self._command_writer)() self._response_reader_task = Tasklet.new(self._response_reader)()
def testReceiveIter(self): test_channel = Channel() def sender(): for i in range(10): test_channel.send(i) t = Tasklet.new(sender)() x = [] for i in test_channel.receive_n(10): x.append(i) self.assertEquals(range(10), x) t = Tasklet.new(sender)() try: Tasklet.set_current_timeout(1.0) x = [] for i in test_channel: x.append(i) except TimeoutError: pass finally: Tasklet.set_current_timeout(TIMEOUT_NEVER) self.assertEquals(range(10), x)
def testerrors(self): mc = Memcached(pool=MemcachedPool(size=4)) tasks = [] for i in xrange(0, 100): tasks.append(Tasklet.new(self.error_thread)(mc)) for i in xrange(0, 100): tasks.append(Tasklet.new(self.handled_thread)(mc)) Tasklet.join_all(tasks)
def __init__(self, dispatcher, devname): "Constructor. devname - serial port devname" self.dispatcher = dispatcher self.stream = SerialStream(devname) self._next_pkt_id = 0 self._devname = devname self._sendlock = Lock() self._calibrated = False Tasklet.new(self.auto_calibrate_baudrate)()
def server(): """accepts connections on a socket, and dispatches new tasks for handling the incoming requests""" server_socket = Socket.new() server_socket.bind(('localhost', 8080)) server_socket.listen() while True: client_socket = server_socket.accept() Tasklet.new(handler)(client_socket)
def server(): server_socket = Socket.new() server_socket.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(("127.0.0.1", 8080)) server_socket.listen(128) print >>sys.stderr, "info: listening on: %r" % (server_socket.socket.getsockname(),) while True: client_socket = server_socket.accept() Tasklet.new(handler)(client_socket)
def testJoinResult(self): """test normal join and checks that parent will get child result""" def child(i): return i ch1 = Tasklet.new(child)(1) ch2 = Tasklet.new(child)(2) self.assertEquals(1, Tasklet.join(ch1)) self.assertEquals(2, Tasklet.join(ch2))
def handle(self, socket, application): stream = BufferedStream(socket) #implements http1.1 keep alive handler #there are several concurrent tasks for each connection; #1 for reading requests, 1 or more for handling requests and 1 for writing responses #the current task (the one created to handle the socket connection) is the controller task, #e.g. it coordinates the actions of it's children by message passing control = Tasklet.current() #writes responses back to the client when they are ready: response_writer = Tasklet.new(self.write_responses, name = 'response_writer')(control, stream) #reads requests from clients: request_reader = Tasklet.new(self.read_requests, name = 'request_reader')(control, stream) #typical flow: #1. reader reads in request, sends notification to control (MSG_REQUEST_READ) #2. control starts handler for the request #3. handler works on request and sends notification to control when finished (MSG_REQUEST_HANDLED) #4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE) #5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN) #control wait for msgs to arrive: for msg, (request, response), kwargs in Tasklet.receive(): if msg.match(self.MSG_REQUEST_READ): #we use reque to be able to send the responses back in the correct order later self._reque.start(request) Tasklet.new(self.handle_request, name = 'request_handler')(control, request, application) elif msg.match(self.MSG_REQUEST_HANDLED): #request.environ["wsgi.input"].read(request.environ["wsgi.input"]._n) #we use reque to retire (send out) the responses in the correct order for request, response in self._reque.finish(request, response): self.MSG_WRITE_RESPONSE.send(response_writer)(request, response) elif msg.match(self.MSG_RESPONSE_WRITTEN): if request.version == 'HTTP/1.0': break #no keep-alive support in http 1.0 elif request.get_response_header('Connection') == 'close': break #response indicated to close after response elif request.get_request_header('Connection') == 'close': break #request indicated to close after response elif msg.match(self.MSG_READ_ERROR): break #stop and close the connection elif msg.match(self.MSG_WRITE_ERROR): break #stop and close the connection else: assert False, "unexpected msg in control loop" #kill reader and writer #any outstanding request will continue, but will exit by themselves response_writer.kill() request_reader.kill() #close our side of the socket stream.close()
def run(self): #show stats every second: Tasklet.interval(1.0, self.show, immediate = True)() #dispenses tokens for doing a request to sessions: Tasklet.new(self.dispense)() #start up sessions, and wait till they are finished Tasklet.join_all([Tasklet.new(self.sessions)() for _ in range(self.options.sessions)]) quit()
def testJoinAfterExit(self): def child(i): return i ch1 = Tasklet.new(child)(1) ch2 = Tasklet.new(child)(2) Tasklet.yield_() #make sure children are run self.assertEquals(1, Tasklet.join(ch1)) self.assertEquals(2, Tasklet.join(ch2))
def start(self, client_socket): ''' Start the task: start one tasklet listing for incomming connections. ''' #self.__receiving_tasklet = Tasklet.new(self.__serve)() self.__client_socket = client_socket self.__stream = BufferedStream(self.__client_socket) q.logger.log("[SocketTaskHandler] Client connected.", 3) self.__sending_tasklet = Tasklet.new(self.__send)() self.__receiving_tasklet = Tasklet.new(self.__receive())()
def tasklet(self): "This tasklet is being run from the main application" # Register service inst = self.app().inst int_app = inst.int_app srv = mg.SingleApplicationWebService(self.app(), "realplexor", "realplexor", "rpl") srv.set("webbackend", "realplexor") srv.set("webbackendport", 8088) srv.serve_any_port() int_app.call("cluster.register-service", srv) Tasklet.new(self.loop)()
def testJoinChildren(self): def t(): return 1 for i in range(4): Tasklet.new(t)() self.assertEquals(4, len(Tasklet.current().children())) result = Tasklet.join_children() self.assertEquals([1,1,1,1], result)
def testConnectionManager(self): try: cm = MemcacheConnectionManager() protocol = MemcacheProtocol.create("text") connections = [] def connector(): connections.append(cm.get_connection((MEMCACHE_IP, 11212), protocol)) Tasklet.new(connector)() Tasklet.new(connector)() Tasklet.new(connector)() Tasklet.join_children() Tasklet.new(connector)() Tasklet.join_children() self.assertEquals(4, len(connections)) self.assertEquals(1, len(cm._connections)) finally: cm.close_all()
def testYield(self): l = [] def child(c): for i in range(5): l.append((c, i)) Tasklet.yield_() ch1 = Tasklet.new(child)(1) ch2 = Tasklet.new(child)(2) Tasklet.join_all([ch1, ch2]) self.assertEquals([(1, 0), (2, 0), (1, 1), (2, 1), (1, 2), (2, 2), (1, 3), (2, 3), (1, 4), (2, 4)], l)
def testJoinAll(self): def sub0(): raise Exception("a proper exc") def sub1(): return 1 def sub2(): return 2 def sub3(): raise Exception("test exc") subs = [Tasklet.new(sub)() for sub in [sub0, sub1, sub2, sub3]] results = Tasklet.join_all(subs) self.assertTrue(isinstance(results[0], JoinError)) self.assertTrue(isinstance(results[0].cause, Exception)) self.assertEquals("a proper exc", str(results[0].cause), Exception) self.assertEquals(1, results[1]) self.assertEquals(2, results[2]) self.assertTrue(isinstance(results[3], JoinError)) self.assertTrue(isinstance(results[3].cause, Exception)) self.assertEquals("test exc", str(results[3].cause), Exception)
def query_services(self, service_type, url, timeout=10, call_int=False, delay=0, *args, **kwargs): inst = self.app().inst daemons = inst.int_app.obj(DBCluster, "daemons", silent=True) tasklets = [] for dmnid, dmninfo in daemons.data.items(): services = dmninfo.get("services", {}) for svcid, svcinfo in services.items(): if svcinfo.get("type") != service_type: continue if call_int: found = False for sid, sinfo in services.items(): if sinfo.get("type") != "int": continue found = True svcid = sid svcinfo = sinfo break if not found: continue if "addr" not in svcinfo: continue if "port" not in svcinfo: continue task = Tasklet.new(self.do_query_service_exc) tasklets.append(task) task(svcid, svcinfo, url, timeout, *args, **kwargs) if delay > 0: Tasklet.sleep(delay) Tasklet.join_all(tasklets)
def testMessageCall(self): class MSG_TEST_SUM(Message): pass class MSG_TEST_MAX(Message): pass class MSG_TEST_SLEEP(Message): pass def c(): for msg, args, kwargs in Tasklet.receive(): if msg.match(MSG_TEST_SUM): msg.reply(sum(args)) elif msg.match(MSG_TEST_MAX): msg.reply(max(args)) elif msg.match(MSG_TEST_SLEEP): Tasklet.sleep(args[0]) msg.reply(True) child = Tasklet.new(c)() self.assertEquals(60, MSG_TEST_SUM.call(child)(10, 20, 30)) self.assertEquals(30, MSG_TEST_MAX.call(child)(10, 20, 30)) self.assertEquals(True, MSG_TEST_SLEEP.call(child)(1)) try: MSG_TEST_SLEEP.call(child, timeout = 1)(2) self.fail("expected timeout") except TimeoutError: pass #expected child.kill()
def testMessageSend(self): class MSG_PONG(Message): pass class MSG_PING(Message): pass def c(parent): for msg, args, kwargs in Tasklet.receive(): if msg.match(MSG_PING): self.assertEquals((10, ), args) MSG_PONG.send(parent)(20) parent = Tasklet.current() child = Tasklet.new(c)(parent) i = 0 MSG_PING.send(child)(10) for msg, args, kwargs in Tasklet.receive(): if msg.match(MSG_PONG): self.assertEquals((20, ), args) i += 1 if i > 5: break MSG_PING.send(child)(10) self.assertEquals(6, i) try: start = time.time() for msg, args, kwargs in Tasklet.receive(2.0): self.fail('expected timeout error') except TimeoutError: end = time.time() self.assertAlmostEqual(2.0, end - start, places = 1) child.kill()
def main(): Tasklet.new(connector)() if sys.argv[1] == 'setter': Tasklet.new(setter)() elif sys.argv[1] == 'getter': Tasklet.new(getter)() Tasklet.new(truncator)() else: assert False, sys.argv
def testSemaphore(self): sema = Semaphore(4) self.assertEquals(True, sema.acquire()) self.assertEquals(3, sema.count) self.assertEquals(True, sema.acquire()) self.assertEquals(2, sema.count) self.assertEquals(True, sema.acquire()) self.assertEquals(1, sema.count) self.assertEquals(True, sema.acquire()) self.assertEquals(0, sema.count) self.assertEquals(False, sema.acquire(False)) self.assertEquals(0, sema.count) self.assertEquals(None, sema.release()) self.assertEquals(1, sema.count) self.assertEquals(None, sema.release()) self.assertEquals(2, sema.count) self.assertEquals(None, sema.release()) self.assertEquals(3, sema.count) self.assertEquals(None, sema.release()) self.assertEquals(4, sema.count) self.assertEquals(None, sema.release()) self.assertEquals(5, sema.count) #possible to go beyond initial count... is this ok? sema = Semaphore(4) xs = [] def t(x): try: with sema: Tasklet.sleep(1.0) xs.append(x) return x except TimeoutError: pass start = time.time() for i in range(8): Tasklet.new(t)(i) join_result = Tasklet.join_children() self.assertEquals(8, len(join_result)) self.assertEquals(28, sum(xs)) end = time.time() self.assertAlmostEqual(2.0, end - start, places = 1)
def __call__(self, event): method_name = "event_%s" % event["type"] method = getattr(self.recipient, method_name, None) if method: Tasklet.new(method)(event)
def loop(self): "Infinitely reads input queue, performs requests and returns responses" host_tasks = [] for host in self.hosts: # Run loop task = Tasklet.new(host.loop) host_tasks.append(task) task() task = Tasklet.new(host.ping) host_tasks.append(task) task() try: while True: try: # Fetch and send requests infinitely request = self.queue.retrieve() # Make routing decision host = self.route(request) if host is None: # No route to send this packet request.deliver_exception(DestinationUnreachable) else: # Creating "callback" channel channel = Channel() self.sent_request = (request, channel) try: # Sending request request.send(host) try: # Waiting for response response = channel.receive(2) except TimeoutError: request.deliver_onetime_exception(TimeoutError) if request.any_waiters(): request.failures += 1 self.queue.add(request) except DeviceUnreachable as e: if e.reason == 2: # reason=2 means we received unexpected packet from another host # occasionally. In this case even onetime requests are performed again self.queue.add(request) else: request.failures += 1 args = e.args request.deliver_onetime_exception(e.__class__, *args) if request.any_waiters(): self.queue.add(request) except CommunicationError as e: # Dispatcher loop received unexpected exception" args = e.args request.deliver_exception(e.__class__, *args) else: # Dispatcher loop received valid response. Delivering it to all waiters request.deliver_response(response) finally: self.sent_request = False except Exception as e: logging.exception(e) finally: for task in host_tasks: task.kill()
def core_abort(self): self.call("cluster.terminate-daemon") Tasklet.new(self.core_abort_delayed)() self.call("web.response_json", {"ok": 1})
def executeScript(self, agentguid, actionname, scriptpath, params, executionparams={}, jobguid=None): """ Execute a script on an agent. The action will be executed in a new job that is the newest child of the current job. The new job will inherit the following properties of the job: name, description, userErrormsg, internalErrormsg, maxduration and customerguid. The executionparams passed to the function will override the properties of the job (if provided). @param agentguid: Guid of the agent on which the script will be executed. @type agentguid: guid @param actionname: The name of the action, will filled in, in the job @param actionname: string @param scriptpath: The path of the script, on the server. @type scriptpath: string @param params: Dictionary containing all parameters required to execute this script. @type params: dictionary @param executionParams: Dictionary can following keys: name, description, userErrormsg, internalErrormsg, maxduration, wait, timetostart, priority @type executionParams: dictionary @param jobguid: Optional parameter, can be used to override the current job. @type jobguid: guid @return: Dictionary with action result as result and the action's jobguid: {'result': <result>, 'jobguid': guid} @rtype: dictionary @raise IOError: If the scriptpath can't be read. @raise TimeOutException: If the agent did not respond within the maxduration: the script will be killed and the exception will be raised. @raise AgentNotAvailableException: If the agent is not available when starting the script, the script is not be started. @raise ScriptFailedException: If an exception occurres on the agent while executing the script. """ #SETUP THE JOB AND THE PARAMS currentjobguid = jobguid or Tasklet.current().jobguid if q.workflowengine.jobmanager.isKilled(currentjobguid): raise Exception("Can't create child jobs: the job is killed !") params['jobguid'] = jobguid = q.workflowengine.jobmanager.createJob( currentjobguid, actionname, executionparams, agentguid, params=params) #START A NEW TASKLET FOR THE JOB q.workflowengine.jobmanager.startJob(jobguid) tasklet = Tasklet.new(self.__execute)(Tasklet.current(), jobguid, agentguid, scriptpath, params) #WAIT FOR THE ANSWER (msg, args, kwargs) = Tasklet.receive().next() if msg.match(MSG_ACTION_NOWAIT): return {'jobguid': jobguid, 'result': None} if msg.match(MSG_ACTION_RETURN): return args[0] elif msg.match(MSG_ACTION_EXCEPTION): raise args[0]
def main(): try: setupLogging() dispatcher = HomeAutoDispatcher() Tasklet.new(dispatcher.loop)() dispatcher.open_ports(conf("hardware", "port1", "/dev/ttyS0"), conf("hardware", "port2", "/dev/ttyS1")) # try: # print "ReadROM: %s" % dispatcher.request(MicroLANReadROM(0)) # except MicroLANError as e: # print type(e).__name__ # try: # print "SearchROM: %s" % dispatcher.request(MicroLANSearchROM(0, 0, [0, 0, 0, 0, 0, 0, 0, 0])) # except MicroLANError as e: # print type(e).__name__ def a(): while True: #for i in xrange(0, 3): # try: # print "Line %d ReadROM: %s" % (i + 1, dispatcher.request(MicroLANReadROM(i))) # except MicroLANError as e: # print "Line %d ReadROM: %s" % (i + 1, type(e).__name__) devs = {} for i in xrange(0, 3): try: req = MicroLANListAll(dispatcher, i) devices = req.execute() except MicroLANError as e: print "Line %d SearchROM: %s" % (i + 1, type(e).__name__) else: #print "Line %d SearchROM: %s" % (i + 1, devices) if devices: try: dispatcher.request( MicroLANConvertTemperature(i)) except MicroLANError as e: print "Line %d ConvertTemperature: %s" % ( i + 1, type(e).__name__) else: devs[i] = devices Tasklet.sleep(1) print "---" for i in xrange(0, 3): devices = devs.get(i) if devices: print "Line %d" % (i + 1) for dev in devices: try: print "ReadTemperature of %s: %.1f" % ( dev, dispatcher.request( MicroLANReadTemperature(i, dev))) except MicroLANError as e: print "ReadTemperature of %s: %s" % ( dev, type(e).__name__) Tasklet.sleep(2) def b(): while True: for j in xrange(0, 3): for i in xrange(0, 30): dispatcher.relay_set(i + 1, True) dispatcher.relay_set((i + 15) % 30 + 1, False) Tasklet.sleep(0.05) dispatcher.relay_clear_all() Tasklet.sleep(1) for i in xrange(0, 30): dispatcher.relay_set(i + 1, True) Tasklet.sleep(0.05) Tasklet.sleep(1) for i in xrange(0, 30): dispatcher.relay_set(i + 1, False) Tasklet.sleep(0.05) Tasklet.sleep(1) for j in xrange(0, 5): dispatcher.relay_set_all() Tasklet.sleep(0.3) dispatcher.relay_clear_all() Tasklet.sleep(0.3) Tasklet.new(a)() #Tasklet.new(b)() logic = HomeLogic(dispatcher) hwapi = HomeLogicAPIServer(dispatcher, logic) server = WSGIServer(hwapi) addr = conf('hwserver', 'addr', '127.0.0.1') port = confInt('hwserver', 'port', 8081) server.serve((addr, port)) logging.getLogger("hwserver").info("Serving API at %s:%s", addr, port) while True: Tasklet.sleep(100) except HardwareError as e: print e os._exit(1) except Exception as e: logging.exception(e) os._exit(1)