コード例 #1
0
ファイル: testcore.py プロジェクト: JoyTeam/concurrence
    def testReceiveIter(self):

        test_channel = Channel()

        def sender():
            for i in range(10):
                test_channel.send(i)

        t = Tasklet.new(sender)()
        x = []
        for i in test_channel.receive_n(10):
            x.append(i)

        self.assertEquals(range(10), x)

        t = Tasklet.new(sender)()



        try:
            Tasklet.set_current_timeout(1.0)
            x = []
            for i in test_channel:
                x.append(i)
        except TimeoutError:
            pass
        finally:
            Tasklet.set_current_timeout(TIMEOUT_NEVER)

        self.assertEquals(range(10), x)
コード例 #2
0
ファイル: hwapi.py プロジェクト: thunder-spb/homeauto
class MlanSearch(object):
    def __init__(self):
        self.tasklet = None

    def search(self, line):
        if self.tasklet is None:
            self.reqchannel = Channel()
            self.queued = {}
            self.tasklet = Tasklet(self._checker)
            self.tasklet()
        ch = Channel()
        if line in self.queued:
            self.queued[line].append(ch)
        else:
            self.queued[line] = [ch]
            self.reqchannel.send(line)
        return ch.recv()

    def _checker(self):
        while True:
            line = self.reqchannel.recv()
            waiters = self.reqchannel[line]
            del self.reqchannel[line]
            for waiter in waiters:
                if waiter.has_receiver():
                    waiter.send(["abc", "def", "ghi"])
コード例 #3
0
ファイル: rchost.py プロジェクト: amlinux/rcrobot
class RequestQueue(object):
    """
    This class represents abstract request queue. It has two basic operations -
    adding request to the queue and retrieving request to be sent just now.
    Different implementation may perform different algorithms of determining a
    the to be sent first.
    """
    def __init__(self):
        self._wait = Channel()

    def add(self, request):
        "Add request to the queue"
        if self._wait.has_receiver():
            self._wait.send(request)
        else:
            self.queue_add(request)

    def retrieve(self):
        "Retrieve request to be sent. If no requests in the queue sleep until request arrives"
        request = self.queue_retrieve()
        if request is None:
            request = self._wait.receive()
        return request

    def find(self, primary_key):
        "Find request in the internal queue by its primary key"

    def queue_add(self, request):
        "Add request to the internal queue"

    def queue_retrieve(self, request):
        "Retrieve request from the internal queue. If no request in the internal queue return None"
コード例 #4
0
    def testReceiveIter(self):

        test_channel = Channel()

        def sender():
            for i in range(10):
                test_channel.send(i)

        t = Tasklet.new(sender)()
        x = []
        for i in test_channel.receive_n(10):
            x.append(i)

        self.assertEquals(range(10), x)

        t = Tasklet.new(sender)()



        try:
            Tasklet.set_current_timeout(1.0)
            x = []
            for i in test_channel:
                x.append(i)
        except TimeoutError:
            pass
        finally:
            Tasklet.set_current_timeout(TIMEOUT_NEVER)

        self.assertEquals(range(10), x)
コード例 #5
0
ファイル: test-offline.py プロジェクト: amlinux/rcrobot
class EmulatedHost(object):
    def __init__(self, dispatcher):
        self.dispatcher = dispatcher
        self._ch = Channel()

    def loop(self):
        while True:
            pkt = self._ch.receive()
            Tasklet.yield_()
            if pkt == [1, 2, 3]:
                # Test request
                self.dispatcher.receive([4, 5, 6])
            elif pkt == ['R', 5, 1, 2, 'V']:
                # Version request
                self.dispatcher.receive([ord('R'), 2, 1, ord('V'), 88])
            elif pkt == [1, 2, 5]:
                # Device unreachable (timeout)
                self.dispatcher.receive([ord('N'), 5, 1])
            elif pkt == [1, 2, 6]:
                # Device unreachable (incorrect response)
                self.dispatcher.receive([ord('N'), 5, 2])
            elif pkt == [1, 2, 7]:
                # Valid radio response
                self.dispatcher.receive([ord('R'), 3, 1, 4, 5, 6])
            elif pkt == ['R', 5, 1, 2, 'D']:
                # ADC data request
                self.dispatcher.receive([ord('R'), 2, 1, ord('D'), 0, 0, 0, 1, 1, 0, 4, 0, 255, 255, 0, 184])

    def send(self, pkt):
        self._ch.send(pkt)

    def next_pkt_id(self):
        return 5
コード例 #6
0
ファイル: client.py プロジェクト: bradjasper/concurrence
 def _do_command(self, cmd, *args):
     block_channel = Channel()
     self._command_queue.append((cmd, args, block_channel))
     try:
         return block_channel.receive()
     except TaskletError, e:
         raise MemcacheError(str(e.cause))
コード例 #7
0
    def __init__(self,
                 connector,
                 dbargs,
                 max_connections=10,
                 connect_timeout=-1,
                 max_connection_age=None,
                 max_connection_age_reaper_interval=60):
        super(Pool, self).__init__(connector, dbargs, connect_timeout)

        self._max_connections = max_connections
        self._max_connection_age = max_connection_age

        #some statistics
        self._queue_wait_timer_statistic = StatisticExtra()
        self._queue_wait_tasks_statistic = StatisticExtra()

        self._pool = Deque()  #the pool of available idle connections

        #watch for server disconnects on idle connections:
        self._idle_disconnect_channel = Channel()
        self._idle_disconnect_reaper_task = Tasklet.loop(
            self._idle_disconnect_reaper, daemon=True)()

        #check for old connections
        if self._max_connection_age is not None:
            self._old_connection_reaper_task = Tasklet.interval(
                max_connection_age_reaper_interval,
                self._old_connection_reaper,
                daemon=True)()
コード例 #8
0
class RequestQueue(object):
    """
    This class represents abstract request queue. It has two basic operations -
    adding request to the queue and retrieving request to be sent just now.
    Different implementation may perform different algorithms of determining a
    the to be sent first.
    """
    def __init__(self):
        self._wait = Channel()

    def add(self, request):
        "Add request to the queue"
        if self._wait.has_receiver():
            self._wait.send(request)
        else:
            self.queue_add(request)

    def retrieve(self):
        "Retrieve request to be sent. If no requests in the queue sleep until request arrives"
        request = self.queue_retrieve()
        if request is None:
            request = self._wait.receive()
        return request

    def find(self, primary_key):
        "Find request in the internal queue by its primary key"

    def queue_add(self, request):
        "Add request to the internal queue"

    def queue_retrieve(self, request):
        "Retrieve request from the internal queue. If no request in the internal queue return None"
コード例 #9
0
ファイル: hwapi.py プロジェクト: wujianguo/homeauto
class MlanSearch(object):
    def __init__(self):
        self.tasklet = None

    def search(self, line):
        if self.tasklet is None:
            self.reqchannel = Channel()
            self.queued = {}
            self.tasklet = Tasklet(self._checker)
            self.tasklet()
        ch = Channel()
        if line in self.queued:
            self.queued[line].append(ch)
        else:
            self.queued[line] = [ch]
            self.reqchannel.send(line)
        return ch.recv()

    def _checker(self):
        while True:
            line = self.reqchannel.recv()
            waiters = self.reqchannel[line]
            del self.reqchannel[line]
            for waiter in waiters:
                if waiter.has_receiver():
                    waiter.send(["abc", "def", "ghi"])
コード例 #10
0
 def _do_command(self, cmd, *args):
     block_channel = Channel()
     self._command_queue.append((cmd, args, block_channel))
     try:
         return block_channel.receive()
     except TaskletError, e:
         raise MemcacheError(str(e.cause))
コード例 #11
0
ファイル: rchost.py プロジェクト: amlinux/rcrobot
 def loop(self):
     "Infinitely reads input queue, performs requests and returns responses"
     host_tasks = []
     for host in self.hosts:
         task = Tasklet.new(host.loop)
         host_tasks.append(task)
         task()
     try:
         while True:
             # Fetch and send requests infinitely
             request = self.queue.retrieve()
             # Make routing decision
             host = self.route(request)
             if host is None:
                 # No route to send this packet
                 request.deliver_exception(DestinationUnreachable)
             else:
                 # Creating "callback" channel
                 channel = Channel()
                 self.sent_request = (request, channel)
                 try:
                     # Sending request
                     request.send(host)
                     try:
                         # Waiting for response
                         response = channel.receive(2)
                     except TimeoutError:
                         request.deliver_onetime_exception(TimeoutError)
                         if request.any_waiters():
                             request.failures += 1
                             self.queue.add(request)
                     except DeviceUnreachable as e:
                         if e.reason == 2:
                             # reason=2 means we received unexpected packet from another host
                             # occasionally. In this case even onetime requests are performed again
                             self.queue.add(request)
                         else:
                             request.failures += 1
                             args = e.args
                             request.deliver_onetime_exception(e.__class__, *args)
                             if request.any_waiters():
                                 self.queue.add(request)
                     except CommunicationError as e:
                         # Dispatcher loop received unexpected exception"
                         args = e.args
                         request.deliver_exception(e.__class__, *args)
                     else:
                         # Dispatcher loop received valid response. Delivering it to all waiters
                         request.deliver_response(response)
                 finally:
                     self.sent_request = False
     finally:
         for task in host_tasks:
             task.kill()
コード例 #12
0
ファイル: rchost.py プロジェクト: amlinux/rcrobot
 def response(self, timeout=-1):
     """
     Wait for response and return it.
     timeout: -1: retry infinitely, 0: one attempt (raise exception if any), >0: timeout in seconds
     """
     channel = Channel()
     if timeout == 0:
         self._waiters.append((channel, True))
         return channel.receive()
     else:
         self._waiters.append((channel, False))
         return channel.receive(timeout)
コード例 #13
0
 def response(self, timeout=-1):
     """
     Wait for response and return it.
     timeout: -1: retry infinitely, 0: one attempt (raise exception if any), >0: timeout in seconds
     """
     channel = Channel()
     if timeout == 0:
         self._waiters.append((channel, True))
         return channel.receive()
     else:
         self._waiters.append((channel, False))
         return channel.receive(timeout)
コード例 #14
0
class WSGIInputStream(object):
    def __init__(self, request, reader):
        transfer_encoding = request.get_request_header('Transfer-Encoding')
        if transfer_encoding is not None and transfer_encoding == 'chunked':
            assert False, 'chunked post not supported yet'

        content_length = request.get_request_header('Content-length')
        if content_length is None:
            self._channel = None
            self._n = None
            self._file = None
        else:
            self._n = int(content_length)
            self._file = reader.file()
            self._channel = Channel()

    def _read_request_data(self):
        if self._n is not None:
            self._channel.receive()  #wait till handler has read all input data

    def read(self, n=-1):
        if self._n > 0:
            data = None
            if n == -1:
                data = self._file.read(self._n)
            else:
                data = self._file.read(min(self._n, n))
            self._n -= len(data)
            if self._n == 0:
                self._n = None
                self._file = None
                self._channel.send(True)  #unblock reader
                self._channel = None
            return data
        else:
            return ''  #EOF

    def readline(self):
        line = ''
        while True:
            data = self._file.read(1)
            line += data
            if data == '\n':
                break
        self._n -= len(line)
        return line

    def readlines(self):
        assert False, 'TODO'

    def __iter__(self):
        assert False, 'TODO'
コード例 #15
0
ファイル: server.py プロジェクト: fullc0de/concurrence
class WSGIInputStream(object):
    def __init__(self, request, reader):
        transfer_encoding = request.get_request_header('Transfer-Encoding')
        if transfer_encoding is not None and transfer_encoding == 'chunked':
            assert False, 'chunked post not supported yet'

        content_length = request.get_request_header('Content-length')
        if content_length is None:
            self._channel = None
            self._n = None
            self._file = None
        else:
            self._n = int(content_length)
            self._file = reader.file()
            self._channel = Channel()

    def _read_request_data(self):
        if self._n is not None:
            self._channel.receive() #wait till handler has read all input data

    def read(self, n = -1):
        if self._n > 0:
            data = None
            if n == -1:
                data = self._file.read(self._n)
            else:
                data = self._file.read(min(self._n, n))
            self._n -= len(data)
            if self._n == 0:
                self._n = None
                self._file = None
                self._channel.send(True) #unblock reader
                self._channel = None
            return data
        else:
            return '' #EOF

    def readline(self):
		line = ''
		while True:
			data = self._file.read(1)
			line += data
			if data == '\n':
				break
		self._n -= len(line)
		return line

    def readlines(self):
        assert False, 'TODO'

    def __iter__(self):
        assert False, 'TODO'
コード例 #16
0
ファイル: hwapi.py プロジェクト: wujianguo/homeauto
 def search(self, line):
     if self.tasklet is None:
         self.reqchannel = Channel()
         self.queued = {}
         self.tasklet = Tasklet(self._checker)
         self.tasklet()
     ch = Channel()
     if line in self.queued:
         self.queued[line].append(ch)
     else:
         self.queued[line] = [ch]
         self.reqchannel.send(line)
     return ch.recv()
コード例 #17
0
ファイル: hwapi.py プロジェクト: thunder-spb/homeauto
 def search(self, line):
     if self.tasklet is None:
         self.reqchannel = Channel()
         self.queued = {}
         self.tasklet = Tasklet(self._checker)
         self.tasklet()
     ch = Channel()
     if line in self.queued:
         self.queued[line].append(ch)
     else:
         self.queued[line] = [ch]
         self.reqchannel.send(line)
     return ch.recv()
コード例 #18
0
    def __init__(self, request, reader):
        transfer_encoding = request.get_request_header('Transfer-Encoding')
        if transfer_encoding is not None and transfer_encoding == 'chunked':
            assert False, 'chunked post not supported yet'

        content_length = request.get_request_header('Content-length')
        if content_length is None:
            self._channel = None
            self._n = None
            self._file = None
        else:
            self._n = int(content_length)
            self._file = reader.file()
            self._channel = Channel()
コード例 #19
0
ファイル: httpperf.py プロジェクト: JoyTeam/concurrence
 def __init__(self, options):
     self.status = {}
     self.request = 0
     self.lastRequest = None
     self.lastTime = None
     self.options = options
     self.dispenser = Channel()
コード例 #20
0
    def testHasSender(self):

        test_channel = Channel()

        def sender():
            test_channel.send(True)

        self.assertEquals(False, test_channel.has_sender())

        r = Tasklet.new(sender)()

        Tasklet.sleep(1.0)

        self.assertEquals(True, test_channel.has_sender())

        r.kill()

        Tasklet.sleep(1.0)

        self.assertEquals(False, test_channel.has_sender())
コード例 #21
0
ファイル: testcore.py プロジェクト: winner0911/concurrence
    def testHasReceiver(self):

        test_channel = Channel()

        def receiver():
            test_channel.receive()

        self.assertEquals(False, test_channel.has_receiver())

        r = Tasklet.new(receiver)()

        Tasklet.sleep(1.0)

        self.assertEquals(True, test_channel.has_receiver())

        r.kill()

        Tasklet.sleep(1.0)

        self.assertEquals(False, test_channel.has_receiver())
コード例 #22
0
ファイル: testcore.py プロジェクト: JoyTeam/concurrence
    def testHasSender(self):

        test_channel = Channel()

        def sender():
            test_channel.send(True)

        self.assertEquals(False, test_channel.has_sender())

        r = Tasklet.new(sender)()

        Tasklet.sleep(1.0)

        self.assertEquals(True, test_channel.has_sender())

        r.kill()

        Tasklet.sleep(1.0)

        self.assertEquals(False, test_channel.has_sender())
コード例 #23
0
ファイル: testcore.py プロジェクト: bradjasper/concurrence
 def testHasReceiver(self):
     
     test_channel = Channel()
     
     def receiver():
         test_channel.receive()
     
     self.assertEquals(False, test_channel.has_receiver())
     
     r = Tasklet.new(receiver)()
     
     Tasklet.sleep(1.0)
     
     self.assertEquals(True, test_channel.has_receiver())
     
     r.kill()
     
     Tasklet.sleep(1.0)
     
     self.assertEquals(False, test_channel.has_receiver())
コード例 #24
0
ファイル: testcore.py プロジェクト: bradjasper/concurrence
 def testRecvTimeout(self):
     
     #receive within timeout
     test_channel = Channel()
     t1 = Tasklet.later(1.0, test_channel.send)(10)
     try:            
         self.assertEqual(10, test_channel.receive(2.0))
     except TimeoutError:
         self.fail('did not expect timeout')
     finally:
         t1.kill()
     
     #receive with timeout
     test_channel = Channel()
     t1 = Tasklet.later(2.0, test_channel.send)(10)
     try:            
         self.assertEqual(10, test_channel.receive(1.0))
         self.fail('expected timeout')
     except TimeoutError:
         pass #expected
     finally:
         t1.kill()
コード例 #25
0
ファイル: hwapi.py プロジェクト: wujianguo/homeauto
 def monitor(self, environ, start_response):
     state = self.monitor_state()
     mismatch = False
     for relay in xrange(1, 31):
         actualState = state["relay%d" % relay]
         recvState = self.param(environ, "relay%d" % relay)
         try:
             recvState = int(recvState)
         except Exception:
             recvState = None
         if recvState != actualState:
             mismatch = True
     # If any API parameter mismatches actual value
     # Send all actual data immediately. Otherwise lock
     # request in timeout.
     if not mismatch:
         ch = Channel()
         self.monitor_waiters.append(ch)
         try:
             state = ch.receive(10)
         except TimeoutError:
             pass
     return self.json(start_response, state)
コード例 #26
0
ファイル: hwapi.py プロジェクト: thunder-spb/homeauto
 def monitor(self, environ, start_response):
     state = self.monitor_state()
     mismatch = False
     for relay in xrange(1, 31):
         actualState = state["relay%d" % relay]
         recvState = self.param(environ, "relay%d" % relay)
         try:
             recvState = int(recvState)
         except Exception:
             recvState = None
         if recvState != actualState:
             mismatch = True
     # If any API parameter mismatches actual value
     # Send all actual data immediately. Otherwise lock
     # request in timeout.
     if not mismatch:
         ch = Channel()
         self.monitor_waiters.append(ch)
         try:
             state = ch.receive(10)
         except TimeoutError:
             pass
     return self.json(start_response, state)
コード例 #27
0
ファイル: server.py プロジェクト: fullc0de/concurrence
    def __init__(self, request, reader):
        transfer_encoding = request.get_request_header('Transfer-Encoding')
        if transfer_encoding is not None and transfer_encoding == 'chunked':
            assert False, 'chunked post not supported yet'

        content_length = request.get_request_header('Content-length')
        if content_length is None:
            self._channel = None
            self._n = None
            self._file = None
        else:
            self._n = int(content_length)
            self._file = reader.file()
            self._channel = Channel()
コード例 #28
0
ファイル: testtimer.py プロジェクト: JoyTeam/concurrence
    def testTimer(self):

        ch = Channel()

        def sender(times):
            for i in range(times):
                Tasklet.sleep(1.0)
                ch.send(True)

        with Timeout.push(10):
            Tasklet.new(sender)(4)
            for i in range(4):
                ch.receive(Timeout.current())

        start = time.time()
        try:
            with Timeout.push(2.5):
                Tasklet.new(sender)(4)
                for i in range(4):
                    ch.receive(Timeout.current())
                self.fail('expected timeout')
        except TimeoutError, e:
            end = time.time()
            self.assertAlmostEqual(2.5, end - start, places = 1)
コード例 #29
0
ファイル: testcore.py プロジェクト: softsoft/concurrence
    def testLoop(self):
        recvd = []

        def looper(channel):
            res = channel.receive()
            if res == None:
                raise Exception(
                    "this is an expected exception!! (not a failed test...)")
            else:
                recvd.append(res)

        looper_channel = Channel()
        looper_task = Tasklet.loop(looper)(looper_channel)

        for i in range(10):
            looper_channel.send(i)
        self.assertEqual(range(10), recvd)

        self.assertEqual(-1, looper_channel.balance)

        self.assertTrue(looper_task.alive)

        looper_channel.send(None)  #will trigger exception loop

        #must still be working
        recvd = []
        for i in range(10):
            looper_channel.send(i)
        self.assertEqual(range(10), recvd)

        self.assertEqual(-1, looper_channel.balance)

        looper_task.kill()

        self.assertEqual(0, looper_channel.balance)

        #assert that looper exitted, because it is not receiving anymore
        self.assertFalse(looper_channel.has_receiver())

        self.assertFalse(looper_task.alive)
コード例 #30
0
ファイル: testcore.py プロジェクト: winner0911/concurrence
    def testSendTimeout(self):
        #send within timeout

        test_channel = Channel()
        tl = Tasklet.later(1.0, test_channel.receive)()
        try:
            test_channel.send(10, 2.0)
        except TimeoutError:
            self.fail('did not expect timeout')
        finally:
            tl.kill()

        #send with timeout
        test_channel = Channel()
        tl = Tasklet.later(2.0, test_channel.receive)()
        try:
            test_channel.send(10, 1.0)
            self.fail('expected timeout')
        except TimeoutError:
            pass  #expected
        finally:
            tl.kill()
コード例 #31
0
ファイル: testcore.py プロジェクト: bradjasper/concurrence
    def testLoop(self):
        recvd = []
        def looper(channel):
            res = channel.receive()
            if res == None:
                raise Exception("some exception")
            else:
                recvd.append(res)
            
        looper_channel = Channel()
        looper_task = Tasklet.loop(looper)(looper_channel)

        for i in range(10):
            looper_channel.send(i)
        self.assertEqual(range(10), recvd)

        self.assertEqual(-1, looper_channel.balance)

        self.assertTrue(looper_task.alive)
        
        looper_channel.send(None) #will trigger exception loop
        
        #must still be working
        recvd = []
        for i in range(10):
            looper_channel.send(i)
        self.assertEqual(range(10), recvd)

        self.assertEqual(-1, looper_channel.balance)
        
        looper_task.kill()

        self.assertEqual(0, looper_channel.balance)
        
        #assert that looper exitted, because it is not receiving anymore
        self.assertFalse(looper_channel.has_receiver())

        self.assertFalse(looper_task.alive)
コード例 #32
0
ファイル: pool.py プロジェクト: bradjasper/concurrence
 def __init__(self, connector, dbargs, max_connections = 10, connect_timeout = -1, max_connection_age = None,
              max_connection_age_reaper_interval = 60):
     super(Pool, self).__init__(connector, dbargs, connect_timeout)
     
     self._max_connections = max_connections
     self._max_connection_age = max_connection_age
     
     #some statistics        
     self._queue_wait_timer_statistic = StatisticExtra()
     self._queue_wait_tasks_statistic = StatisticExtra()        
     
     self._pool = Deque() #the pool of available idle connections
             
     #watch for server disconnects on idle connections:
     self._idle_disconnect_channel = Channel()
     self._idle_disconnect_reaper_task = Tasklet.loop(self._idle_disconnect_reaper, daemon = True)()
     
     #check for old connections
     if self._max_connection_age is not None:
         self._old_connection_reaper_task = Tasklet.interval(max_connection_age_reaper_interval, 
                                                             self._old_connection_reaper, daemon = True)()
コード例 #33
0
ファイル: core.py プロジェクト: CyrilSha/metagam
 def __init__(self, app, uuid, rules, fqn="mg.mmorpg.combats.core.Combat"):
     mg.constructor.ConstructorModule.__init__(self, app, fqn)
     CombatParamsContainer.__init__(self)
     self.members = []
     self.log = None
     self.member_id = 0
     self.rules = rules
     self.uuid = uuid
     self.controllers = []
     self.rulesinfo = self.conf("combats-%s.rules" % rules, {})
     self.paramsinfo = self.conf("combats-%s.params" % rules, {})
     self.actionsinfo = self.conf("combats-%s.actions" % rules, [])
     self.commands = []
     self.wakeup_channel = Channel()
     self.running_actions = []
     self.ready_actions = []
     self._turn_order_check = False
     self._check_end_condition = True
     self.not_delivered_log = []
     self.start_time = time.time()
     self._flags = set()
     self._textlog_ring = []
コード例 #34
0
ファイル: testcore.py プロジェクト: winner0911/concurrence
    def testSendRecv(self):
        """test simple send and receive on a channel"""
        def sender(channel):
            for i in range(3):
                channel.send(i)

        def receiver(channel):
            while True:
                recvd.append(channel.receive())

        recvd = []

        test_channel = Channel()

        send_task = Tasklet.new(sender)(test_channel)
        recv_task = Tasklet.new(receiver)(test_channel)

        Tasklet.join(send_task)

        self.assertEquals([0, 1, 2], recvd)

        recv_task.kill()
コード例 #35
0
ファイル: testcore.py プロジェクト: winner0911/concurrence
    def testRecvTimeout(self):

        #receive within timeout
        test_channel = Channel()
        t1 = Tasklet.later(1.0, test_channel.send)(10)
        try:
            self.assertEqual(10, test_channel.receive(2.0))
        except TimeoutError:
            self.fail('did not expect timeout')
        finally:
            t1.kill()

        #receive with timeout
        test_channel = Channel()
        t1 = Tasklet.later(2.0, test_channel.send)(10)
        try:
            self.assertEqual(10, test_channel.receive(1.0))
            self.fail('expected timeout')
        except TimeoutError:
            pass  #expected
        finally:
            t1.kill()
コード例 #36
0
ファイル: testcore.py プロジェクト: bradjasper/concurrence
    def testSendTimeout(self):
        #send within timeout

        test_channel = Channel()
        tl = Tasklet.later(1.0, test_channel.receive)()
        try:            
            test_channel.send(10, 2.0)
        except TimeoutError:
            self.fail('did not expect timeout')
        finally:
            tl.kill()
        
        #send with timeout
        test_channel = Channel()
        tl = Tasklet.later(2.0, test_channel.receive)()
        try:            
            test_channel.send(10, 1.0)
            self.fail('expected timeout')
        except TimeoutError:
            pass #expected
        finally:
            tl.kill()
コード例 #37
0
ファイル: testtimer.py プロジェクト: softsoft/concurrence
    def testTimer(self):

        ch = Channel()

        def sender(times):
            for i in range(times):
                Tasklet.sleep(1.0)
                ch.send(True)

        with Timeout.push(10):
            Tasklet.new(sender)(4)
            for i in range(4):
                ch.receive(Timeout.current())

        start = time.time()
        try:
            with Timeout.push(2.5):
                Tasklet.new(sender)(4)
                for i in range(4):
                    ch.receive(Timeout.current())
                self.fail('expected timeout')
        except TimeoutError, e:
            end = time.time()
            self.assertAlmostEqual(2.5, end - start, places=1)
コード例 #38
0
ファイル: test-offline.py プロジェクト: amlinux/rcrobot
 def __init__(self, dispatcher):
     self.dispatcher = dispatcher
     self._ch = Channel()
コード例 #39
0
ファイル: server.py プロジェクト: softsoft/concurrence
class WSGIInputStream(object):
    def __init__(self, request, reader):
        transfer_encoding = request.get_request_header('Transfer-Encoding')
        if transfer_encoding is not None and transfer_encoding == 'chunked':
            assert False, 'chunked post not supported yet'

        content_length = request.get_request_header('Content-length')
        if content_length is None:
            self._channel = None
            self._n = None
            self._file = None
        else:
            self._n = int(content_length)
            self._file = reader.file()
            self._channel = Channel()
        self.readline_buffer = None

    def _read_request_data(self):
        if self._n is not None:
            self._channel.receive() #wait till handler has read all input data

    def read(self, n):
        if self._n > 0:
            data = self._file.read(min(self._n, n))
            self._n -= len(data)
            if self._n == 0:
                self._n = None
                self._file = None
                self._channel.send(True) #unblock reader
                self._channel = None
            return data
        else:
            return '' #EOF

    def readline(self, maxlen=-1):
        if self.readline_buffer is None:
            self.readline_buffer = ''
            self.re_line = re.compile(r'^(.*?(?:\r\n|\n))(.*)', re.DOTALL)
        while True:
            m = self.re_line.match(self.readline_buffer)
            if m:
                line, self.readline_buffer = m.group(1, 2)
                return line
            elif self._n <= 0:
                if len(self.readline_buffer):
                    line = self.readline_buffer
                    self.readline_buffer = ''
                    return line
                return None
            data = self._file.read(min(self._n, 16384))
            if len(data) == 0:
                self._n = 0
            else:
                self._n -= len(data)
            self.readline_buffer = self.readline_buffer + data

    def readlines(self):
        assert False, 'TODO'

    def __iter__(self):
        assert False, 'TODO'
コード例 #40
0
ファイル: pool.py プロジェクト: bradjasper/concurrence
class Pool(BasePool):
    log = logging.getLogger('Pool')
    
    def __init__(self, connector, dbargs, max_connections = 10, connect_timeout = -1, max_connection_age = None,
                 max_connection_age_reaper_interval = 60):
        super(Pool, self).__init__(connector, dbargs, connect_timeout)
        
        self._max_connections = max_connections
        self._max_connection_age = max_connection_age
        
        #some statistics        
        self._queue_wait_timer_statistic = StatisticExtra()
        self._queue_wait_tasks_statistic = StatisticExtra()        
        
        self._pool = Deque() #the pool of available idle connections
                
        #watch for server disconnects on idle connections:
        self._idle_disconnect_channel = Channel()
        self._idle_disconnect_reaper_task = Tasklet.loop(self._idle_disconnect_reaper, daemon = True)()
        
        #check for old connections
        if self._max_connection_age is not None:
            self._old_connection_reaper_task = Tasklet.interval(max_connection_age_reaper_interval, 
                                                                self._old_connection_reaper, daemon = True)()
        
    def __statistics__(self):
        return {'connections': {'total': self.connection_count, 
                                'connection_failed': self._failed_connect,
                                'connection_new': self._new_connection_timer_statistic,
                                'connection_close': self._close_connection_timer_statistic,
                                'queue_wait_time': self._queue_wait_timer_statistic,
                                'queue_wait_task': self._queue_wait_tasks_statistic}}
        
    @property
    def idle_connection_count(self):
        return len(self._pool)
    
    def _idle_disconnect_reaper(self):
        """waits for readability events in the idle_disconnect_channel
        this signals a EOF from the database, so we can remove the connection 
        from the pool"""
        readable = self._idle_disconnect_channel.receive()
        #now we now which fd became readable, figure out which connection it was
        disconnected_connection = None
        for connection in self._pool:
            if connection.socket.readable == readable:
                disconnected_connection = connection
        if disconnected_connection is None:
            self.log.error("%s: received disconnected event, but could not find corresponding connection!", self)
        else:
            self._close(disconnected_connection)
            self.log.warn("%s: connection disconnected by database server while idle.", self)

    def _old_connection_reaper(self):
        """checks all connections in the pool for their age
        if too old and idle, gets closed immediatly
        if too old and in use, gets closed on next disconnect"""
        now = time.time()
        close_connections = [] #to prevent concurrent modification in loop:
        for connection in self._connections:
            age = now - connection._created_time
            if age > self._max_connection_age:
                close_connections.append(connection)
                
        for connection in close_connections:
            if connection in self._pool: #it is idle, close now
                self.log.debug("%s: closing idle connection with old age", self)
                self._close(connection)
            else:
                self.log.debug("%s: will close busy connection with old age on next disconnect", self)
                connection.__close__ = True #not idle, will be closed on next disconnect
                
    def _get_connection_from_pool(self):
        self.log.debug("get conn from pool")
        return self._pool.pop(True, Timeout.current())
        
    def _return_connection_to_pool(self, connection):
        """when connection becomes readable while in the idle pool,
        this signals a server disconnect"""
        self.log.debug("return conn to pool")
        connection.socket.readable.notify(self._idle_disconnect_channel)
        self._pool.append(connection)
        
    def connect(self):
        """get a connection from the pool, will wait for maxWaitTime for connection to become
        available, or will create a new connection if connectioncount < max_connections"""
        with Timeout.push(self._connect_timeout):
            if (not self._pool) and (self.connection_count < self._max_connections):
                #none available, but still allowed to create new connection
                try:                    
                    return (True, self._new())
                except TaskletExit:
                    raise #server exiting
                except TimeoutError:
                    raise
                except:
                    self.log.exception("%s: could not create new connection for pool", self)
                    #we will continue from here waiting for idle connection
    
            #if we are here, either connection is available, not available but no more connections are allowed,
            #or there was some exception creating a new connection        
            self.log.debug("waiting for connection")
            with self._queue_wait_timer_statistic.time():
                #keep track off the amount of other tasks waiting for a connection
                balance = self._pool.channel.balance
                waiters = -balance if balance < 0 else 0
                self._queue_wait_tasks_statistic.set_count(waiters)
                self._queue_wait_tasks_statistic.update_avg(waiters)
                connection = self._get_connection_from_pool()
                self.log.debug("got connection")
                return (False, connection)
    
    def _close(self, connection):
        """close given connection and remove it from the pool"""
        #if it is currently in the pool, remove it
        if connection in self._pool:
            self._pool.remove(connection)   
        #close it
        super(Pool, self)._close(connection)
    
    def disconnect(self, connection, close = False):
        """return connection to pool. if close is given, it is closed and removed instead"""
        assert hasattr(connection, '_pool'), "this connection did not come from a pool"
        assert connection._pool == self, "this connection did not come from this pool"
        
        if hasattr(connection, '__close__'): #set by old age reaper
            self.log.debug("close on old age")
            close = True
        
        if close:
            #close connection and remove from pool
            self._close(connection)
            return True
        else:
            #return the connection to idle queue
            self._return_connection_to_pool(connection)                        
            self.log.debug("returning connection to pool %s", self)
            return False
コード例 #41
0
ファイル: core.py プロジェクト: CyrilSha/metagam
class Combat(mg.constructor.ConstructorModule, CombatParamsContainer):
    "Combat is the combat itself. It is created in the combat daemon process."

    system_params = set(["stage", "title", "time", "timetext"])
    def __init__(self, app, uuid, rules, fqn="mg.mmorpg.combats.core.Combat"):
        mg.constructor.ConstructorModule.__init__(self, app, fqn)
        CombatParamsContainer.__init__(self)
        self.members = []
        self.log = None
        self.member_id = 0
        self.rules = rules
        self.uuid = uuid
        self.controllers = []
        self.rulesinfo = self.conf("combats-%s.rules" % rules, {})
        self.paramsinfo = self.conf("combats-%s.params" % rules, {})
        self.actionsinfo = self.conf("combats-%s.actions" % rules, [])
        self.commands = []
        self.wakeup_channel = Channel()
        self.running_actions = []
        self.ready_actions = []
        self._turn_order_check = False
        self._check_end_condition = True
        self.not_delivered_log = []
        self.start_time = time.time()
        self._flags = set()
        self._textlog_ring = []

    def script_code(self, tag):
        "Get combat script code (syntax tree)"
        return self.conf("combats-%s.script-%s" % (self.rules, tag), [])

    def join(self, member):
        "Join member to the combat"
        self.member_id += 1
        member.id = self.member_id
        self.members.append(member)
        # script event
        globs = self.globs()
        globs["member"] = member
        self.execute_script("joined", globs, lambda: self._("Member joined script"))
        # if combat is started already, notify all other members
        if self.running:
            for controller in self.controllers:
                if controller.connected:
                    controller.deliver_member_joined(member)
        # register member's controllers
        for controller in member.controllers:
            self.add_controller(controller)
        # log join
        self.syslog({
            "type": "join",
            "member": member.id,
            "text": self._("<b>[{time}]</b> Member {id} ({name}) has joined team {team}").format(
                time=self.now(),
                id=member.id,
                name=member.name,
                team=member.team,
            ),
            "cls": "combat-syslog-joined",
        })

    def member(self, memberId):
        for m in self.members:
            if m.id == memberId:
                return m
        return None

    def close(self):
        "Notify combat about it's terminated and about to be destroyed"
        if self.log:
            self.log.close()
        self.flush()

    @property
    def actions(self):
        "Dictionary of available combat actions"
        try:
            return self._actions
        except AttributeError:
            pass
        self._actions = {}
        for act in self.conf("combats-%s.actions" % self.rules, []):
            self._actions[act["code"]] = act
        return self._actions

    @property
    def running(self):
        "True when combat is running"
        return self.stage != "init"

    def run(self, turn_order):
        """
        Run combat (switch to 'combat' stage).
        turn_order - CombatTurnOrder object
        """
        if self.running:
            raise CombatAlreadyRunning(self._("Combat was started twice"))
        self.turn_order = turn_order
        self.set_stage("combat")
        self.log_combat_time()
        # execute start script
        globs = self.globs()
        self.execute_script("start", globs, lambda: self._("Combat start script"))
        # notify all members
        for member in self.members:
            member.started()
        # notify turn order manager
        if self.stage_flag("actions"):
            self.turn_order.start()

    @property
    def stage(self):
        return self._params.get("stage", "init")

    def set_stage(self, stage):
        "Switch combat stage"
        if self.stages.get(stage) is None:
            raise CombatInvalidStage(self._("Combat stage '%s' is not defined") % stage)
        self.set_param("stage", stage)
        self.syslog({
            "type": "stage",
            "stage": stage,
            "text": self._("Combat stage: %s") % stage,
            "cls": "combat-syslog-stage",
        })
        self.wakeup()

    @property
    def flags(self):
        return self._flags

    def set_flags(self, flags):
        self._flags = set(flags)

    @property
    def title(self):
        return self._params.get("title", self._("Combat"))

    def set_title(self, title):
        "Set combat title"
        self.set_param("title", title)
        if self.log:
            self.log.set_title(title)
        self.wakeup()

    @property
    def timetext(self):
        time_format = self.rulesinfo.get("time_format", "mmss")
        time = self.time
        if time_format == "mmss":
            return "%d:%02d" % (time / 60, time % 60)
        elif time_format == "num":
            return self.time
        elif time_format == "realhhmmss":
            return self.now_local().split(" ")[1]

    @property
    def time_mode(self):
        try:
            return self._time_mode
        except AttributeError:
            pass
        self._time_mode = self.rulesinfo.get("time_mode", "begin")
        return self._time_mode

    @property
    def time(self):
        return self._params.get("time", 0)

    def log_combat_time(self):
        self.syslog({
            "text": self._("Combat time: %s") % self.time,
            "time": self.time,
            "cls": "combat-syslog-time",
        })
        if self.time_mode == "change":
            self.textlog({
                "text": self.timetext,
                "cls": "combat-log-time-header",
            })

    def add_time(self, val):
        val = intz(val)
        if val < 1:
            return
        self.set_param("time", self.time + val)
        self.set_param("timetext", self.timetext)
        self.log_combat_time()

    def add_controller(self, controller):
        "Register member controller"
        self.controllers.append(controller)

    @property
    def stages(self):
        "Dictionary of stages and their flags"
        try:
            return self._stages
        except AttributeError:
            pass
        val = self.conf("combats-%s.stages" % self.rules)
        if val is None:
            val = {
                "init": {
                },
                "combat": {
                    "actions": True,
                },
                "finish": {
                },
                "done": {
                   "done": True
                }
            }
        self._stages = val
        return val

    def stage_flag(self, flag):
        "Returns flag value of the current stage. If no flag with such code defined return None"
        return self.stages[self.stage].get(flag)

    def stopped(self):
        "Return True when the combat is stopped"
        return self.stage_flag("done")

    def add_command(self, command):
        "Put command to the combat queue to be executed immediately"
        self.commands.append(command)
        self.wakeup()

    def wakeup(self):
        "Wake up main combat loop if it's busy with processing now"
        if self.wakeup_channel.has_receiver():
            self.wakeup_channel.send(None)

    def process(self, timeout=1):
        "Process combat logic"
        if self._turn_order_check:
            self._turn_order_check = False
            self.turn_order.check()
        if self._check_end_condition:
            self._check_end_condition = False
            self.check_end_condition()
        self.process_commands()
        if self.stage_flag("actions"):
            self.process_actions()
        self.heartbeat()
        self.flush()
        try:
            self.wakeup_channel.receive(timeout)
        except TimeoutError:
            self.idle()

    def globs(self):
        return {
            "local": ScriptMemoryObject()
        }

    def execute_script(self, tag, globs, description=None):
        "Execute combat script with given code"
        self.call("combats.execute-script", self, self.script_code(tag), globs, description=description)

    def execute_member_script(self, member, tag, globs, description=None):
        "Execute combat script for given member"
        globs["member"] = member
        self.execute_script(tag, globs, description)
        self.enqueue_check_end_condition()

    def heartbeat(self):
        "Called on every iteration of the main loop"
        globs = self.globs()
        self.for_each_member(self.execute_member_script, "heartbeat-member", globs, lambda: self._("Member heartbeat script"))
        self.execute_script("heartbeat", globs, lambda: self._("Combat heartbeat script"))

    def process_commands(self):
        "Process enqueued commands"
        while self.commands:
            cmd = self.commands.pop(0)
            cmd.execute()

    def idle(self):
        "Do background processing"
        # execute scripts
        globs = self.globs()
        self.execute_script("idle", globs, lambda: self._("Combat idle script"))
        self.for_each_member(self.execute_member_script, "idle-member", globs, lambda: self._("Member idle script"))
        # call idle for all objects
        self.turn_order.idle()
        for member in self.members:
            member.idle()
        # process general timeouts
        elapsed = time.time() - self.start_time
        timeout = self.rulesinfo.get("timeout", 4 * 3600)
        if elapsed > timeout + 600:
            self.warning(self._("Combat %s terminated due to too long timeout"), self.uuid)
            os._exit(0)
        elif self.stage_flag("actions") and elapsed > timeout:
            self.info(self._("Combat %s timed out"), self.uuid)
            self.draw()

    def flush(self):
        "Flush pending messages"
        # deliver changed parameters
        params = self.changed_params()
        if params:
            for controller in self.controllers:
                controller.combat_params_changed(params)
        for member in self.members:
            params = member.changed_params()
            if params:
                for controller in self.controllers:
                    controller.member_params_changed(member, params)
        # commit logs
        if self.log:
            self.log.flush()
        # deliver new log entries
        if self.not_delivered_log:
            for controller in self.controllers:
                controller.deliver_log(self.not_delivered_log)
            self.not_delivered_log = []
        # flush everything to the clients
        for controller in self.controllers:
            controller.flush()
        self.call("stream.flush")

    def set_log(self, log):
        "Attach logging system to the combat"
        self.log = log

    def textlog(self, entry):
        "Add entry to combat log"
        if self.time_mode == "begin":
            entry["text"] = u'<span class="combat-log-time">%s</span> %s' % (self.timetext, entry.get("text", u""))
        self.not_delivered_log.append(entry)
        if self.log:
            self.log.textlog(entry)
        self._textlog_ring.append(entry)
        l = len(self._textlog_ring)
        if l > textlog_ring_size:
            del self._textlog_ring[0:l - textlog_ring_size]

    def syslog(self, entry):
        "Add entry to combat debug log"
        if self.log:
            self.log.syslog(entry)

    def stop(self):
        "Terminate combat"
        self.set_stage("done")

    # Scripting

    def script_attr(self, attr, handle_exceptions=True):
        if attr == "id":
            return self.uuid
        elif attr == "stage":
            return self.stage
        elif attr == "stage_flags":
            return CombatStageFlags(self)
        elif attr == "time":
            return self.time
        elif attr == "timetext":
            return self.timetext
        elif attr == "now":
            return self.now_local()
        # team list
        m = re_team_list.match(attr)
        if m:
            team = intz(m.group(1))
            return self.call("l10n.literal_enumeration", [u'<span class="combat-log-member">%s</span>' % member.name for member in self.members if member.team == team])
        # parameters
        m = re_param_attr.match(attr)
        if m:
            return self.param(attr, handle_exceptions)
        if handle_exceptions:
            return None
        else:
            raise AttributeError(attr)

    def script_set_attr(self, attr, val, env):
        if attr == "stage":
            return self.set_stage(val)
        # parameters
        m = re_param_attr.match(attr)
        if m:
            return self.set_param(attr, val)
        raise ScriptRuntimeError(self._("Invalid attribute '%s'") % attr, env)

    def store(self):
        pass

    # Actions

    def execute_action(self, action):
        "Start executing action"
        self.ready_actions.append(action)
    
    def process_actions(self):
        "Process actions logic"
        self.process_ready_actions()
        self.process_stopped_actions()

    def process_ready_actions(self):
        """
        For every ready action call begin() method and move the action to the list
        of running actions
        """
        if self.ready_actions:
            actions = self.ready_actions
            self.ready_actions = []
            for act in actions:
                if act.source.active:
                    act.begin()
                    self.running_actions.append(act)
            self.enqueue_turn_order_check()
            self.actions_started()
            self.enqueue_check_end_condition()

    def process_stopped_actions(self):
        "For every stopped action call end() method and remove the action from the list"
        if self.running_actions:
            i = 0
            while i < len(self.running_actions):
                act = self.running_actions[i]
                if act.stopped():
                    act.end()
                    del self.running_actions[i]
                else:
                    i += 1
            self.enqueue_turn_order_check()
            self.actions_stopped()
            self.enqueue_check_end_condition()

    def enqueue_check_end_condition(self):
        "Enqueue check_end_condition() to be called on the next iteration of the main loop"
        self._check_end_condition = True
        self.wakeup()

    def check_end_condition(self):
        "Check combat end condition (0 or 1 teams active)"
        if self.stage_flag("actions"):
            teams = set()
            for member in self.members:
                if member.active:
                    teams.add(member.team)
            teams = list(teams)
            if len(teams) == 0:
                self.draw()
            elif len(teams) == 1:
                self.victory(teams[0])

    def draw(self):
        "Combat finished with draw"
        self.syslog({
            "type": "draw",
            "text": self._("Combat was a draw"),
            "cls": "combat-syslog-end",
        })
        for member in self.members:
            member.draw()
        globs = self.globs()
        self.for_each_member(self.execute_member_script, "draw-member", globs, lambda: self._("Combat draw script for a member"))
        self.execute_script("draw", globs, lambda: self._("Combat draw script"))

    def victory(self, team):
        "Combat finished with victory of specified team"
        self.syslog({
            "type": "victory",
            "team": team,
            "text": self._("Victory of team {team}").format(
                team=team,
            ),
            "cls": "combat-syslog-end",
        })
        winners_list = []
        loosers_list = []
        first_winner = None
        first_looser = None
        for member in self.members:
            if member.team != team:
                member.defeat()
                loosers_list.append(member)
                if first_looser is None:
                    first_looser = member
        for member in self.members:
            if member.team == team:
                member.victory()
                winners_list.append(member)
                if first_winner is None:
                    first_winner = member
        globs = self.globs()
        globs["winner_team"] = team
        globs["winners_list"] = self.call("l10n.literal_enumeration", [u'<span class="combat-log-member">%s</span>' % member.name for member in winners_list])
        globs["loosers_list"] = self.call("l10n.literal_enumeration", [u'<span class="combat-log-member">%s</span>' % member.name for member in loosers_list])
        globs["first_winner"] = first_winner
        globs["first_looser"] = first_looser
        globs["winners_count"] = len(winners_list)
        globs["loosers_count"] = len(loosers_list)
        for member in self.members:
            if member.team != team:
                self.execute_member_script(member, "defeat-member", globs, lambda: self._("Combat defeat script for a member"))
        for member in self.members:
            if member.team == team:
                self.execute_member_script(member, "victory-member", globs, lambda: self._("Combat victory script for a member"))
        self.execute_script("victory", globs, lambda: self._("Combat victory script"))

    def notify_stopped(self):
        "Call this method to signal combat that it's finally stopped"
        for member in self.members:
            if member.may_turn:
                member.turn_take()
        for member in self.members:
            member.stopped()

    def actions_started(self):
        "Called after ready actions started"
        globs = self.globs()
        self.for_each_member(self.execute_member_script, "actions-started-member", globs, lambda: self._("Combat actions started script for a member"))
        self.execute_script("actions-started", globs, lambda: self._("Combat actions started script"))

    def actions_stopped(self):
        "Called after ready actions stopped"
        globs = self.globs()
        self.for_each_member(self.execute_member_script, "actions-stopped-member", globs, lambda: self._("Combat actions stopped script for a member"))
        self.execute_script("actions-stopped", globs, lambda: self._("Combat actions stopped script"))

    def enqueue_turn_order_check(self):
        "Ask combat server to call turn_order check() on the next iteration"
        self._turn_order_check = True
        self.wakeup()

    def for_each_member(self, callback, *args, **kwargs):
        "Call callback for every combat member. Member is passed as a first argument"
        for member in self.members:
            callback(member, *args, **kwargs)

    def __unicode__(self):
        return self._("[Combat %s]") % self.uuid

    def __str__(self):
        return utf2str(unicode(self))
コード例 #42
0
 def __init__(self):
     self._wait = Channel()
コード例 #43
0
 def loop(self):
     "Infinitely reads input queue, performs requests and returns responses"
     host_tasks = []
     for host in self.hosts:
         # Run loop
         task = Tasklet.new(host.loop)
         host_tasks.append(task)
         task()
         task = Tasklet.new(host.ping)
         host_tasks.append(task)
         task()
     try:
         while True:
             try:
                 # Fetch and send requests infinitely
                 request = self.queue.retrieve()
                 # Make routing decision
                 host = self.route(request)
                 if host is None:
                     # No route to send this packet
                     request.deliver_exception(DestinationUnreachable)
                 else:
                     # Creating "callback" channel
                     channel = Channel()
                     self.sent_request = (request, channel)
                     try:
                         # Sending request
                         request.send(host)
                         try:
                             # Waiting for response
                             response = channel.receive(2)
                         except TimeoutError:
                             request.deliver_onetime_exception(TimeoutError)
                             if request.any_waiters():
                                 request.failures += 1
                                 self.queue.add(request)
                         except DeviceUnreachable as e:
                             if e.reason == 2:
                                 # reason=2 means we received unexpected packet from another host
                                 # occasionally. In this case even onetime requests are performed again
                                 self.queue.add(request)
                             else:
                                 request.failures += 1
                                 args = e.args
                                 request.deliver_onetime_exception(e.__class__, *args)
                                 if request.any_waiters():
                                     self.queue.add(request)
                         except CommunicationError as e:
                             # Dispatcher loop received unexpected exception"
                             args = e.args
                             request.deliver_exception(e.__class__, *args)
                         else:
                             # Dispatcher loop received valid response. Delivering it to all waiters
                             request.deliver_response(response)
                     finally:
                         self.sent_request = False
             except Exception as e:
                 logging.exception(e)
     finally:
         for task in host_tasks:
             task.kill()
コード例 #44
0
class Pool(BasePool):
    log = logging.getLogger('Pool')

    def __init__(self,
                 connector,
                 dbargs,
                 max_connections=10,
                 connect_timeout=-1,
                 max_connection_age=None,
                 max_connection_age_reaper_interval=60):
        super(Pool, self).__init__(connector, dbargs, connect_timeout)

        self._max_connections = max_connections
        self._max_connection_age = max_connection_age

        #some statistics
        self._queue_wait_timer_statistic = StatisticExtra()
        self._queue_wait_tasks_statistic = StatisticExtra()

        self._pool = Deque()  #the pool of available idle connections

        #watch for server disconnects on idle connections:
        self._idle_disconnect_channel = Channel()
        self._idle_disconnect_reaper_task = Tasklet.loop(
            self._idle_disconnect_reaper, daemon=True)()

        #check for old connections
        if self._max_connection_age is not None:
            self._old_connection_reaper_task = Tasklet.interval(
                max_connection_age_reaper_interval,
                self._old_connection_reaper,
                daemon=True)()

    def __statistics__(self):
        return {
            'connections': {
                'total': self.connection_count,
                'connection_failed': self._failed_connect,
                'connection_new': self._new_connection_timer_statistic,
                'connection_close': self._close_connection_timer_statistic,
                'queue_wait_time': self._queue_wait_timer_statistic,
                'queue_wait_task': self._queue_wait_tasks_statistic
            }
        }

    @property
    def idle_connection_count(self):
        return len(self._pool)

    def _idle_disconnect_reaper(self):
        """waits for readability events in the idle_disconnect_channel
        this signals a EOF from the database, so we can remove the connection 
        from the pool"""
        readable = self._idle_disconnect_channel.receive()
        #now we now which fd became readable, figure out which connection it was
        disconnected_connection = None
        for connection in self._pool:
            if connection.socket.readable == readable:
                disconnected_connection = connection
        if disconnected_connection is None:
            self.log.error(
                "%s: received disconnected event, but could not find corresponding connection!",
                self)
        else:
            self._close(disconnected_connection)
            self.log.warn(
                "%s: connection disconnected by database server while idle.",
                self)

    def _old_connection_reaper(self):
        """checks all connections in the pool for their age
        if too old and idle, gets closed immediatly
        if too old and in use, gets closed on next disconnect"""
        now = time.time()
        close_connections = []  #to prevent concurrent modification in loop:
        for connection in self._connections:
            age = now - connection._created_time
            if age > self._max_connection_age:
                close_connections.append(connection)

        for connection in close_connections:
            if connection in self._pool:  #it is idle, close now
                self.log.debug("%s: closing idle connection with old age",
                               self)
                self._close(connection)
            else:
                self.log.debug(
                    "%s: will close busy connection with old age on next disconnect",
                    self)
                connection.__close__ = True  #not idle, will be closed on next disconnect

    def _get_connection_from_pool(self):
        self.log.debug("get conn from pool")
        return self._pool.pop(True, Timeout.current())

    def _return_connection_to_pool(self, connection):
        """when connection becomes readable while in the idle pool,
        this signals a server disconnect"""
        self.log.debug("return conn to pool")
        connection.socket.readable.notify(self._idle_disconnect_channel)
        self._pool.append(connection)

    def connect(self):
        """get a connection from the pool, will wait for maxWaitTime for connection to become
        available, or will create a new connection if connectioncount < max_connections"""
        with Timeout.push(self._connect_timeout):
            if (not self._pool) and (self.connection_count <
                                     self._max_connections):
                #none available, but still allowed to create new connection
                try:
                    return (True, self._new())
                except TaskletExit:
                    raise  #server exiting
                except TimeoutError:
                    raise
                except:
                    self.log.exception(
                        "%s: could not create new connection for pool", self)
                    #we will continue from here waiting for idle connection

            #if we are here, either connection is available, not available but no more connections are allowed,
            #or there was some exception creating a new connection
            self.log.debug("waiting for connection")
            with self._queue_wait_timer_statistic.time():
                #keep track off the amount of other tasks waiting for a connection
                balance = self._pool.channel.balance
                waiters = -balance if balance < 0 else 0
                self._queue_wait_tasks_statistic.set_count(waiters)
                self._queue_wait_tasks_statistic.update_avg(waiters)
                connection = self._get_connection_from_pool()
                self.log.debug("got connection")
                return (False, connection)

    def _close(self, connection):
        """close given connection and remove it from the pool"""
        #if it is currently in the pool, remove it
        if connection in self._pool:
            self._pool.remove(connection)
        #close it
        super(Pool, self)._close(connection)

    def disconnect(self, connection, close=False):
        """return connection to pool. if close is given, it is closed and removed instead"""
        assert hasattr(connection,
                       '_pool'), "this connection did not come from a pool"
        assert connection._pool == self, "this connection did not come from this pool"

        if hasattr(connection, '__close__'):  #set by old age reaper
            self.log.debug("close on old age")
            close = True

        if close:
            #close connection and remove from pool
            self._close(connection)
            return True
        else:
            #return the connection to idle queue
            self._return_connection_to_pool(connection)
            self.log.debug("returning connection to pool %s", self)
            return False
コード例 #45
0
ファイル: rchost.py プロジェクト: amlinux/rcrobot
 def __init__(self):
     self._wait = Channel()
コード例 #46
0
ファイル: httpperf.py プロジェクト: JoyTeam/concurrence
class HttpPerf(object):
    def __init__(self, options):
        self.status = {}
        self.request = 0
        self.lastRequest = None
        self.lastTime = None
        self.options = options
        self.dispenser = Channel()
        
    def session_response_reader(self, cnn, pipeline_tokens):
        #TODO use tasklet.loop, must be extended such that you can stop the loop by returning something (or StopIteration?)
        while True:
            response = cnn.receive()

            #read status
            self.count('status', response.status)
            
            connection_header = response.get_header('Connection')
            if connection_header == 'close' and self.options.requests != 1:
                print >> sys.stderr, "WARNING: Server closed connection, no Keep Alive!, please use --requests=1"
                
            
            #this will read the complete response
            if self.options.dump:
                print response.status
                for k, v in response.headers:
                    print "%s: %s" % (k, v)
                for chunk in response:
                    sys.stdout.write(chunk)
                sys.stdout.flush()
                print
            else:
                list(response)
            #print 'resp'
            pipeline_tokens.append(True)
                
    def session(self, host, port, path):
        cnn = None

        pipeline_tokens = Deque()

        for _ in range(self.options.pipeline): # can append take iterator?, or list?
            pipeline_tokens.append(True)

        try:
            cnn = HTTPConnection()
            cnn.connect((host, port))

            Tasklet.new(self.session_response_reader)(cnn, pipeline_tokens)
            
            requests = 0 #no requests in this session
            while True:

                if self.options.requests != -1 and requests >= self.options.requests:
                    break #we are done with this session
                       
                if self.dispenser.receive() is None:
                    return False #we are done globally

                pipeline_tokens.popleft(True)

                #do the request
                cnn.send(cnn.get(path))
                #print response
                  
                requests += 1
                
                self.count('request')
                
        finally:    
            #if response_reader_task is not None:   
            #    response_reader_task.kill()
            if cnn is not None:
                cnn.close() 

        return True

    def sessions(self):
        u = urlparse.urlparse(self.options.url)
        
        if ':' in u.netloc:
            host, port = u.netloc.split(':')
            port = int(port)
        else:
            host, port = u.netloc, 80
    
        path = urlparse.urlunsplit(['', '', u.path, u.query, u.fragment])
        if path == '':
            path = '/'
    
        try:
            while True:
                if not self.session(host, port, path):
                    return
                    
        except TaskletExit:
            raise
        except:
            logging.exception("exception in http session")
        
    def count(self, attr, key = None, inc = 1):
        a = getattr(self, attr)
        if key is None:
            v = a + inc
            setattr(self, attr, v)
            return v
        else:
            if not key in a:
                a[key] = inc
            else:
                a[key] = a[key] + inc
            return a[key]
       
    def show(self):
        now = time.time()
        
        if self.lastTime is not None:
            reqSec = (self.request - self.lastRequest) / (now - self.lastTime)
            reqSec = gamma_filter(self.lastReqSec, reqSec, 0.60)
        else:
            reqSec = 0.0
            
        print >> sys.stderr, self.status, self.request, reqSec
        
        self.lastTime = time.time()
        self.lastRequest = self.request
        self.lastReqSec = reqSec
        
    def dispense(self):
        if self.options.count == -1: 
            #run forever
            while True:
                self.dispenser.send(True)
                if self.options.delay > 0.0:
                    Tasklet.sleep(self.options.delay)
        else:
            #a fixed number of total requests
            for i in range(self.options.count):
                self.dispenser.send(True)
                if self.options.delay > 0.0:
                    Tasklet.sleep(self.options.delay)
            for i in range(self.options.sessions):
                self.dispenser.send(None)
        
    def run(self):
        #show stats every second:
        Tasklet.interval(1.0, self.show, immediate = True)()
        
        #dispenses tokens for doing a request to sessions:
        Tasklet.new(self.dispense)()
        
        #start up sessions, and wait till they are finished
        Tasklet.join_all([Tasklet.new(self.sessions)() for _ in range(self.options.sessions)])
        
        quit()
コード例 #47
0
ファイル: channels.py プロジェクト: nereati/metagam3d
def blocking(func, *args, **kwargs):
    ch = Channel()
    func(ch, *args, **kwargs)
    return ch.receive()
コード例 #48
0
def slideCameraStart(interval, eye, lookat):
    return _slideCamera(Channel(), interval, eye, lookat)