예제 #1
0
 def __init__(self, account):
     self.account = account
     self.started = False
     self.active = False
     self.subscribed = False
     self._command_proc = None
     self._command_channel = coros.queue()
     self._data_channel = coros.queue()
     self._subscription = None
     self._subscription_proc = None
     self._subscription_timer = None
예제 #2
0
 def __init__(self, account):
     self.account = account
     self.started = False
     self.active = False
     self.registered = False
     self._command_proc = None
     self._command_channel = coros.queue()
     self._data_channel = coros.queue()
     self._registration = None
     self._dns_wait = 1
     self._register_wait = 1
     self._registration_timer = None
예제 #3
0
 def __init__(self, sip_identity, xmpp_identity):
     self.ended = False
     self.sip_identity = sip_identity
     self.xmpp_identity = xmpp_identity
     self.subscribed = False
     self._command_proc = None
     self._command_channel = coros.queue()
     self._data_channel = coros.queue()
     self._sip_subscription = None
     self._sip_subscription_proc = None
     self._sip_subscription_timer = None
     self._xmpp_subscription = None
예제 #4
0
 def __init__(self, account):
     self.account = account
     self.started = False
     self.active = False
     self.publishing = False
     self._lock = Lock()
     self._command_proc = None
     self._command_channel = coros.queue()
     self._data_channel = coros.queue()
     self._publication = None
     self._dns_wait = 1
     self._publish_wait = 1
     self._publication_timer = None
     self.__dict__['state'] = None
예제 #5
0
    def __init__(self, protocol):
        self.account = DefaultAccount()
        self._protocol = protocol

        self._id = None
        self._local_identity = None
        self._remote_identity = None
        self._local_jid = None
        self._remote_jid = None

        self._channel = coros.queue()
        self._current_operation = None
        self._proc = proc.spawn(self._run)
        self._timer = None

        self._sdp_negotiator = None
        self._pending_transport_info_stanzas = []

        self.direction = None
        self.state = None
        self.streams = None
        self.proposed_streams = None
        self.start_time = None
        self.end_time = None
        self.on_hold = False
        self.local_focus = False
예제 #6
0
 def _NH_MediaStreamDidNotInitialize(self, notification):
     message_queue, self.message_queue = self.message_queue, queue()
     while message_queue:
         message = message_queue.wait()
         if message.notify_progress:
             data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream was closed')
             notification.center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
예제 #7
0
    def __init__(self, protocol):
        self.account = DefaultAccount()
        self._protocol = protocol

        self._id = None
        self._local_identity = None
        self._remote_identity = None
        self._local_jid = None
        self._remote_jid = None

        self._channel = coros.queue()
        self._current_operation = None
        self._proc = proc.spawn(self._run)
        self._timer = None

        self._sdp_negotiator = None
        self._pending_transport_info_stanzas = []

        self.direction = None
        self.state = None
        self.streams = None
        self.proposed_streams = None
        self.start_time = None
        self.end_time = None
        self.on_hold = False
        self.local_focus = False
        self.candidates = set()
예제 #8
0
 def __init__(self):
     super(ChatStream, self).__init__(direction='sendrecv')
     self.message_queue = queue()
     self.sent_messages = set()
     self.incoming_queue = defaultdict(list)
     self.message_queue_thread = None
     self.encryption = OTREncryption(self)
예제 #9
0
    def test_blocks_on_pool(self):
        waiter = coros.queue(0)

        def greedy():
            self.pool.get()
            self.pool.get()
            self.pool.get()
            self.pool.get()
            # No one should be waiting yet.
            self.assertEqual(self.pool.waiting(), 0)
            # The call to the next get will unschedule this routine.
            self.pool.get()
            # So this send should never be called.
            waiter.send('Failed!')

        killable = api.spawn(greedy)

        # no one should be waiting yet.
        self.assertEqual(self.pool.waiting(), 0)

        ## Wait for greedy
        api.sleep(0)

        ## Greedy should be blocking on the last get
        self.assertEqual(self.pool.waiting(), 1)

        ## Send will never be called, so balance should be 0.
        self.assertFalse(waiter.ready())

        api.kill(killable)
예제 #10
0
 def _NH_MediaStreamDidNotInitialize(self, notification):
     message_queue, self.message_queue = self.message_queue, queue()
     while message_queue:
         message = message_queue.wait()
         if message.notify_progress:
             data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream was closed')
             notification.center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
예제 #11
0
    def fan(self, block, input_list):
        queue = coros.queue(0)
        results = []
        exceptional_results = 0
        for index, input_item in enumerate(input_list):
            pool_item = self.get()

            ## Fan out
            api.spawn(self._invoke, block, pool_item, input_item, index, queue)

        ## Fan back in
        for i in range(len(input_list)):
            ## Wait for all guys to send to the queue
            index, value = queue.wait()
            if isinstance(value, Exception):
                exceptional_results += 1
            results.append((index, value))

        results.sort()
        results = [value for index, value in results]

        if exceptional_results:
            if exceptional_results == len(results):
                raise AllFailed(results)
            raise SomeFailed(results)
        return results
예제 #12
0
    def fan(self, block, input_list):
        queue = coros.queue(0)
        results = []
        exceptional_results = 0
        for index, input_item in enumerate(input_list):
            pool_item = self.get()

            ## Fan out
            api.spawn(
                self._invoke, block, pool_item, input_item, index, queue)

        ## Fan back in
        for i in range(len(input_list)):
            ## Wait for all guys to send to the queue
            index, value = queue.wait()
            if isinstance(value, Exception):
                exceptional_results += 1
            results.append((index, value))

        results.sort()
        results = [value for index, value in results]

        if exceptional_results:
            if exceptional_results == len(results):
                raise AllFailed(results)
            raise SomeFailed(results)
        return results
예제 #13
0
 def __init__(self):
     super(ChatStream, self).__init__(direction='sendrecv')
     self.message_queue = queue()
     self.sent_messages = set()
     self.incoming_queue = defaultdict(list)
     self.message_queue_thread = None
     self.encryption = OTREncryption(self)
예제 #14
0
    def test_multiple_waiters(self):
        # tests that multiple waiters get their results back
        q = coros.queue()

        def waiter(q, evt):
            evt.send(q.wait())

        sendings = ['1', '2', '3', '4']
        evts = [coros.event() for x in sendings]
        for i, x in enumerate(sendings):
            api.spawn(waiter, q, evts[i])

        api.sleep(0.01)  # get 'em all waiting

        results = set()

        def collect_pending_results():
            for i, e in enumerate(evts):
                timer = api.exc_after(0.001, api.TimeoutError)
                try:
                    x = e.wait()
                    results.add(x)
                    timer.cancel()
                except api.TimeoutError:
                    pass  # no pending result at that event
            return len(results)

        q.send(sendings[0])
        self.assertEquals(collect_pending_results(), 1)
        q.send(sendings[1])
        self.assertEquals(collect_pending_results(), 2)
        q.send(sendings[2])
        q.send(sendings[3])
        self.assertEquals(collect_pending_results(), 4)
예제 #15
0
 def __init__(self, local_identity, remote_identity):
     self.local_identity = local_identity
     self.remote_identity = remote_identity
     self.state = None
     self.channel = coros.queue()
     self._proc = None
     from sylk.applications.xmppgateway.xmpp import XMPPManager
     self.xmpp_manager = XMPPManager()
예제 #16
0
 def __init__(self, local_identity, remote_identity):
     self.local_identity = local_identity
     self.remote_identity = remote_identity
     self.state = None
     self.channel = coros.queue()
     self._proc = None
     from sylk.applications.xmppgateway.xmpp import XMPPManager
     self.xmpp_manager = XMPPManager()
예제 #17
0
 def __init__(self):
     self._stopped = True
     self._files = []
     self._command_channel = coros.queue()
     self._select_proc = None
     self._register_timer = None
     self._update_timer = None
     self._lock = Lock()
예제 #18
0
    def test_senders_that_die(self):
        q = coros.queue()

        def do_send(q):
            q.send('sent')

        api.spawn(do_send, q)
        self.assertEqual(q.wait(), 'sent')
예제 #19
0
파일: muc.py 프로젝트: samykabu/sylkserver
 def __init__(self, sender, recipient, participant):
     self.sender = sender
     self.recipient = recipient
     self.participant = participant
     self.active = False
     self.route = None
     self._channel = coros.queue()
     self._referral = None
     self._failure = None
예제 #20
0
 def __init__(self):
     BlinkLogger().log_debug('Starting Bonjour Conference Services')
     self._stopped = True
     self._files = []
     self._servers = {}
     self._command_channel = coros.queue()
     self._select_proc = None
     self._discover_timer = None
     self._wakeup_timer = None
예제 #21
0
 def __init__(self):
     BlinkLogger().log_debug('Starting Bonjour Conference Services')
     self._stopped = True
     self._files = []
     self._servers = {}
     self._command_channel = coros.queue()
     self._select_proc = None
     self._discover_timer = None
     self._wakeup_timer = None
예제 #22
0
 def _NH_MediaStreamDidEnd(self, notification):
     if self.message_queue_thread is not None:
         self.message_queue_thread.kill()
     else:
         message_queue, self.message_queue = self.message_queue, queue()
         while message_queue:
             message = message_queue.wait()
             if message.notify_progress:
                 data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream ended')
                 notification.center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
예제 #23
0
 def __init__(self, min_size=0, max_size=4, track_events=False):
     if min_size > max_size:
         raise ValueError('min_size cannot be bigger than max_size')
     self.max_size = max_size
     self.sem = coros.Semaphore(max_size)
     self.procs = proc.RunningProcSet()
     if track_events:
         self.results = coros.queue()
     else:
         self.results = None
예제 #24
0
 def __init__(self, min_size=0, max_size=4, track_events=False):
     if min_size > max_size:
         raise ValueError('min_size cannot be bigger than max_size')
     self.max_size = max_size
     self.sem = coros.Semaphore(max_size)
     self.procs = proc.RunningProcSet()
     if track_events:
         self.results = coros.queue()
     else:
         self.results = None
예제 #25
0
 def test_send(self):
     s = proc.Source()
     q1, q2, q3 = coros.queue(), coros.queue(), coros.queue()
     s.link_value(q1)
     self.assertRaises(api.TimeoutError, s.wait, 0)
     assert s.wait(0, None) is None
     assert s.wait(0.001, None) is None
     self.assertRaises(api.TimeoutError, s.wait, 0.001)
     s.send(1)
     assert not q1.ready()
     assert s.wait() == 1
     api.sleep(0)
     assert q1.ready()
     s.link_exception(q2)
     s.link(q3)
     assert not q2.ready()
     api.sleep(0)
     assert q3.ready()
     assert s.wait() == 1
예제 #26
0
 def _NH_MediaStreamDidEnd(self, notification):
     if self.message_queue_thread is not None:
         self.message_queue_thread.kill()
     else:
         message_queue, self.message_queue = self.message_queue, queue()
         while message_queue:
             message = message_queue.wait()
             if message.notify_progress:
                 data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream ended')
                 notification.center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
예제 #27
0
파일: api.py 프로젝트: madhawa/sylkserver
 def onOpen(self):
     self.factory.connections.add(self)
     self.accounts_map = {}
     self.account_handles_map = {}
     self.sessions_map = {}
     self.session_handles_map = {}
     self.ready_event = GreenEvent()
     self.resolver = DNSLookup()
     self.proc = proc.spawn(self._operations_handler)
     self.operations_queue = coros.queue()
     self._create_janus_session()
예제 #28
0
 def play(self):
     if self._state != 'stopped':
         raise WavePlayerError('already playing')
     self._state = 'started'
     self._channel = coros.queue()
     self._current_loop = 0
     if self.initial_delay:
         reactor.callLater(self.initial_delay, self._channel.send, Command('play'))
     else:
         self._channel.send(Command('play'))
     self._run().wait()
예제 #29
0
 def __init__(self, logger):
     ConnectBase.__init__(self, logger)
     self.ports = {}  # maps interface -> port -> (use_tls, listening Port)
     self.queue = coros.queue()
     self.expected_local_uris = {}  # maps local_uri -> Logger instance
     self.expected_remote_paths = {}  # maps full_remote_path -> event
     self.new_full_remote_path_notifier = Notifier()
     self.factory = SpawnFactory(self._incoming_handler,
                                 MSRPTransport,
                                 local_uri=None,
                                 logger=self.logger)
예제 #30
0
파일: im.py 프로젝트: turbojavaC/sylkserver
 def __init__(self, message):
     # TODO: sometimes we may want to send it to the GRUU, for example when a XMPP client
     # replies to one of our messages. MESSAGE requests don't need a Contact header, though
     # so how should we communicate our GRUU to the recipient?
     self.from_uri = message.sender.uri.as_sip_uri()
     self.from_uri.parameters.pop('gr', None)  # No GRUU in From header
     self.to_uri = message.recipient.uri.as_sip_uri()
     self.to_uri.parameters.pop('gr', None)  # Don't send it to the GRUU
     self.body = message.body
     self.content_type = 'text/plain'
     self._requests = set()
     self._channel = coros.queue()
예제 #31
0
 def play(self):
     if self._state != 'stopped':
         raise WavePlayerError('already playing')
     self._state = 'started'
     self._channel = coros.queue()
     self._current_loop = 0
     if self.initial_delay:
         reactor.callLater(self.initial_delay, self._channel.send,
                           Command('play'))
     else:
         self._channel.send(Command('play'))
     self._run().wait()
예제 #32
0
파일: im.py 프로젝트: madhawa/sylkserver
 def __init__(self, message):
     # TODO: sometimes we may want to send it to the GRUU, for example when a XMPP client
     # replies to one of our messages. MESSAGE requests don't need a Contact header, though
     # so how should we communicate our GRUU to the recipient?
     self.from_uri = message.sender.uri.as_sip_uri()
     self.from_uri.parameters.pop('gr', None)    # No GRUU in From header
     self.to_uri = message.recipient.uri.as_sip_uri()
     self.to_uri.parameters.pop('gr', None)      # Don't send it to the GRUU
     self.body = message.body
     self.content_type = 'text/plain'
     self._requests = set()
     self._channel = coros.queue()
예제 #33
0
    def test_send_last(self):
        q = coros.queue()

        def waiter(q):
            timer = api.exc_after(0.1, api.TimeoutError)
            self.assertEqual(q.wait(), 'hi2')
            timer.cancel()

        api.spawn(waiter, q)
        api.sleep(0)
        api.sleep(0)
        q.send('hi2')
예제 #34
0
 def __init__(self, account):
     self.account = account
     self._started = False
     self._files = []
     self._neighbours = {}
     self._command_channel = coros.queue()
     self._select_proc = None
     self._discover_timer = None
     self._register_timer = None
     self._update_timer = None
     self._lock = Lock()
     self.__dict__['presence_state'] = None
예제 #35
0
 def __init__(self, account):
     self.account = account
     self._started = False
     self._files = []
     self._neighbours = {}
     self._command_channel = coros.queue()
     self._select_proc = None
     self._discover_timer = None
     self._register_timer = None
     self._update_timer = None
     self._lock = Lock()
     self.__dict__['presence_state'] = None
예제 #36
0
 def __init__(self):
     default_resolver = InternalResolver()
     self.search = default_resolver.search
     self.domain = default_resolver.domain
     self.nameservers = default_resolver.nameservers
     self.google_nameservers = ['8.8.8.8', '8.8.4.4']
     self.probed_domain = 'sip2sip.info.'
     self._channel = coros.queue()
     self._proc = None
     self._timer = None
     self._wakeup_timer = None
     notification_center = NotificationCenter()
     notification_center.add_observer(self, name='SystemIPAddressDidChange')
     notification_center.add_observer(self, name='SystemDidWakeUpFromSleep')
예제 #37
0
 def __init__(self):
     default_resolver = InternalResolver()
     self.search = default_resolver.search
     self.domain = default_resolver.domain
     self.nameservers = default_resolver.nameservers
     self.google_nameservers = ['8.8.8.8', '8.8.4.4']
     self.probed_domain = 'sip2sip.info.'
     self._channel = coros.queue()
     self._proc = None
     self._timer = None
     self._wakeup_timer = None
     notification_center = NotificationCenter()
     notification_center.add_observer(self, name='SystemIPAddressDidChange')
     notification_center.add_observer(self, name='SystemDidWakeUpFromSleep')
예제 #38
0
 def __init__(self, uri):
     self.uri = uri
     self.identity = ChatIdentity.parse('<sip:%s>' % self.uri)
     self.sessions = []
     self.sessions_with_proposals = []
     self.subscriptions = []
     self.pending_messages = []
     self.state = 'stopped'
     self.incoming_message_queue = coros.queue()
     self.message_dispatcher = None
     self.audio_conference = None
     self.conference_info_payload = None
     self.conference_info_version = count(1)
     self.irc_connector = None
     self.irc_protocol = None
예제 #39
0
 def __init__(self, service='sipfocus', name='SylkServer', uri_user=None, is_focus=True):
     self.account = DefaultAccount()
     self.service = service
     self.name = name
     self.uri_user = uri_user
     self.is_focus = is_focus
     self.id = str(uuid.uuid4())
     self._stopped = True
     self._files = []
     self._command_channel = coros.queue()
     self._select_proc = None
     self._register_timer = None
     self._update_timer = None
     self._lock = Lock()
     self.__dict__['presence_state'] = None
예제 #40
0
    def test_waiting(self):
        def do_wait(q, evt):
            result = q.wait()
            evt.send(result)

        q = coros.queue()
        e1 = coros.event()
        api.spawn(do_wait, q, e1)
        api.sleep(0)
        self.assertEqual(1, waiting(q))
        q.send('hi')
        api.sleep(0)
        self.assertEqual(0, waiting(q))
        self.assertEqual('hi', e1.wait())
        self.assertEqual(0, waiting(q))
예제 #41
0
    def test_waiters_that_cancel(self):
        q = coros.queue()

        def do_receive(q, evt):
            api.exc_after(0, RuntimeError())
            try:
                result = q.wait()
                evt.send(result)
            except RuntimeError:
                evt.send('timed out')

        evt = coros.event()
        api.spawn(do_receive, q, evt)
        self.assertEqual(evt.wait(), 'timed out')

        q.send('hi')
        self.assertEqual(q.wait(), 'hi')
예제 #42
0
    def set_links_timeout(self, link):
        # stuff that won't be touched
        event = coros.event()
        link(event)

        proc_finished_flag = []

        def myproc():
            sleep(10)
            proc_finished_flag.append('finished')
            return 555

        myproc = proc.spawn(myproc)
        link(myproc)

        queue = coros.queue(0)
        link(queue)
        return event, myproc, proc_finished_flag, queue
예제 #43
0
    def test_putting_to_queue(self):
        timer = api.exc_after(0.1, api.TimeoutError)
        size = 2
        self.pool = IntPool(min_size=0, max_size=size)
        queue = coros.queue()
        results = []

        def just_put(pool_item, index):
            self.pool.put(pool_item)
            queue.send(index)

        for index in xrange(size + 1):
            pool_item = self.pool.get()
            api.spawn(just_put, pool_item, index)

        while results != range(size + 1):
            x = queue.wait()
            results.append(x)
        timer.cancel()
예제 #44
0
 def __init__(self,
              service='sipfocus',
              name='SylkServer',
              uri_user=None,
              is_focus=True):
     self.account = DefaultAccount()
     self.service = service
     self.name = name
     self.uri_user = uri_user
     self.is_focus = is_focus
     self.id = str(uuid.uuid4())
     self._stopped = True
     self._files = []
     self._command_channel = coros.queue()
     self._select_proc = None
     self._register_timer = None
     self._update_timer = None
     self._lock = Lock()
     self.__dict__['presence_state'] = None
예제 #45
0
 def __init__(self, msrptransport, accept_types=['*'], on_incoming_cb=None, automatic_reports=True):
     self.msrp = msrptransport
     self.accept_types = accept_types
     self.automatic_reports = automatic_reports
     if not callable(on_incoming_cb):
         raise TypeError('on_incoming_cb must be callable: %r' % on_incoming_cb)
     self._on_incoming_cb = on_incoming_cb
     self.expected_responses = {}
     self.outgoing = coros.queue()
     self.reader_job = proc.spawn(self._reader)
     self.writer_job = proc.spawn(self._writer)
     self.state = 'CONNECTED' # -> 'FLUSHING' -> 'CLOSING' -> 'DONE'
     # in FLUSHING writer sends only while there's something in the outgoing queue
     # then it exits and sets state to 'CLOSING' which makes reader only pay attention
     # to responses and success reports. (XXX it could now discard incoming data chunks
     # with direct write() since writer is dead)
     self.reader_job.link(self.writer_job)
     self.last_expected_response = 0
     self.keepalive_proc = proc.spawn(self._keepalive)
예제 #46
0
def waitall(lst, trap_errors=False, queue=None):
    if queue is None:
        queue = coros.queue()
    index = -1
    for (index, linkable) in enumerate(lst):
        linkable.link(decorate_send(queue, index))
    len = index + 1
    results = [None] * len
    count = 0
    while count < len:
        try:
            index, value = queue.wait()
        except Exception:
            if not trap_errors:
                raise
        else:
            results[index] = value
        count += 1
    return results
예제 #47
0
    def test_two_bogus_waiters(self):
        def do_receive(q, evt):
            api.exc_after(0, RuntimeError())
            try:
                result = q.wait()
                evt.send(result)
            except RuntimeError:
                evt.send('timed out')

        q = coros.queue()
        e1 = coros.event()
        e2 = coros.event()
        api.spawn(do_receive, q, e1)
        api.spawn(do_receive, q, e2)
        api.sleep(0)
        q.send('sent')
        self.assertEqual(e1.wait(), 'timed out')
        self.assertEqual(e2.wait(), 'timed out')
        self.assertEqual(q.wait(), 'sent')
예제 #48
0
 def __init__(self, msrptransport, accept_types=['*'], on_incoming_cb=None, automatic_reports=True):
     self.msrp = msrptransport
     self.accept_types = accept_types
     self.automatic_reports = automatic_reports
     if not callable(on_incoming_cb):
         raise TypeError('on_incoming_cb must be callable: %r' % on_incoming_cb)
     self._on_incoming_cb = on_incoming_cb
     self.expected_responses = {}
     self.outgoing = coros.queue()
     self.reader_job = proc.spawn(self._reader)
     self.writer_job = proc.spawn(self._writer)
     self.state = 'CONNECTED' # -> 'FLUSHING' -> 'CLOSING' -> 'DONE'
     # in FLUSHING writer sends only while there's something in the outgoing queue
     # then it exits and sets state to 'CLOSING' which makes reader only pay attention
     # to responses and success reports. (XXX it could now discard incoming data chunks
     # with direct write() since writer is dead)
     self.reader_job.link(self.writer_job)
     self.last_expected_response = 0
     self.keepalive_proc = proc.spawn(self._keepalive)
예제 #49
0
def waitall(lst, trap_errors=False, queue=None):
    if queue is None:
        queue = coros.queue()
    index = -1
    for (index, linkable) in enumerate(lst):
        linkable.link(decorate_send(queue, index))
    len = index + 1
    results = [None] * len
    count = 0
    while count < len:
        try:
            index, value = queue.wait()
        except Exception:
            if not trap_errors:
                raise
        else:
            results[index] = value
        count += 1
    return results
예제 #50
0
    def test_zero_max_size(self):
        q = coros.queue(0)

        def sender(evt, q):
            q.send('hi')
            evt.send('done')

        def receiver(evt, q):
            x = q.wait()
            evt.send(x)

        e1 = coros.event()
        e2 = coros.event()

        api.spawn(sender, e1, q)
        api.sleep(0)
        self.assertTrue(not e1.ready())
        api.spawn(receiver, e2, q)
        self.assertEqual(e2.wait(), 'hi')
        self.assertEqual(e1.wait(), 'done')
예제 #51
0
    def test_max_size(self):
        q = coros.queue(2)
        results = []

        def putter(q):
            q.send('a')
            results.append('a')
            q.send('b')
            results.append('b')
            q.send('c')
            results.append('c')

        api.spawn(putter, q)
        api.sleep(0)
        self.assertEqual(results, ['a', 'b'])
        self.assertEqual(q.wait(), 'a')
        api.sleep(0)
        self.assertEqual(results, ['a', 'b', 'c'])
        self.assertEqual(q.wait(), 'b')
        self.assertEqual(q.wait(), 'c')
예제 #52
0
    def __init__(self, min_size=0, max_size=4, order_as_stack=False):
        """ Pre-populates the pool with *min_size* items.  Sets a hard limit to
        the size of the pool -- it cannot contain any more items than
        *max_size*, and if there are already *max_size* items 'checked out' of
        the pool, the pool will cause any getter to cooperatively yield until an
        item is put in.

        *order_as_stack* governs the ordering of the items in the free pool.  If
        False (the default), the free items collection (of items that were
        created and were put back in the pool) acts as a round-robin, giving
        each item approximately equal utilization.  If True, the free pool acts
        as a FILO stack, which preferentially re-uses items that have most
        recently been used.
        """
        self.min_size = min_size
        self.max_size = max_size
        self.order_as_stack = order_as_stack
        self.current_size = 0
        self.channel = coros.queue(0)
        self.free_items = collections.deque()
        for x in xrange(min_size):
            self.current_size += 1
            self.free_items.append(self.create())
예제 #53
0
    def generate_results(self, function, iterable, qsize=None):
        """For each tuple (sequence) in iterable, launch function(*tuple) in
        its own coroutine -- like itertools.starmap(), but in parallel.
        Yield each of the values returned by function(), in the order they're
        completed rather than the order the coroutines were launched.

        Iteration stops when we've yielded results for each arguments tuple in
        iterable. Unlike wait_all() and process_all(), this function does not
        wait for any previously-submitted execute() or execute_async() calls.

        Results are temporarily buffered in a queue. If you pass qsize=, this
        value is used to limit the max size of the queue: an attempt to buffer
        too many results will suspend the completed CoroutinePool coroutine
        until the requesting coroutine (the caller of generate_results()) has
        retrieved one or more results by calling this generator-iterator's
        next().

        If any coroutine raises an uncaught exception, that exception will
        propagate to the requesting coroutine via the corresponding next() call.

        What I particularly want these tests to illustrate is that using this
        generator function:

        for result in generate_results(function, iterable):
            # ... do something with result ...

        executes coroutines at least as aggressively as the classic eventlib
        idiom:

        events = [pool.execute(function, *args) for args in iterable]
        for event in events:
            result = event.wait()
            # ... do something with result ...

        even without a distinct event object for every arg tuple in iterable,
        and despite the funny flow control from interleaving launches of new
        coroutines with yields of completed coroutines' results.

        (The use case that makes this function preferable to the classic idiom
        above is when the iterable, which may itself be a generator, produces
        millions of items.)

        >>> from eventlib import coros
        >>> import string
        >>> pool = coros.CoroutinePool(max_size=5)
        >>> pausers = [coros.event() for x in xrange(2)]
        >>> def longtask(evt, desc):
        ...     print "%s woke up with %s" % (desc, evt.wait())
        ...
        >>> pool.launch_all(longtask, zip(pausers, "AB"))
        >>> def quicktask(desc):
        ...     print "returning %s" % desc
        ...     return desc
        ...

        (Instead of using a for loop, step through generate_results()
        items individually to illustrate timing)

        >>> step = iter(pool.generate_results(quicktask, string.ascii_lowercase))
        >>> print step.next()
        returning a
        returning b
        returning c
        a
        >>> print step.next()
        b
        >>> print step.next()
        c
        >>> print step.next()
        returning d
        returning e
        returning f
        d
        >>> pausers[0].send("A")
        >>> print step.next()
        e
        >>> print step.next()
        f
        >>> print step.next()
        A woke up with A
        returning g
        returning h
        returning i
        g
        >>> print "".join([step.next() for x in xrange(3)])
        returning j
        returning k
        returning l
        returning m
        hij
        >>> pausers[1].send("B")
        >>> print "".join([step.next() for x in xrange(4)])
        B woke up with B
        returning n
        returning o
        returning p
        returning q
        klmn
        """
        # Get an iterator because of our funny nested loop below. Wrap the
        # iterable in enumerate() so we count items that come through.
        tuples = iter(enumerate(iterable))
        # If the iterable is empty, this whole function is a no-op, and we can
        # save ourselves some grief by just quitting out. In particular, once
        # we enter the outer loop below, we're going to wait on the queue --
        # but if we launched no coroutines with that queue as the destination,
        # we could end up waiting a very long time.
        try:
            index, args = tuples.next()
        except StopIteration:
            return
        # From this point forward, 'args' is the current arguments tuple and
        # 'index+1' counts how many such tuples we've seen.
        # This implementation relies on the fact that _execute() accepts an
        # event-like object, and -- unless it's None -- the completed
        # coroutine calls send(result). We slyly pass a queue rather than an
        # event -- the same queue instance for all coroutines. This is why our
        # queue interface intentionally resembles the event interface.
        q = coros.queue(max_size=qsize)
        # How many results have we yielded so far?
        finished = 0
        # This first loop is only until we've launched all the coroutines. Its
        # complexity is because if iterable contains more args tuples than the
        # size of our pool, attempting to _execute() the (poolsize+1)th
        # coroutine would suspend until something completes and send()s its
        # result to our queue. But to keep down queue overhead and to maximize
        # responsiveness to our caller, we'd rather suspend on reading the
        # queue. So we stuff the pool as full as we can, then wait for
        # something to finish, then stuff more coroutines into the pool.
        try:
            while True:
                # Before each yield, start as many new coroutines as we can fit.
                # (The self.free() test isn't 100% accurate: if we happen to be
                # executing in one of the pool's coroutines, we could _execute()
                # without waiting even if self.free() reports 0. See _execute().)
                # The point is that we don't want to wait in the _execute() call,
                # we want to wait in the q.wait() call.
                # IMPORTANT: at start, and whenever we've caught up with all
                # coroutines we've launched so far, we MUST iterate this inner
                # loop at least once, regardless of self.free() -- otherwise the
                # q.wait() call below will deadlock!
                # Recall that index is the index of the NEXT args tuple that we
                # haven't yet launched. Therefore it counts how many args tuples
                # we've launched so far.
                while self.free() > 0 or finished == index:
                    # Just like the implementation of execute_async(), save that
                    # we're passing our queue instead of None as the "event" to
                    # which to send() the result.
                    self._execute(q, function, args, {})
                    # We've consumed that args tuple, advance to next.
                    index, args = tuples.next()
                # Okay, we've filled up the pool again, yield a result -- which
                # will probably wait for a coroutine to complete. Although we do
                # have q.ready(), so we could iterate without waiting, we avoid
                # that because every yield could involve considerable real time.
                # We don't know how long it takes to return from yield, so every
                # time we do, take the opportunity to stuff more requests into the
                # pool before yielding again.
                yield q.wait()
                # Be sure to count results so we know when to stop!
                finished += 1
        except StopIteration:
            pass
        # Here we've exhausted the input iterable. index+1 is the total number
        # of coroutines we've launched. We probably haven't yielded that many
        # results yet. Wait for the rest of the results, yielding them as they
        # arrive.
        while finished < index + 1:
            yield q.wait()
            finished += 1
예제 #54
0
 def __init__(self, **params):
     for name, value in params.iteritems():
         setattr(self, name, value)
     for param in set(self.__params__).difference(params):
         raise ValueError("missing operation parameter: '%s'" % param)
     self.channel = coros.queue()
예제 #55
0
                except ProcExit:
                    if notify_progress:
                        data = NotificationData(message_id=message_id, message=None, code=0, reason='Stream ended')
                        notification_center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
                    raise
                else:
                    if notify_progress:
                        self.sent_messages.add(message_id)
                        notification_center.post_notification('ChatStreamDidSendMessage', sender=self, data=NotificationData(message=chunk))
        finally:
            self.message_queue_thread = None
            while self.sent_messages:
                message_id = self.sent_messages.pop()
                data = NotificationData(message_id=message_id, message=None, code=0, reason='Stream ended')
                notification_center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
            message_queue, self.message_queue = self.message_queue, queue()
            while message_queue:
                message = message_queue.wait()
                if message.notify_progress:
                    data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream ended')
                    notification_center.post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)

    @run_in_twisted_thread
    def _enqueue_message(self, message):
        if self._done:
            if message.notify_progress:
                data = NotificationData(message_id=message.id, message=None, code=0, reason='Stream ended')
                NotificationCenter().post_notification('ChatStreamDidNotDeliverMessage', sender=self, data=data)
        else:
            self.message_queue.send(message)