class TestQueueTimeouts(object): def setup(self): self.result = Event() self.queue = Queue() self.timeouts = 0 diesel.fork(self.consumer, 0.01) diesel.fork(self.producer, 0.05) diesel.fork(self.consumer, 0.10) ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result]) if ev == 'sleep': assert 0, 'timed out' def consumer(self, timeout): try: self.queue.get(timeout=timeout) self.result.set() except QueueTimeout: self.timeouts += 1 def producer(self, delay): diesel.sleep(delay) self.queue.put('test') def test_a_consumer_timed_out(self): assert self.timeouts == 1 def test_a_consumer_got_a_value(self): assert self.result.is_set
class ChatClient(Client): def __init__(self, *args, **kw): Client.__init__(self, *args, **kw) self.input = Queue() def read_chat_message(self, prompt): msg = raw_input(prompt) return msg def input_handler(self): nick = thread(self.read_chat_message, "nick: ").strip() self.nick = nick self.input.put(nick) while True: msg = thread(self.read_chat_message, "").strip() self.input.put(msg) @call def chat(self): fork(self.input_handler) nick = self.input.get() send("%s\r\n" % nick) while True: evt, data = first(until_eol=True, waits=[self.input]) if evt == "until_eol": print data.strip() else: send("%s\r\n" % data)
def do_upgrade(self, req): if req.headers.get_one('Upgrade') != 'WebSocket': return self.web_handler(req) # do upgrade response org = req.headers.get_one('Origin') send( '''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r WebSocket-Origin: %s\r WebSocket-Location: %s\r WebSocket-Protocol: diesel-generic\r \r ''' % (org, self.ws_location)) inq = Queue() outq = Queue() def wrap(inq, outq): self.web_socket_handler(inq, outq) outq.put(WebSocketDisconnect()) fork(wrap, inq, outq) while True: try: typ, val = first(receive=1, waits=[outq.wait_id]) if typ == 'receive': assert val == '\x00' val = until('\xff')[:-1] if val == '': inq.put(WebSocketDisconnect()) else: data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems()) inq.put(WebSocketData(data)) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: send('\x00\xff') break else: data = dumps(dict(v)) send('\x00%s\xff' % data) except ConnectionClosed: inq.put(WebSocketDisconnect()) raise ConnectionClosed("remote disconnected")
class ConvoyNameService(object): def __init__(self, servers): self.servers = servers self.request_queue = Queue() self.pool_locks = {} def __call__(self): while True: server = random.choice(self.servers) with ConvoyConsensusClient(*server) as client: while True: req, rq = self.request_queue.get() if type(req) is ConvoyGetRequest: resp = client.get(req.key) elif type(req) is ConvoySetRequest: resp = client.add_to_set(req.key, req.value, req.cap, req.timeout, req.lock) elif type(req) is ConvoyWaitRequest: resp = client.wait(req.timeout, req.clocks) elif type(req) is ConvoyAliveRequest: resp = client.keep_alive() else: assert 0 rq.put(resp) def lookup(self, key): rq = Queue() self.request_queue.put((ConvoyGetRequest(key), rq)) return rq.get() def clear(self, key): rq = Queue() self.request_queue.put((ConvoySetRequest(key, None, 0, 5, 0), rq)) return rq.get() def set(self, key, value): rq = Queue() self.request_queue.put((ConvoySetRequest(key, value, 0, 5, 0), rq)) return rq.get() def add(self, key, value, cap, to=0): rq = Queue() self.request_queue.put((ConvoySetRequest(key, value, cap, to, 1), rq)) return rq.get() def wait(self, timeout, clocks): rq = Queue() self.request_queue.put((ConvoyWaitRequest(timeout, clocks), rq)) return rq.get() def alive(self): rq = Queue() self.request_queue.put((ConvoyAliveRequest(), rq)) return rq.get()
def network_set(self, client, key, value, new): proposal_id = idgen.next() resq = Queue() if new: rollback = '|' + new + ':' + proposal_id value += rollback else: rollback = None for q in self.proposal_qs: q.put((proposal_id, key, resq)) success = 0 while True: # XXX timeout etc v = resq.get() if v == PROPOSE_SUCCESS: success += 1 if success == self.quorum_size: break elif v == PROPOSE_FAIL: return None else: assert 0 for q in self.save_qs: q.put((proposal_id, key, value, client, rollback, resq)) success = 0 while True: # XXX timeout etc v = resq.get() if v == PROPOSE_SUCCESS: pass # don't care elif v == PROPOSE_FAIL: pass # don't care elif v == SAVE_SUCCESS: success += 1 if success == self.quorum_size: return proposal_id else: assert 0
class ProcessPool(object): """A bounded pool of subprocesses. An instance is callable, just like a Process, and will return the result of executing the function in a subprocess. If all subprocesses are busy, the caller will wait in a queue. """ def __init__(self, concurrency, handler): """Creates a new ProcessPool with subprocesses that run the handler. Args: concurrency (int): The number of subprocesses to spawn. handler (callable): A callable that the subprocesses will execute. """ self.concurrency = concurrency self.handler = handler self.available_procs = Queue() self.all_procs = [] def __call__(self, *args, **params): """Gets a process from the pool, executes it, and returns the results. This call will block until there is a process available to handle it. """ if not self.all_procs: raise NoSubProcesses("Did you forget to start the pool?") try: p = self.available_procs.get() result = p(*args, **params) return result finally: self.available_procs.put(p) def pool(self): """A callable that starts the processes in the pool. This is useful as the callable to pass to a diesel.Loop when adding a ProcessPool to your application. """ for i in xrange(self.concurrency): proc = spawn(self.handler) self.available_procs.put(proc) self.all_procs.append(proc)
def handle(request): """Handle a request for a websocket. """ if request.transport != 'xhr-polling': raise Response(404) org = request.headers.one('Origin') inq = Queue() outq = Queue() def wrap(request, inq, outq): handler(request, inq, outq) outq.put(WebSocketDisconnect()) fork(wrap, request, inq, outq) while True: try: log.debug("trying websocket thing") typ, val = first(receive=1, waits=[outq.wait_id]) log.debug(typ) log.debug(val) if typ == 'receive': assert val == '\x00' val = until('\xff')[:-1] if val == '': inq.put(WebSocketDisconnect()) else: inq.put(request) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: send('\x00\xff') break else: send('\x00%s\xff' % response.to_http(request.version)) except ConnectionClosed: inq.put(WebSocketDisconnect()) raise ConnectionClosed("remote disconnected")
def get_many(self, keys, concurrency_limit=100, no_failures=False): assert self.used_client_context, "Cannot fetch in parallel without a pooled make_client_context!" inq = Queue() outq = Queue() for k in keys: inq.put(k) for x in xrange(min(len(keys), concurrency_limit)): diesel.fork(self._subrequest, inq, outq) failure = False okay, err = [], [] for k in keys: (key, success, val) = outq.get() if success: okay.append((key, val)) else: err.append((key, val)) if no_failures: raise BucketSubrequestException("Error in parallel subrequests", err) return okay, err
def network_get(self, key): answers = defaultdict(int) resq = Queue() for gq in self.get_qs: gq.put((key, resq)) ans = None # XXX - timeout for x in xrange(self.num_hosts): value = resq.get() answers[value] += 1 if answers[value] == self.quorum_size: ans = value break if ans is not None and (key not in store or store[key].proposal_id < ans.proposal_id): clog.error("read-repair %s" % ans) store.set(key, ans) return ans
class QueueHarness(object): def setup(self): self.queue = Queue() self.done = Countdown(N) self.results = [] self.handled = defaultdict(int) self.populate() self.consume() self.trigger() def consume(self): def worker(myid): while True: # Test both queue.get and wait() on queue (both are valid # APIs for getting items from the queue). The results should # be the same. if random.random() > 0.5: v = self.queue.get() else: v = diesel.wait(self.queue) self.results.append(v) self.handled[myid] += 1 self.done.tick() for i in xrange(W): diesel.fork(worker, i) def trigger(self): ev, val = diesel.first(sleep=TIMEOUT, waits=[self.done]) if ev == 'sleep': assert 0, "timed out" def test_results_are_ordered_as_expected(self): assert self.results == range(N), self.results def test_results_are_balanced(self): for wid, count in self.handled.iteritems(): assert count == N / W, count
class QueueHarness(object): def setup(self): self.queue = Queue() self.done = Countdown(N) self.results = [] self.handled = defaultdict(int) self.populate() self.consume() self.trigger() def consume(self): def worker(myid): while True: # Test both queue.get and wait() on queue (both are valid # APIs for getting items from the queue). The results should # be the same. if random.random() > 0.5: v = self.queue.get() else: v = diesel.wait(self.queue) self.results.append(v) self.handled[myid] += 1 self.done.tick() for i in xrange(W): diesel.fork(worker, i) def trigger(self): ev, val = diesel.first(sleep=TIMEOUT, waits=[self.done]) if ev == 'sleep': assert 0, "timed out" def test_results_are_ordered_as_expected(self): assert self.results == range(N), self.results def test_results_are_balanced(self): for wid, count in self.handled.iteritems(): assert count == N/W, count
def get_many(self, keys, concurrency_limit=100, no_failures=False): assert self.used_client_context,\ "Cannot fetch in parallel without a pooled make_client_context!" inq = Queue() outq = Queue() for k in keys: inq.put(k) for x in xrange(min(len(keys), concurrency_limit)): diesel.fork(self._subrequest, inq, outq) failure = False okay, err = [], [] for k in keys: (key, success, val) = outq.get() if success: okay.append((key, val)) else: err.append((key, val)) if no_failures: raise BucketSubrequestException("Error in parallel subrequests", err) return okay, err
def alive(self): rq = Queue() self.request_queue.put((ConvoyAliveRequest(), rq)) return rq.get()
def wait(self, timeout, clocks): rq = Queue() self.request_queue.put((ConvoyWaitRequest(timeout, clocks), rq)) return rq.get()
def add(self, key, value, cap, to=0): rq = Queue() self.request_queue.put((ConvoySetRequest(key, value, cap, to, 1), rq)) return rq.get()
def set(self, key, value): rq = Queue() self.request_queue.put((ConvoySetRequest(key, value, 0, 5, 0), rq)) return rq.get()
def clear(self, key): rq = Queue() self.request_queue.put((ConvoySetRequest(key, None, 0, 5, 0), rq)) return rq.get()
def lookup(self, key): rq = Queue() self.request_queue.put((ConvoyGetRequest(key), rq)) return rq.get()
def do_upgrade(self, req): if req.headers.get_one('Upgrade') != 'WebSocket': return self.web_handler(req) # do upgrade response org = req.headers.get_one('Origin') if 'Sec-WebSocket-Key1' in req.headers: protocol = (req.headers.get_one('Sec-WebSocket-Protocol') if 'Sec-WebSocket-Protocol' in req.headers else None) key1 = req.headers.get_one('Sec-WebSocket-Key1') key2 = req.headers.get_one('Sec-WebSocket-Key2') key3 = receive(8) num1 = int(''.join(c for c in key1 if c in '0123456789')) num2 = int(''.join(c for c in key2 if c in '0123456789')) assert num1 % key1.count(' ') == 0 assert num2 % key2.count(' ') == 0 final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3) secure_response = hashlib.md5(final).digest() send('''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r Sec-WebSocket-Origin: %s\r Sec-WebSocket-Location: %s\r ''' % (org, self.ws_location)) if protocol: send("Sec-WebSocket-Protocol: %s\r\n" % (protocol, )) send("\r\n") send(secure_response) else: send('''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r WebSocket-Origin: %s\r WebSocket-Location: %s\r WebSocket-Protocol: diesel-generic\r \r ''' % (org, self.ws_location)) inq = Queue() outq = Queue() def wrap(inq, outq): self.web_socket_handler(inq, outq) outq.put(WebSocketDisconnect()) fork(wrap, inq, outq) while True: try: typ, val = first(receive=1, waits=[outq.wait_id]) if typ == 'receive': assert val == '\x00' val = until('\xff')[:-1] if val == '': inq.put(WebSocketDisconnect()) else: data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems()) inq.put(WebSocketData(data)) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: send('\x00\xff') break else: data = dumps(dict(v)) send('\x00%s\xff' % data) except ConnectionClosed: inq.put(WebSocketDisconnect()) raise ConnectionClosed("remote disconnected")
def do_upgrade(self, req): if req.headers.get_one('Upgrade') != 'websocket' and req.headers.get_one('Upgrade') != 'WebSocket': return self.web_handler(req) hybi = False # do upgrade response org = req.headers.get_one('Origin') if 'Sec-WebSocket-Key' in req.headers: assert req.headers.get_one('Sec-WebSocket-Version') == '8', \ "We currently only support version 8 and below" protocol = (req.headers.get_one('Sec-WebSocket-Protocol') if 'Sec-WebSocket-Protocol' in req.headers else None) key = req.headers.get_one('Sec-WebSocket-Key') accept = b64encode(hashlib.sha1(key + self.GUID).digest()) send(server_handshake_hybi % accept) if protocol: send("Sec-WebSocket-Protocol: %s\r" % protocol) send("\r\n\r\n") hybi = True elif 'Sec-WebSocket-Key1' in req.headers: protocol = (req.headers.get_one('Sec-WebSocket-Protocol') if 'Sec-WebSocket-Protocol' in req.headers else None) key1 = req.headers.get_one('Sec-WebSocket-Key1') key2 = req.headers.get_one('Sec-WebSocket-Key2') key3 = receive(8) num1 = int(''.join(c for c in key1 if c in '0123456789')) num2 = int(''.join(c for c in key2 if c in '0123456789')) assert num1 % key1.count(' ') == 0 assert num2 % key2.count(' ') == 0 final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3) secure_response = hashlib.md5(final).digest() send( '''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r Sec-WebSocket-Origin: %s\r Sec-WebSocket-Location: %s\r '''% (org, self.ws_location)) if protocol: send("Sec-WebSocket-Protocol: %s\r\n" % (protocol,)) send("\r\n") send(secure_response) else: send( '''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r WebSocket-Origin: %s\r WebSocket-Location: %s\r WebSocket-Protocol: diesel-generic\r \r ''' % (org, self.ws_location)) inq = Queue() outq = Queue() def wrap(inq, outq): self.web_socket_handler(inq, outq) outq.put(WebSocketDisconnect()) fork(wrap, inq, outq) while True: try: if hybi: typ, val = first(receive=2, waits=[outq.wait_id]) if typ == 'receive': b1, b2 = unpack(">BB", val) opcode = b1 & 0x0f fin = (b1 & 0x80) >> 7 has_mask = (b2 & 0x80) >> 7 assert has_mask == 1, "Frames must be masked" if opcode == 8: inq.put(WebSocketDisconnect()) else: assert opcode == 1, "Currently only opcode 1 is supported" length = b2 & 0x7f if length == 126: length = unpack('>H', receive(2)) elif length == 127: length = unpack('>L', receive(8)) mask = unpack('>BBBB', receive(4)) payload = array('>B', receive(length)) for i in xrange(len(payload)): payload[i] ^= mask[i % 4] data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(payload.tostring()).iteritems()) inq.put(WebSocketData(data)) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: b1 = 0x80 | (8 & 0x0f) # FIN + opcode send(pack('>BB', b1, 0)) break else: payload = dumps(v) b1 = 0x80 | (1 & 0x0f) # FIN + opcode payload_len = len(payload) if payload_len <= 125: header = pack('>BB', b1, payload_len) elif payload_len > 125 and payload_len < 65536: header = pack('>BBH', b1, 126, payload_len) elif payload_len >= 65536: header = pack('>BBQ', b1, 127, payload_len) send(header + payload) else: typ, val = first(receive=1, waits=[outq.wait_id]) if typ == 'receive': assert val == '\x00' val = until('\xff')[:-1] if val == '': inq.put(WebSocketDisconnect()) else: data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems()) inq.put(WebSocketData(data)) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: send('\x00\xff') break else: data = dumps(dict(v)) send('\x00%s\xff' % data) except ConnectionClosed: inq.put(WebSocketDisconnect()) raise ConnectionClosed("remote disconnected")
def do_upgrade(self, req): if req.headers.get_one('Upgrade') != 'WebSocket': return self.web_handler(req) # do upgrade response org = req.headers.get_one('Origin') if 'Sec-WebSocket-Key1' in req.headers: protocol = (req.headers.get_one('Sec-WebSocket-Protocol') if 'Sec-WebSocket-Protocol' in req.headers else None) key1 = req.headers.get_one('Sec-WebSocket-Key1') key2 = req.headers.get_one('Sec-WebSocket-Key2') key3 = receive(8) num1 = int(''.join(c for c in key1 if c in '0123456789')) num2 = int(''.join(c for c in key2 if c in '0123456789')) assert num1 % key1.count(' ') == 0 assert num2 % key2.count(' ') == 0 final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3) secure_response = hashlib.md5(final).digest() send( '''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r Sec-WebSocket-Origin: %s\r Sec-WebSocket-Location: %s\r '''% (org, self.ws_location)) if protocol: send("Sec-WebSocket-Protocol: %s\r\n" % (protocol,)) send("\r\n") send(secure_response) else: send( '''HTTP/1.1 101 Web Socket Protocol Handshake\r Upgrade: WebSocket\r Connection: Upgrade\r WebSocket-Origin: %s\r WebSocket-Location: %s\r WebSocket-Protocol: diesel-generic\r \r ''' % (org, self.ws_location)) inq = Queue() outq = Queue() def wrap(inq, outq): self.web_socket_handler(inq, outq) outq.put(WebSocketDisconnect()) fork(wrap, inq, outq) while True: try: typ, val = first(receive=1, waits=[outq.wait_id]) if typ == 'receive': assert val == '\x00' val = until('\xff')[:-1] if val == '': inq.put(WebSocketDisconnect()) else: data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems()) inq.put(WebSocketData(data)) else: try: v = outq.get(waiting=False) except QueueEmpty: pass else: if type(v) is WebSocketDisconnect: send('\x00\xff') break else: data = dumps(dict(v)) send('\x00%s\xff' % data) except ConnectionClosed: inq.put(WebSocketDisconnect()) raise ConnectionClosed("remote disconnected")