def request(self, method, path, headers, body=None, timeout=None):
'''Issues a `method` request to `path` on the
connected server. Sends along `headers`, and
body.
Very low level--you must set "host" yourself,
for example. It will set Content-Length,
however.
'''
timeout_handler = TimeoutHandler(timeout or 60)
req = HttpRequest(method, path, '1.1')
if body:
headers.set('Content-Length', len(body))
send('%s\r\n%s\r\n\r\n' % (req.format(), headers.format()))
if body:
send(body)
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
resp_line = val
try:
version, code, reason = resp_line.split(None, 2)
except ValueError:
# empty reason string
version, code = resp_line.split(None, 1)
reason = ''
code = int(code)
ev, val = first(until="\r\n\r\n", sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
header_block = val
heads = HttpHeaders()
heads.parse(header_block)
if method == 'HEAD':
body = None
elif heads.get_one('Transfer-Encoding') == 'chunked':
body = handle_chunks(heads, timeout_handler.remaining())
elif heads.get_one(
'Connection') == 'close' and 'Content-Length' not in heads:
body = ''
try:
while True:
s = receive(2**16)
body += s
except ConnectionClosed, e:
if e.buffer:
body += e.buffer
def request(self, method, path, headers, body=None, timeout=None):
'''Issues a `method` request to `path` on the
connected server. Sends along `headers`, and
body.
Very low level--you must set "host" yourself,
for example. It will set Content-Length,
however.
'''
timeout_handler = TimeoutHandler(timeout or 60)
req = HttpRequest(method, path, '1.1')
if body:
headers.set('Content-Length', len(body))
send('%s\r\n%s\r\n\r\n' % (req.format(),
headers.format()))
if body:
send(body)
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
resp_line = val
try:
version, code, reason = resp_line.split(None, 2)
except ValueError:
# empty reason string
version, code = resp_line.split(None, 1)
reason = ''
code = int(code)
ev, val = first(until="\r\n\r\n", sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
header_block = val
heads = HttpHeaders()
heads.parse(header_block)
if method == 'HEAD':
body = None
elif heads.get_one('Transfer-Encoding') == 'chunked':
body = handle_chunks(heads, timeout_handler.remaining())
elif heads.get_one('Connection') == 'close' and 'Content-Length' not in heads:
body = ''
try:
while True:
s = receive(2**16)
body += s
except ConnectionClosed, e:
if e.buffer:
body += e.buffer
def wait(self, timeout=None):
kw = dict(waits=[self])
if timeout:
kw['sleep'] = timeout
mark, data = first(**kw)
if mark != self:
raise EventTimeout()
def pubsub_socket(req, inq, outq):
c = hub.make_client()
with hub.subq('foo') as group:
while True:
q, v = first(waits=[inq, group])
if q == inq: # getting message from client
print "(inq) %s" % v
cmd = v.get("cmd", "")
if cmd == "":
print "published message to %i subscribers" % c.publish(
"foo",
dumps({
'nick': cgi.escape(v['nick'].strip()),
'message': cgi.escape(v['message'].strip()),
}))
else:
outq.put(dict(message="test bot"))
elif q == group: # getting message for broadcasting
chan, msg_str = v
try:
msg = loads(msg_str)
data = dict(message=msg['message'], nick=msg['nick'])
print "(outq) %s" % data
outq.put(data)
except JSONDecodeError:
print "error decoding message %s" % msg_str
elif isinstance(
v, WebSocketDisconnect): # getting a disconnect signal
return
else:
print "oops %s" % v
def get(self, waiting=True, timeout=None):
start = time()
while not self.inp and waiting:
if timeout:
remaining = timeout - (time() - start)
if remaining <= 0:
raise QueueTimeout()
else:
first(waits=[self.wait_id], sleep=remaining)
else:
wait(self.wait_id)
if self.inp:
return self.inp.popleft()
elif not waiting:
raise QueueEmpty()
def _handle_client_requests_and_responses(self, remote_client):
assert self.nitro_socket
queues = [remote_client.incoming]
try:
while True:
(evt, value) = diesel.first(waits=queues, sleep=self.timeout)
if evt is remote_client.incoming:
assert isinstance(value, Message)
remote_client.async_frame = value.orig_frame
resp = self.handle_client_packet(value.data, remote_client.context)
if resp:
if isinstance(resp, basestring):
output = [resp]
else:
output = iter(resp)
for part in output:
msg = Message(
value.orig_frame,
remote_client.identity,
self.serialize_message(remote_client.identity, part),
)
self.outgoing.put(msg)
elif evt == 'sleep':
break
finally:
self._cleanup_client(remote_client)
def pubsub_socket(req, inq, outq):
c = hub.make_client()
with hub.subq('foo') as group:
while True:
q, v = first(waits=[inq, group])
if q == inq: # getting message from client
print "(inq) %s" % v
cmd = v.get("cmd", "")
if cmd=="":
print "published message to %i subscribers" % c.publish("foo", dumps({
'nick' : cgi.escape(v['nick'].strip()),
'message' : cgi.escape(v['message'].strip()),
}))
else:
outq.put(dict(message="test bot"))
elif q == group: # getting message for broadcasting
chan, msg_str = v
try:
msg = loads(msg_str)
data = dict(message=msg['message'], nick=msg['nick'])
print "(outq) %s" % data
outq.put(data)
except JSONDecodeError:
print "error decoding message %s" % msg_str
elif isinstance(v, WebSocketDisconnect): # getting a disconnect signal
return
else:
print "oops %s" % v
def _handle_client_requests_and_responses(self, remote_client):
assert self.nitro_socket
queues = [remote_client.incoming]
try:
while True:
(evt, value) = diesel.first(waits=queues, sleep=self.timeout)
if evt is remote_client.incoming:
assert isinstance(value, Message)
remote_client.async_frame = value.orig_frame
resp = self.handle_client_packet(value.data,
remote_client.context)
if resp:
if isinstance(resp, basestring):
output = [resp]
else:
output = iter(resp)
for part in output:
msg = Message(
value.orig_frame,
remote_client.identity,
self.serialize_message(remote_client.identity,
part),
)
self.outgoing.put(msg)
elif evt == 'sleep':
break
finally:
self._cleanup_client(remote_client)
def consumer(q, done):
for i in xrange(OPERATIONS):
evt, data = diesel.first(waits=[q], sleep=10000)
if evt == "sleep":
print "sleep was triggered!"
break
done.tick()
def test_Signal_instances_trigger_multiple_times():
usr1 = Signal(signal.SIGUSR1)
for i in xrange(5):
os.kill(os.getpid(), signal.SIGUSR1)
evt, _ = diesel.first(sleep=0.1, waits=[usr1])
assert evt is usr1, evt
usr1.rearm()
def consumer(q, done):
for i in xrange(OPERATIONS):
evt, data = diesel.first(waits=[q], sleep=10000)
if evt == "sleep":
print "sleep was triggered!"
break
done.tick()
def hi_server(addr):
while 1:
ev, val = first(until_eol=True, sleep=3)
if ev == 'sleep':
log.warn('%s timeout!' % time.asctime())
else:
send("you said %s" % val)
def get(self, waiting=True, timeout=None):
start = time()
while not self.inp and waiting:
if timeout:
remaining = timeout - (time() - start)
if remaining <= 0:
raise QueueTimeout()
else:
first(waits=[self.wait_id], sleep=remaining)
else:
wait(self.wait_id)
if self.inp:
return self.inp.popleft()
elif not waiting:
raise QueueEmpty()
def _handle_client_requests_and_responses(self, remote_client):
assert self.zmq_socket
queues = [remote_client.incoming, remote_client.outgoing]
try:
while True:
(evt, value) = diesel.first(waits=queues, sleep=self.timeout)
if evt is remote_client.incoming:
assert isinstance(value, Message)
resp = self.handle_client_packet(value.data, remote_client.context)
elif evt is remote_client.outgoing:
resp = value
elif evt == 'sleep':
break
if resp:
if isinstance(resp, basestring):
output = [resp]
else:
output = iter(resp)
for part in output:
msg = Message(
remote_client.identity,
part,
)
msg.zmq_return = remote_client.zmq_return
self.outgoing.put(msg)
finally:
self._cleanup_client(remote_client)
def _handle_client_requests_and_responses(self, remote_client):
assert self.zmq_socket
queues = [remote_client.incoming, remote_client.outgoing]
try:
while True:
(evt, value) = diesel.first(waits=queues, sleep=self.timeout)
if evt is remote_client.incoming:
assert isinstance(value, Message)
# Update return path with latest (in case of reconnect)
remote_client.zmq_return = value.zmq_return
resp = self.handle_client_packet(value.data,
remote_client.context)
elif evt is remote_client.outgoing:
resp = value
elif evt == 'sleep':
break
if resp:
if isinstance(resp, basestring):
output = [resp]
else:
output = iter(resp)
for part in output:
msg = Message(
remote_client.identity,
part,
)
msg.zmq_return = remote_client.zmq_return
self.outgoing.put(msg)
finally:
self._cleanup_client(remote_client)
def wait(self, timeout=None):
kw = dict(waits=[self])
if timeout:
kw['sleep'] = timeout
mark, data = first(**kw)
if mark != self:
raise EventTimeout()
def _actually_resolve(self, name, timeout):
timeout = timeout / float(len(self.nameservers))
try:
for server in self.nameservers:
# Try each nameserver in succession.
self.addr = server
query = make_query(name, A)
send(query.to_wire())
start = time.time()
remaining = timeout
while True:
# Handle the possibility of responses that are not to our
# original request - they are ignored and we wait for a
# response that matches our query.
item, data = first(datagram=True, sleep=remaining)
if item == "datagram":
response = from_wire(data)
if query.is_response(response):
if response.answer:
a_records = [r for r in response.answer if r.rdtype == A]
return [item.address for item in a_records[0].items]
raise NotFound
else:
# Not a response to our query - continue waiting for
# one that is.
remaining = remaining - (time.time() - start)
elif item == "sleep":
break
else:
raise Timeout(name)
finally:
self.addr = self.primary
def test_overwriting_custom_signal_handler_fails():
readings = []
success = Event()
failure = Event()
def append_reading(sig, frame):
readings.append(sig)
signal.signal(signal.SIGUSR1, signal.SIG_DFL)
signal.signal(signal.SIGUSR1, append_reading)
def overwriter():
try:
diesel.signal(signal.SIGUSR1)
except diesel.ExistingSignalHandler:
success.set()
else:
failure.set()
diesel.fork(overwriter)
diesel.sleep()
os.kill(os.getpid(), signal.SIGUSR1)
evt, _ = diesel.first(waits=[success, failure])
assert evt is success
assert readings
def hi_server(addr):
while 1:
ev, val = first(until_eol=True, sleep=3)
if ev == 'sleep':
log.warn('%s timeout!' % time.asctime())
else:
send("you said %s" % val)
def handle_rfc_6455_frames(self, inq, outq):
disconnecting = False
while True:
typ, val = first(receive=2, waits=[outq])
if typ == 'receive':
b1, b2 = unpack(">BB", val)
opcode = b1 & 0x0f
fin = (b1 & 0x80) >> 7
has_mask = (b2 & 0x80) >> 7
assert has_mask == 1, "Frames must be masked"
if opcode == 8:
inq.put(WebSocketDisconnect())
if disconnecting:
break
disconnecting = True
assert opcode in [1,8], "Currently only opcodes 1 & 8 are supported (opcode=%s)" % opcode
length = b2 & 0x7f
if length == 126:
length = unpack('>H', receive(2))[0]
elif length == 127:
length = unpack('>L', receive(8))[0]
mask = unpack('>BBBB', receive(4))
if length:
payload = array('B', receive(length))
if disconnecting:
continue
for i in xrange(len(payload)):
payload[i] ^= mask[i % 4]
try:
data = loads(payload.tostring())
inq.put(data)
except JSONDecodeError:
pass
elif typ == outq:
if type(val) is WebSocketDisconnect:
b1 = 0x80 | (8 & 0x0f) # FIN + opcode
send(pack('>BB', b1, 0))
if disconnecting:
break
disconnecting = True
else:
payload = dumps(val)
b1 = 0x80 | (1 & 0x0f) # FIN + opcode
payload_len = len(payload)
if payload_len <= 125:
header = pack('>BB', b1, payload_len)
elif payload_len > 125 and payload_len < 65536:
header = pack('>BBH', b1, 126, payload_len)
elif payload_len >= 65536:
header = pack('>BBQ', b1, 127, payload_len)
send(header + payload)
def chat_server(addr):
my_nick = until_eol().strip()
while True:
evt, data = first(until_eol=True, waits=["chat_message"])
if evt == "until_eol":
fire("chat_message", (my_nick, data.strip()))
else:
nick, message = data
send("<%s> %s\r\n" % (nick, message))
def chat_server(addr):
my_nick = until_eol().strip()
while True:
evt, data = first(until_eol=True, waits=['chat_message'])
if evt == 'until_eol':
fire('chat_message', (my_nick, data.strip()))
else:
nick, message = data
send("<%s> %s\r\n" % (nick, message))
def read(self, size=None):
assert size is not None, "Sorry, have to pass a size to read()"
if size == 0:
return ''
evt, data = diesel.first(sleep=self._timeout, receive=size)
if evt == 'sleep':
self._timeout = None
raise requests.exceptions.Timeout
return data
def read(self, size=None):
assert size is not None, "Sorry, have to pass a size to read()"
if size == 0:
return ''
evt, data = diesel.first(sleep=self._timeout, receive=size)
if evt == 'sleep':
self._timeout = None
raise requests.exceptions.Timeout
return data
def chat_server(addr):
my_nick = until_eol().strip()
while True:
ev, val = first(until_eol=True, waits=['chat_message'])
if ev == 'until_eol':
fire('chat_message', (my_nick, val.strip()))
else:
nick, message = val
send("<%s> %s\r\n" % (nick, message))
def request(self, method, url, headers={}, body=None, timeout=None):
'''Issues a `method` request to `path` on the
connected server. Sends along `headers`, and
body.
Very low level--you must set "host" yourself,
for example. It will set Content-Length,
however.
'''
url_info = urlparse(url)
fake_wsgi = dict(
(cgi_name(n), v) for n, v in headers.iteritems())
fake_wsgi.update({
'HTTP_METHOD' : method,
'SCRIPT_NAME' : '',
'PATH_INFO' : url_info[2],
'QUERY_STRING' : url_info[4],
'wsgi.version' : (1,0),
'wsgi.url_scheme' : 'http', # XXX incomplete
'wsgi.input' : cStringIO.StringIO(body or ''),
'wsgi.errors' : FileLikeErrorLogger(hlog),
'wsgi.multithread' : False,
'wsgi.multiprocess' : False,
'wsgi.run_once' : False,
})
req = Request(fake_wsgi)
timeout_handler = TimeoutHandler(timeout or 60)
send('%s %s HTTP/1.1\r\n%s' % (req.method, req.url, str(req.headers)))
if body:
send(body)
h = HttpParser()
body = []
data = None
while True:
if data:
used = h.execute(data, len(data))
if h.is_headers_complete():
body.append(h.recv_body())
if h.is_message_complete():
data = data[used:]
break
ev, val = first(receive_any=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
data = val
resp = Response(
response=''.join(body),
status=h.get_status_code(),
headers=h.get_headers(),
)
return resp
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
def chat(self):
fork(self.input_handler)
nick = self.input.get()
send("%s\r\n" % nick)
while True:
evt, data = first(until_eol=True, waits=[self.input])
if evt == "until_eol":
print data.strip()
else:
send("%s\r\n" % data)
def setup(self):
self.result = Event()
self.queue = Queue()
self.timeouts = 0
diesel.fork(self.consumer, 0.01)
diesel.fork(self.producer, 0.05)
diesel.fork(self.consumer, 0.10)
ev, val = diesel.first(sleep=TIMEOUT, waits=[self.result])
if ev == 'sleep':
assert 0, 'timed out'
def chat(self):
fork(self.input_handler)
nick = self.input.get()
send("%s\r\n" % nick)
while True:
evt, data = first(until_eol=True, waits=[self.input])
if evt == "until_eol":
print data.strip()
else:
send("%s\r\n" % data)
def websocket_protocol(self, req):
"""Runs the WebSocket protocol after the handshake is complete.
Creates two `Queue` instances for incoming and outgoing messages and
passes them to the `web_socket_handler` that was supplied to the
`WebSocketServer` constructor.
"""
inq = Queue()
outq = Queue()
if req.rfc_handshake:
handle_frames = self.handle_rfc_6455_frames
else:
# Finish the non-RFC handshake
key1 = req.headers.get('Sec-WebSocket-Key1')
key2 = req.headers.get('Sec-WebSocket-Key2')
# The final key can be in two places. The first is in the
# `Request.data` attribute if diesel is *not* being proxied
# to by a smart proxy that parsed HTTP requests. If it is being
# proxied to, that data will not have been sent until after our
# initial 101 Switching Protocols response, so we will need to
# receive it here.
if req.data:
key3 = req.data
else:
evt, key3 = first(receive=8, sleep=5)
assert evt == "receive", "timed out while finishing handshake"
num1 = int(''.join(c for c in key1 if c in '0123456789'))
num2 = int(''.join(c for c in key2 if c in '0123456789'))
assert num1 % key1.count(' ') == 0
assert num2 % key2.count(' ') == 0
final = pack('!II8s', num1 / key1.count(' '),
num2 / key2.count(' '), key3)
handshake_finish = hashlib.md5(final).digest()
send(handshake_finish)
handle_frames = self.handle_non_rfc_frames
def wrap(req, inq, outq):
self.web_socket_handler(req, inq, outq)
outq.put(WebSocketDisconnect())
handler_loop = fork(wrap, req, inq, outq)
try:
handle_frames(inq, outq)
except ConnectionClosed:
if handler_loop.running:
inq.put(WebSocketDisconnect())
raise
def main():
usr1 = Signal(SIGUSR1)
ticks = 0
log.fields(pid=os.getpid()).info('started')
while True:
evt, _ = diesel.first(sleep=1, waits=[usr1])
if evt == 'sleep':
ticks += 1
elif evt == usr1:
log.fields(ticks=ticks).info('stats')
# must rearm() to use again
evt.rearm()
def get_messages():
outsock = DieselZMQSocket(zctx.socket(zmq.DEALER), bind="tcp://127.0.0.1:5000")
for x in xrange(1000):
t, m = first(sleep=1.0, waits=[outsock])
if t == 'sleep':
print "sleep timeout!"
else:
print "zmq:", m
quickstop()
def do_upgrade(self, req):
if req.headers.get_one('Upgrade') != 'WebSocket':
return self.web_handler(req)
# do upgrade response
org = req.headers.get_one('Origin')
send(
'''HTTP/1.1 101 Web Socket Protocol Handshake\r
Upgrade: WebSocket\r
Connection: Upgrade\r
WebSocket-Origin: %s\r
WebSocket-Location: %s\r
WebSocket-Protocol: diesel-generic\r
\r
''' % (org, self.ws_location))
inq = Queue()
outq = Queue()
def wrap(inq, outq):
self.web_socket_handler(inq, outq)
outq.put(WebSocketDisconnect())
fork(wrap, inq, outq)
while True:
try:
typ, val = first(receive=1, waits=[outq.wait_id])
if typ == 'receive':
assert val == '\x00'
val = until('\xff')[:-1]
if val == '':
inq.put(WebSocketDisconnect())
else:
data = dict((k, v[0]) if len(v) == 1 else (k, v) for k, v in cgi.parse_qs(val).iteritems())
inq.put(WebSocketData(data))
else:
try:
v = outq.get(waiting=False)
except QueueEmpty:
pass
else:
if type(v) is WebSocketDisconnect:
send('\x00\xff')
break
else:
data = dumps(dict(v))
send('\x00%s\xff' % data)
except ConnectionClosed:
inq.put(WebSocketDisconnect())
raise ConnectionClosed("remote disconnected")
def readline(self, max_size=None):
evt, line = diesel.first(sleep=self._timeout, until='\n')
if evt == 'sleep':
self._timeout = None
raise requests.exceptions.Timeout
if max_size:
line = "".join([self._extra, line])
nl = line.find('\n') + 1
if nl > max_size:
nl = max_size
line, self._extra = line[:nl], line[nl:]
return line
def websocket_protocol(self, req):
"""Runs the WebSocket protocol after the handshake is complete.
Creates two `Queue` instances for incoming and outgoing messages and
passes them to the `web_socket_handler` that was supplied to the
`WebSocketServer` constructor.
"""
inq = Queue()
outq = Queue()
if req.rfc_handshake:
handle_frames = self.handle_rfc_6455_frames
else:
# Finish the non-RFC handshake
key1 = req.headers.get('Sec-WebSocket-Key1')
key2 = req.headers.get('Sec-WebSocket-Key2')
# The final key can be in two places. The first is in the
# `Request.data` attribute if diesel is *not* being proxied
# to by a smart proxy that parsed HTTP requests. If it is being
# proxied to, that data will not have been sent until after our
# initial 101 Switching Protocols response, so we will need to
# receive it here.
if req.data:
key3 = req.data
else:
evt, key3 = first(receive=8, sleep=5)
assert evt == "receive", "timed out while finishing handshake"
num1 = int(''.join(c for c in key1 if c in '0123456789'))
num2 = int(''.join(c for c in key2 if c in '0123456789'))
assert num1 % key1.count(' ') == 0
assert num2 % key2.count(' ') == 0
final = pack('!II8s', num1 / key1.count(' '), num2 / key2.count(' '), key3)
handshake_finish = hashlib.md5(final).digest()
send(handshake_finish)
handle_frames = self.handle_non_rfc_frames
def wrap(req, inq, outq):
self.web_socket_handler(req, inq, outq)
outq.put(WebSocketDisconnect())
handler_loop = fork(wrap, req, inq, outq)
try:
handle_frames(inq, outq)
except ConnectionClosed:
if handler_loop.running:
inq.put(WebSocketDisconnect())
raise
def test_basic_fork():
v = [0]
done = Event()
def parent():
fork(tottering_child, v)
sleep(0.1)
done.set()
runtime.current_app.add_loop(Loop(parent))
ev, _ = first(sleep=1, waits=[done])
if ev == 'sleep':
assert 0, "timed out"
assert (v[0] == 1)
def main():
usr1 = Signal(SIGUSR1)
ticks = 0
log.fields(pid=os.getpid()).info('started')
while True:
evt, _ = diesel.first(sleep=1, waits=[usr1])
if evt == 'sleep':
ticks += 1
elif evt == usr1:
log.fields(ticks=ticks).info('stats')
# must rearm() to use again
evt.rearm()
def _readline(self):
if self.timeout > 0:
timeout_handler = TimeoutHandler(self.timeout)
ev, line = diesel.first(until="\n", sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
else: return line
else:
line = diesel.until("\n")
if line:
return line
else:
raise StompTimeout
def readline(self, max_size=None):
evt, line = diesel.first(sleep=self._timeout, until='\n')
if evt == 'sleep':
self._timeout = None
raise requests.exceptions.Timeout
if max_size:
line = "".join([self._extra, line])
nl = line.find('\n') + 1
if nl > max_size:
nl = max_size
line, self._extra = line[:nl], line[nl:]
return line
def socket_handler(req, inq, outq):
with f.sub() as group:
while True:
q, v = first(waits=[inq, group])
if q == group:
outq.put(dict(message=v['message'], nick=v['nick']))
elif isinstance(v, WebSocketDisconnect):
return
elif v.get('nick', '').strip() and v.get('message', '').strip():
f.pub({
'nick': cgi.escape(v['nick'].strip()),
'message': cgi.escape(v['message'].strip()),
})
def socket_handler(req, inq, outq):
with f.sub() as group:
while True:
q, v = first(waits=[inq, group])
if q == group:
outq.put(dict(message=v['message'], nick=v['nick']))
elif isinstance(v, WebSocketDisconnect):
return
elif v.get('nick', '').strip() and v.get('message', '').strip():
f.pub({
'nick' : cgi.escape(v['nick'].strip()),
'message' : cgi.escape(v['message'].strip()),
})
def _readline(self):
if self.timeout > 0:
timeout_handler = TimeoutHandler(self.timeout)
ev, line = diesel.first(until="\n",
sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
else: return line
else:
line = diesel.until("\n")
if line:
return line
else:
raise StompTimeout
def get_messages():
outsock = DieselZMQSocket(zctx.socket(zmq.DEALER),
bind="tcp://127.0.0.1:5000")
for x in xrange(1000):
t, m = first(sleep=1.0, waits=[outsock])
if t == 'sleep':
print "sleep timeout!"
else:
print "zmq:", m
quickstop()
def handle_chunks(headers, timeout=None):
'''Generic chunk handling code, used by both client
and server.
Modifies the passed-in HttpHeaders instance.
'''
timeout_handler = TimeoutHandler(timeout or 60)
chunks = []
while True:
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
chunk_head = val
if ';' in chunk_head:
# we don't support any chunk extensions
chunk_head = chunk_head[:chunk_head.find(';')]
size = int(chunk_head, 16)
if size == 0:
break
else:
chunks.append(receive(size))
_ = receive(2) # ignore trailing CRLF
while True:
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
trailer = val
if trailer.strip():
headers.add(*tuple(trailer.split(':', 1)))
else:
body = ''.join(chunks)
headers.set('Content-Length', len(body))
headers.remove('Transfer-Encoding')
break
return body
def handle_chunks(headers, timeout=None):
'''Generic chunk handling code, used by both client
and server.
Modifies the passed-in HttpHeaders instance.
'''
timeout_handler = TimeoutHandler(timeout or 60)
chunks = []
while True:
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
chunk_head = val
if ';' in chunk_head:
# we don't support any chunk extensions
chunk_head = chunk_head[:chunk_head.find(';')]
size = int(chunk_head, 16)
if size == 0:
break
else:
chunks.append(receive(size))
_ = receive(2) # ignore trailing CRLF
while True:
ev, val = first(until_eol=True, sleep=timeout_handler.remaining())
if ev == 'sleep': timeout_handler.timeout()
trailer = val
if trailer.strip():
headers.add(*tuple(trailer.split(':', 1)))
else:
body = ''.join(chunks)
headers.set('Content-Length', len(body))
headers.remove('Transfer-Encoding')
break
return body
def test_basic_fork():
v = [0]
done = Event()
def parent():
fork(tottering_child, v)
sleep(0.1)
done.set()
runtime.current_app.add_loop(Loop(parent))
ev, _ = first(sleep=1, waits=[done])
if ev == 'sleep':
assert 0, "timed out"
assert (v[0] == 1)
def test_pending_events_dont_break_ordering_when_handling_early_values():
# This test confirms that "early values" returned from a Waiter do
# not give other pending event sources the chance to switch their
# values into the greenlet while it context switches to give other
# greenlets a chance to run.
# First we setup a fake connection. It mimics a connection that does
# not have data waiting in the buffer, and has to wait for the system
# to call it back when data is ready on the socket. The delay argument
# specifies how long the test should wait before simulating that data
# is ready.
conn1 = FakeConnection(1, delay=[None, 0.1])
# Next we setup a Queue instance and prime it with a value, so it will
# be ready early and return an EarlyValue.
q = Queue()
q.put(1)
# Force our fake connection into the connection stack for the current
# loop so we can make network calls (like until_eol).
loop = core.current_loop
loop.connection_stack.append(conn1)
try:
# OK, this first() call does two things.
# 1) It calls until_eol, finds that no data is ready, and sets up a
# callback to be triggered when data is ready (which our
# FakeConnection will simulate).
# 2) Fetches from the 'q' which will result in an EarlyValue.
source, value = diesel.first(until_eol=True, waits=[q])
assert source == q, source
# What must happen is that the callback registered to handle data
# from the FakeConnection when it arrives MUST BE CANCELED/DISCARDED/
# FORGOTTEN/NEVER CALLED. If it gets called, it will muck with
# internal state, and possibly switch back into the running greenlet
# with an unexpected value, which will throw off the ordering of
# internal state and basically break everything.
v = diesel.until_eol()
assert v == 'expected value 1\r\n', 'actual value == %r !!!' % (v, )
finally:
loop.connection_stack = []
def test_pending_events_dont_break_ordering_when_handling_early_values():
# This test confirms that "early values" returned from a Waiter do
# not give other pending event sources the chance to switch their
# values into the greenlet while it context switches to give other
# greenlets a chance to run.
# First we setup a fake connection. It mimics a connection that does
# not have data waiting in the buffer, and has to wait for the system
# to call it back when data is ready on the socket. The delay argument
# specifies how long the test should wait before simulating that data
# is ready.
conn1 = FakeConnection(1, delay=[None, 0.1])
# Next we setup a Queue instance and prime it with a value, so it will
# be ready early and return an EarlyValue.
q = Queue()
q.put(1)
# Force our fake connection into the connection stack for the current
# loop so we can make network calls (like until_eol).
loop = core.current_loop
loop.connection_stack.append(conn1)
try:
# OK, this first() call does two things.
# 1) It calls until_eol, finds that no data is ready, and sets up a
# callback to be triggered when data is ready (which our
# FakeConnection will simulate).
# 2) Fetches from the 'q' which will result in an EarlyValue.
source, value = diesel.first(until_eol=True, waits=[q])
assert source == q, source
# What must happen is that the callback registered to handle data
# from the FakeConnection when it arrives MUST BE CANCELED/DISCARDED/
# FORGOTTEN/NEVER CALLED. If it gets called, it will muck with
# internal state, and possibly switch back into the running greenlet
# with an unexpected value, which will throw off the ordering of
# internal state and basically break everything.
v = diesel.until_eol()
assert v == 'expected value 1\r\n', 'actual value == %r !!!' % (v,)
finally:
loop.connection_stack = []
def recv(self, matchid=None):
evt, frame = first(waits=[self.sock], sleep=self.timeout)
if evt == 'sleep':
err = "Timed out after %.2f secs waiting for zmq frame"
raise MontageRequestTimeout(err % self.timeout)
env = proto.MontageEnvelope(frame.data)
if matchid:
assert env.msgid == matchid.bytes, "request/response id pairs did not match!"
if env.mtype == proto.MONTAGE_ERROR:
raise RiakException(proto.MontageError(env.msg).error)
return id_to_packet[env.mtype](env.msg)
def test_multiple_signal_waiters():
N_WAITERS = 5
c = Countdown(N_WAITERS)
def mwaiter():
diesel.signal(signal.SIGUSR1)
c.tick()
for i in xrange(N_WAITERS):
diesel.fork(mwaiter)
diesel.sleep()
os.kill(os.getpid(), signal.SIGUSR1)
evt, data = diesel.first(sleep=1, waits=[c])
assert evt is c, "all waiters were not triggered!"
def _handle_all_inbound_and_outbound_traffic(self):
assert self.zmq_socket
diesel.fork_child(self._receive_incoming_messages)
queues = [self.incoming, self.outgoing]
while True:
(queue, msg) = diesel.first(waits=queues)
if queue is self.incoming:
if msg.remote_identity not in self.clients:
self._register_client(msg)
self.clients[msg.remote_identity].incoming.put(msg)
elif queue is self.outgoing:
self.zmq_socket.send(msg.zmq_return, zmq.SNDMORE)
self.zmq_socket.send(msg.data)
def _handle_all_inbound_and_outbound_traffic(self):
assert self.zmq_socket
diesel.fork_child(self._receive_incoming_messages)
queues = [self.incoming, self.outgoing]
while True:
(queue, msg) = diesel.first(waits=queues)
if queue is self.incoming:
if msg.remote_identity not in self.clients:
self._register_client(msg)
self.clients[msg.remote_identity].incoming.put(msg)
elif queue is self.outgoing:
self.zmq_socket.send(msg.zmq_return, zmq.SNDMORE)
self.zmq_socket.send(msg.data)
def manage_peer_connection(host, port, proposals, saves, gets):
import traceback
sleep_time = 0.2
client_alive_timer = Timer(5.0)
while True:
try:
with PeerClient(host, port) as peer:
peer.signon()
peer.sync()
sleep_time = 0.2
while True:
if client_alive_timer.is_due:
peer.keep_clients_alive()
client_alive_timer.touch()
e, v = first(sleep=client_alive_timer.till_due, waits=[proposals, saves, gets])
if e == proposals:
id, key, resp = v
try:
resp.put(PROPOSE_SUCCESS
if peer.propose(id, key)
else PROPOSE_FAIL)
except:
proposals.put(v) # retry
raise
elif e == saves:
id, key, value, client, rollback, resp = v
try:
peer.save(id, key, value, client, rollback)
resp.put(SAVE_SUCCESS)
except:
saves.put(v) # retry
raise
elif e == gets:
key, resp = v
try:
v = peer.get(key)
resp.put(v)
except:
gets.put(v) # retry
raise
except ClientConnectionError, e:
clog.error("Connection problem to (%s, %s): %s" % (
host, port, e))
sleep(sleep_time)
sleep_time *= 2
sleep_time = min(sleep_time, 5.0)
except:
def test_signals_are_handled_while_event_loop_is_blocked():
done = Event()
def handler():
diesel.signal(signal.SIGUSR1)
done.set()
def killer():
time.sleep(0.1)
os.kill(os.getpid(), signal.SIGUSR1)
diesel.fork(handler)
diesel.thread(killer)
diesel.sleep()
evt, _ = diesel.first(sleep=1, waits=[done])
assert evt is done
def get(self, waiting=True, timeout=None):
if self.inp:
return self.inp.popleft()
mark = None
if waiting:
kw = dict(waits=[self])
if timeout:
kw['sleep'] = timeout
mark, val = first(**kw)
if mark == self:
return val
else:
raise QueueTimeout()
raise QueueEmpty()
def make_and_wait():
q1 = Queue()
q2 = Queue()
both = [q1, q2]
fork(fire_random, both)
while True:
q, v = first(waits=both)
assert v is None
if q == q1:
print 'q1'
elif q == q2:
print 'q2'
else:
assert 0
def manage_peer_connection(host, port, proposals, saves, gets):
import traceback
sleep_time = 0.2
client_alive_timer = Timer(5.0)
while True:
try:
with PeerClient(host, port) as peer:
peer.signon()
peer.sync()
sleep_time = 0.2
while True:
if client_alive_timer.is_due:
peer.keep_clients_alive()
client_alive_timer.touch()
e, v = first(sleep=client_alive_timer.till_due,
waits=[proposals, saves, gets])
if e == proposals:
id, key, resp = v
try:
resp.put(PROPOSE_SUCCESS if peer.
propose(id, key) else PROPOSE_FAIL)
except:
proposals.put(v) # retry
raise
elif e == saves:
id, key, value, client, rollback, resp = v
try:
peer.save(id, key, value, client, rollback)
resp.put(SAVE_SUCCESS)
except:
saves.put(v) # retry
raise
elif e == gets:
key, resp = v
try:
v = peer.get(key)
resp.put(v)
except:
gets.put(v) # retry
raise
except ClientConnectionError, e:
clog.error("Connection problem to (%s, %s): %s" % (host, port, e))
sleep(sleep_time)
sleep_time *= 2
sleep_time = min(sleep_time, 5.0)
except:
def _handle_all_inbound_and_outbound_traffic(self):
assert self.nitro_socket
queues = [self.nitro_socket, self.outgoing]
socket = self.nitro_socket
make_frame = pynitro.NitroFrame
while self.should_run:
(queue, msg) = diesel.first(waits=queues)
if queue is self.outgoing:
socket.reply(msg.orig_frame, make_frame(msg.data))
else:
id, obj = self.parse_message(msg.data)
msg.clear_data()
msg = Message(msg, id, obj)
if msg.identity not in self.clients:
self._register_client(msg)
self.clients[msg.identity].incoming.put(msg)
def whois(self, query):
"""
Perform a query against the server. Either returns the server's
response or raises a `Timeout` exception if the downstream server
took too long.
"""
diesel.send(query + CRLF)
result = []
try:
while True:
evt, data = diesel.first(sleep=5, receive=2048)
if evt == 'sleep':
raise Timeout(self.addr)
result.append(data)
except diesel.ConnectionClosed, ex:
if ex.buffer:
result.append(ex.buffer)
