def thrash_tnetstring():
for obj, tns, json, msh in TESTS:
if TEST_DUMP_ONLY:
tnetstring.dumps(obj)
elif TEST_LOAD_ONLY:
assert tnetstring.loads(tns) == obj
else:
assert tnetstring.loads(tnetstring.dumps(obj)) == obj
def test_unicode_handling(self):
self.assertRaises(ValueError,tnetstring.dumps,u"hello")
self.assertEquals(tnetstring.dumps(u"hello","utf8"),"5:hello,")
self.assertEquals(type(tnetstring.loads("5:hello,")),str)
self.assertEquals(type(tnetstring.loads("5:hello,","utf8")),unicode)
ALPHA = u"\N{GREEK CAPITAL LETTER ALPHA}lpha"
self.assertEquals(tnetstring.dumps(ALPHA,"utf8"),"6:"+ALPHA.encode("utf8")+",")
self.assertEquals(tnetstring.dumps(ALPHA,"utf16"),"12:"+ALPHA.encode("utf16")+",")
self.assertEquals(tnetstring.loads("12:\xff\xfe\x91\x03l\x00p\x00h\x00a\x00,","utf16"),ALPHA)
def test_unicode_handling(self):
self.assertRaises(ValueError, tnetstring.dumps, u"hello")
self.assertEquals(tnetstring.dumps(u"hello", "utf8"), "5:hello,")
self.assertEquals(type(tnetstring.loads("5:hello,")), str)
self.assertEquals(type(tnetstring.loads("5:hello,", "utf8")), unicode)
ALPHA = u"\N{GREEK CAPITAL LETTER ALPHA}lpha"
self.assertEquals(tnetstring.dumps(ALPHA, "utf8"),
"6:" + ALPHA.encode("utf8") + ",")
self.assertEquals(tnetstring.dumps(ALPHA, "utf16"),
"12:" + ALPHA.encode("utf16") + ",")
self.assertEquals(
tnetstring.loads("12:\xff\xfe\x91\x03l\x00p\x00h\x00a\x00,",
"utf16"), ALPHA)
def process_data(mtype, mdata):
"""
This method processes the data sent from Pushpin's socket
"""
mdecode = tnetstring.loads(mdata[1:])
print('received: %s, %s' % mtype % mdecode)
if mtype == b'conn':
# unavail = False
c_id = 'none'
conn_num = 0
peer_ip = '0.0.0.0'
unavail = b'unavailable' in mdecode
if b'id' in mdecode:
c_id = mdecode[b'id'].decode('UTF-8')
ppstatconn = PushpinStat_Conn(unavailable=unavail, conn_id=c_id, conn_num=conn_num, peer_ip=peer_ip, type='ws')
ppstatconn.save()
elif mytpe == b'sub' or mtype == b'report' or mtype == b'activity' or mtype == b'message':
channel = ''
mode = ''
sub_cnt = 0
if b'channel' in mdecode:
channel = mdecode[b'channel']
if b'mode' in mdecode:
mode = mdecode[b'mode']
if b'subscribers' in mdecode:
sub_cnt = mdecode[b'subscribers']
unavailable = b'unavailable' in mdecode
ppstatsub = PushpinStat_Sub(channel=channel, mode=mode, sub_cnt=sub_cnt, unavailable=unavailable)
ppstatsub.save()
else:
print('not a valid type')
def test_roundtrip_format_unicode(self):
for _ in xrange(500):
v = get_random_object(unicode=True)
self.assertEqual(
v, tnetstring.loads(tnetstring.dumps(v, "utf8"), "utf8"))
self.assertEqual((v, ""),
tnetstring.pop(tnetstring.dumps(v, "utf16"),
"utf16"))
def get(self, sid):
if not self.is_valid_key(sid):
return self.new()
key = self.get_session_key(sid)
saved = self.redis.get(key)
if saved:
data = tnetstring.loads(saved)
return self.session_class(data, sid, False)
def __init__(self, key, fstore, path, create=False):
self.id = key
self.store = fstore
self.path = path
try:
self.data = tnetstring.loads(fstore.open(path).read())
except IOError:
if not create:
raise KeyError('Item does not exist')
self.data = {}
def read_packets (filename):
try:
os.stat (filename)
except OSError:
print "No such file : %s"%filename
sys.exit (1)
pkts = open (filename).read ()
pkts = tnetstring.loads (pkts, 'iso-8859-15')
for data in pkts:
yield data
def __init__(self, key, fstore, path, create=False):
self.id = key
self.store = fstore
self.path = path
try:
self.data = tnetstring.loads(fstore.open(path).read())
except IOError:
if not create:
raise KeyError("Item does not exist")
self.data = {}
def process_data(m_type, m_data):
"""
This method processes the data sent from Pushpin's socket
"""
m_decode = tnetstring.loads(m_data[1:])
print('in process_data: m_type=', m_type)
print('m_decode=', m_decode)
# If there is new data, need to notify frontend that data is updated
if decode_data(m_decode, m_type):
send_update_notify()
def start_push(self):
while True:
m_raw = self.sock.recv().decode(
"ISO-8859-1") # .decode("ISO-8859-1")
mtype, mdata = m_raw.split(" ", 1)
if mdata[0] != "T":
print("unsupported format")
continue
m = tnetstring.loads(mdata[1:].encode("ISO-8859-1"))
print("%s %s" % (mtype, m))
self.send_prom_push_gateway(msg_type=mtype, msg=m)
def _validate (self):
if not self._raw_data or 2 is not len (self._raw_data):
self._logger.error (
"Invalid data from REDIS: '%s'"%self._raw_data)
return False
self._table, self._data = self._raw_data
try:
self._data = tnetstring.loads (self._data, 'iso-8859-15')
except ValueError, e:
self._logger.critical (
"Invalid tnetstring data : '%s'"%self._data)
return False
def call(self, method, args=dict()):
req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = ensure_utf8(method)
req['args'] = ensure_utf8(args)
req_raw = tnetstring.dumps(req)
try:
if not self.sock.poll(30000, zmq.POLLOUT):
raise CallError('send-timeout')
m_list = list()
m_list.append('')
m_list.append(req_raw)
self.sock.send_multipart(m_list)
except zmq.ZMQError as e:
raise CallError('send-failed', e.message)
start = int(time.clock() * 1000)
while True:
elapsed = max(int(time.clock() * 1000) - start, 0)
try:
if not self.sock.poll(max(30000 - elapsed, 0), zmq.POLLIN):
raise CallError('receive-timeout')
m_list = self.sock.recv_multipart()
except zmq.ZMQError as e:
raise CallError('receive-failed', e.message)
if len(m_list) != 2:
print "response has wrong number of parts, skipping"
continue
if len(m_list[0]) > 0:
print "response first part is not empty, skipping"
continue
resp_raw = m_list[1]
try:
resp = tnetstring.loads(resp_raw)
except:
print "failed to parse response as tnetstring, skipping"
continue
if 'id' not in resp:
print "response missing id field, skipping"
continue
if resp['id'] != req['id']:
print "unexpected response id, skipping"
continue
break
if 'success' not in resp:
raise CallError('invalid-response', 'missing success field')
if resp['success']:
if 'value' not in resp:
raise CallError('invalid-response', 'missing value field')
return resp['value']
else:
if 'condition' not in resp:
raise CallError('invalid-response', 'missing condition field')
raise CallError(resp['condition'])
def call(self, method, args, timeout=10000):
req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = ensure_utf8(method)
req['args'] = ensure_utf8(args)
req_raw = tnetstring.dumps(req)
start = int(time.clock() * 1000)
try:
if not self.sock.poll(timeout, zmq.POLLOUT):
raise CallError('send-timeout')
m_list = list()
m_list.append('')
m_list.append(req_raw)
self.sock.send_multipart(m_list)
except zmq.ZMQError as e:
raise CallError('send-failed', e.message)
while True:
elapsed = max(int(time.clock() * 1000) - start, 0)
try:
if not self.sock.poll(max(timeout - elapsed, 0), zmq.POLLIN):
raise CallError('receive-timeout')
m_list = self.sock.recv_multipart()
except zmq.ZMQError as e:
raise CallError('receive-failed', e.message)
if len(m_list) != 2:
print "response has wrong number of parts, skipping"
continue
if len(m_list[0]) > 0:
print "response first part is not empty, skipping"
continue
resp_raw = m_list[1]
try:
resp = tnetstring.loads(resp_raw)
except:
print "failed to parse response as tnetstring, skipping"
continue
if 'id' not in resp:
print "response missing id field, skipping"
continue
if resp['id'] != req['id']:
print "unexpected response id, skipping"
continue
break
if 'success' not in resp:
raise CallError('invalid-response', 'missing success field')
if resp['success']:
if 'value' not in resp:
raise CallError('invalid-response', 'missing value field')
return resp['value']
else:
if 'condition' not in resp:
raise CallError('invalid-response', 'missing condition field')
raise CallError(resp['condition'])
def pkt_read_parse (self, info, timeout = None, maxtry = 10,
reemission = False, sid = False):
_reply = SIPpacket ()
_r = self.pkt_read (info, timeout, sid) if not reemission else \
self.pkt_read_retry (info, maxtry, sid)
if not _r:
return None
_a = tnetstring.loads (_r [1], 'iso-8859-15')
s_id, timestamp, addr, pkt = _a
if self._debug:
print "<=== received from %s at %s"%(addr, time.ctime (timestamp))
print str (pkt)
pkt = self.pkt_parse (pkt)
return (s_id, timestamp, addr, pkt)
def __init__(self, base, create=False):
self.base = op.expandvars(op.expanduser(base))
if not op.exists(self.base):
if create:
os.mkdir(self.base)
else:
raise ValueError("%s does not exist" % base)
if not op.isdir(self.base):
raise ValueError("%s is not a directory" % base)
try:
self.index = tnetstring.loads(self.open("index.gz").read())
except IOError:
self.index = {}
def __init__(self, base, create=False):
self.base = op.expandvars(op.expanduser(base))
if not op.exists(self.base):
if create:
os.mkdir(self.base)
else:
raise ValueError('%s does not exist' % base)
if not op.isdir(self.base):
raise ValueError('%s is not a directory' % base)
try:
self.index = tnetstring.loads(self.open('index.gz').read())
except IOError:
self.index = {}
def run(self, handler, data):
poller = zmq.Poller()
poller.register(self.control_sock, zmq.POLLIN)
poller.register(self.rep_sock, zmq.POLLIN)
while True:
socks = dict(poller.poll())
if socks.get(self.control_sock) == zmq.POLLIN:
s = self.control_sock.recv()
if s == 'stop':
break
elif socks.get(self.rep_sock) == zmq.POLLIN:
req_raw = self.rep_sock.recv()
try:
req = tnetstring.loads(req_raw)
except:
self._respond_error('bad-request')
continue
self.req_id = req.get('id')
method = req.get('method')
if not method:
self._respond_error('bad-request')
continue
args = req.get('args')
if args is None:
args = dict()
if not isinstance(args, dict):
self._respond_error('bad-request')
continue
try:
ret = handler(method, args, data=data)
self._respond(ret)
except CallError as e:
self._respond_error(e.condition)
except:
traceback.print_exc()
self._respond_error('internal-server-error')
self.rep_sock.linger = 0
self.rep_sock.close()
self.control_sock.send('finished')
self.control_sock.close()
def _discover_uris(self):
self._thread_cond.acquire()
if self._discovery_in_progress:
self._thread_cond.wait()
self._thread_cond.release()
self._verify_discovered_uris()
return
else:
self._discovery_in_progress = True
self._thread_cond.release()
if (self.uri is None or self._discovery_completed or
(self.pub_uri and self.push_uri)):
self._end_discovery(False)
return
sock = self._context.socket(zmq.REQ)
sock.linger = 0
sock.connect(self.uri)
start = int(timeit.default_timer() * 1000)
if not sock.poll(3000, zmq.POLLOUT):
sock.close()
self._end_discovery(False)
raise ValueError('uri discovery request failed: pollout timeout')
req = {_ensure_utf8('method'): _ensure_utf8('get-zmq-uris')}
sock.send(tnetstring.dumps(req))
elapsed = max(int(timeit.default_timer() * 1000) - start, 0)
if not sock.poll(max(3000 - elapsed, 0), zmq.POLLIN):
sock.close()
self._end_discovery(False)
raise ValueError('uri discovery request failed: pollin timeout')
resp = tnetstring.loads(sock.recv())
sock.close()
if (not resp.get(_ensure_utf8('success')) or
not resp.get(_ensure_utf8('value'))):
self._end_discovery(False)
raise ValueError('uri discovery request failed: %s' % resp)
self._set_discovered_uris(resp[_ensure_utf8('value')])
self._end_discovery(True)
self._verify_discovered_uris()
def stream_rtp (pkt = None):
# SDP
body = pkt.body
sdp = SDP (str (body))
rtp_address = sdp.conndata.address
rtp_port = sdp.mediadata.port
print "sending RTP _to_ %s:%d"%(str(rtp_address), int (rtp_port))
sys.stdout.flush ()
# RTP
rtp_pkt = open ('../rtp/rtp.out.g711.py').read ()
rtp_pkt = tnetstring.loads (rtp_pkt, 'iso-8859-15')
try:
for data_rtp in rtp_pkt:
print type (data_rtp)
client.pkt_send (session_id ['server_id_rtp'], data_rtp,
rtp_address, int (rtp_port))
time.sleep (config.rtp_framing)
print ".",
sys.stdout.flush ()
except KeyboardInterrupt:
print "Stop sending RTP... waiting for BYE"
session_id ['stream'] = False
return
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect('ipc:///tmp/pushpin-stats')
sock.setsockopt(zmq.SUBSCRIBE, 'sub ')
poller = zmq.Poller()
poller.register(sock, zmq.POLLIN)
while True:
socks = dict(poller.poll(1000))
if socks.get(sock) == zmq.POLLIN:
m_raw = sock.recv()
if not m_raw.startswith('sub T'):
continue
m = tnetstring.loads(m_raw[5:])
now = int(time.time())
sub_key = (m['mode'], m['channel'])
sub = subs.get(sub_key)
if m.get('unavailable'):
if sub:
del subs[sub_key]
print 'UNSUB mode=%s channel=%s' % (m['mode'], m['channel'])
else:
if not sub:
sub = Subscription()
subs[sub_key] = sub
print 'SUB mode=%s channel=%s' % (m['mode'], m['channel'])
sub.ttl = m['ttl']
sub.last_refresh = now
poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)
while True:
socks = dict(poller.poll(None))
if socks.get(in_sock) == zmq.POLLIN:
m_raw = in_sock.recv()
elif socks.get(in_stream_sock) == zmq.POLLIN:
m_list = in_stream_sock.recv_multipart()
m_raw = m_list[2]
else:
continue
req = tnetstring.loads(m_raw[1:])
print('IN {}'.format(req))
if req.get(b'type'):
# skip all non-data messages
continue
if req.get(b'uri', b'').startswith(b'ws'):
resp = {}
resp[b'from'] = instance_id
resp[b'id'] = req[b'id']
resp[b'seq'] = 0
resp[b'code'] = 101
resp[b'reason'] = b'Switching Protocols'
resp[b'credits'] = 1024
def session_update(sid, last_ids):
s = sessions.get(sid)
if s is None:
raise ValueError('unknown sid')
for k, v in last_ids.iteritems():
s.last_ids[k] = v
def session_get_last_ids(sid):
s = sessions.get(sid)
if s is not None:
return s.last_ids
else:
return None
while True:
req = tnetstring.loads(sock.recv())
method = req['method']
args = req['args']
print 'IN %s %s' % (method, args)
try:
resp = None
ret = None
if method == 'session-detect-rules-set':
rule_data_list = args['rules']
rlist = list()
for rule_data in rule_data_list:
r = Rule()
r.domain = rule_data['domain']
r.path_prefix = rule_data['path-prefix']
r.sid_ptr = rule_data['sid-ptr']
r.json_param = rule_data.get('json-param')
def test_roundtrip_format_random(self):
for _ in xrange(500):
v = get_random_object()
self.assertEqual(v, tnetstring.loads(tnetstring.dumps(v)))
self.assertEqual((v, ""), tnetstring.pop(tnetstring.dumps(v)))
def test_roundtrip_big_integer(self):
i1 = math.factorial(30000)
s = tnetstring.dumps(i1)
i2 = tnetstring.loads(s)
self.assertEquals(i1, i2)
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect('ipc:///tmp/pushpin-stats')
sock.setsockopt(zmq.SUBSCRIBE, 'sub ')
poller = zmq.Poller()
poller.register(sock, zmq.POLLIN)
while True:
socks = dict(poller.poll(1000))
if socks.get(sock) == zmq.POLLIN:
m_raw = sock.recv()
if not m_raw.startswith('sub T'):
continue
m = tnetstring.loads(m_raw[5:])
now = int(time.time())
sub_key = (m['mode'], m['channel'])
sub = subs.get(sub_key)
if m.get('unavailable'):
if sub:
del subs[sub_key]
print 'UNSUB mode=%s channel=%s' % (m['mode'], m['channel'])
else:
if not sub:
sub = Subscription()
subs[sub_key] = sub
print 'SUB mode=%s channel=%s' % (m['mode'], m['channel'])
sub.ttl = m['ttl']
sub.last_refresh = now
import sys
import json
import tnetstring
import zmq
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect(sys.argv[1])
sock.setsockopt(zmq.SUBSCRIBE, '')
while True:
m_raw = sock.recv()
at = m_raw.find(' ')
mtype = m_raw[:at]
mdata = m_raw[at + 1:]
if mdata[0] == 'T':
m = tnetstring.loads(mdata[1:])
elif mdata[0] == 'J':
m = json.loads(mdata[1:])
else:
m = mdata
print '%s %s' % (mtype, m)
pull_sock = zmq_context.socket(zmq.PULL) pull_sock.bind(pull_spec) pub_sock = zmq_context.socket(zmq.XPUB) pub_sock.bind(pub_spec) poller = zmq.Poller() poller.register(pull_sock, zmq.POLLIN) poller.register(pub_sock, zmq.POLLIN) subs = set() while True: socks = dict(poller.poll()) if socks.get(pull_sock) == zmq.POLLIN: m = tnetstring.loads(pull_sock.recv()) channel = m['channel'] if channel in subs: del m['channel'] pub_sock.send_multipart([channel, tnetstring.dumps(m)]) elif socks.get(pub_sock) == zmq.POLLIN: m = pub_sock.recv() mtype = m[0] topic = m[1:] if mtype == '\x01': assert(topic not in subs) print 'subscribing [%s]' % topic subs.add(topic) elif mtype == '\x00': assert(topic in subs) print 'unsubscribing [%s]' % topic
state = 0 # 0=connecting, 1=connected, 2=closing
m = dict()
m['from'] = client_id
m['id'] = rid
m['seq'] = outseq
m['uri'] = sys.argv[1]
m['credits'] = 200000
print 'OUT: %s' % m
out_sock.send('T' + tnetstring.dumps(m))
outseq += 1
while True:
m_raw = in_sock.recv()
at = m_raw.find(' ')
m = tnetstring.loads(m_raw[at + 2:])
if m['id'] != rid:
continue
print 'IN: %s' % m
assert(m['seq'] == inseq)
inseq += 1
if state == 0:
mtype = m.get('type')
if mtype is not None:
break
# connected
state = 1
raddr = m['from']
"method": "POST",
"url": sys.argv[1],
"headers": [["Content-Length", str(len(file))]],
"stream": True,
"credits": 200000,
"more": True,
}
)
)
outseq += 1
while True:
buf = in_sock.recv()
at = buf.find(" ")
receiver = buf[:at]
data = tnetstring.loads(buf[at + 1 :])
print receiver, data
if "error" in data or ("body" in data and "more" not in data):
break
if "credits" in data:
outcredits += data["credits"]
raddr = data["reply-address"]
if "body" in data and len(data["body"]) > 0:
out_stream_sock.send_multipart(
[raddr, "", tnetstring.dumps({"id": rid, "sender": client_id, "seq": outseq, "credits": len(data["body"])})]
)
outseq += 1
if pos < len(file) and outcredits > 0:
chunk = file[pos : pos + outcredits]
outcredits -= len(chunk)
pos += len(chunk)
def test_roundtrip_big_integer(self):
i1 = math.factorial(30000)
s = tnetstring.dumps(i1)
i2 = tnetstring.loads(s)
self.assertEquals(i1, i2)
def nodestats_worker(c):
in_sock = zmq_context.socket(zmq.SUB)
in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
connected_to = set()
out_sock = zmq_context.socket(zmq.PUB)
out_sock.linger = 0
out_sock.connect(update_spec)
stats = dict()
cur_id = -1
cur_body = ''
updated = False
last_send = 0
send_freq = 100
poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
while True:
nodes = db.node_get_all()
# remove any
for id in connected_to:
if id not in nodes:
# lost a connection, start over
in_sock.close()
in_sock = zmq_context.socket(zmq.SUB)
in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
connected_to.clear()
break
# add any
for id, node in nodes.iteritems():
if node['type'] != 'client':
continue
if id not in connected_to and node.get('public-addr'):
in_sock.connect('tcp://%s:10101' % node['public-addr'])
connected_to.add(id)
logger.info('stats connected to %s' % id)
to_remove = list()
for pub_id, s in stats.iteritems():
found = False
for id, node in nodes.iteritems():
if node['type'] != 'client':
continue
if node.get('manager-id') == pub_id:
found = True
break
if not found:
to_remove.append(pub_id)
for pub_id in to_remove:
del stats[pub_id]
timeout = 5000 # max timeout, so we can update stats connections
if updated:
now = int(time.time() * 1000)
if last_send + send_freq > now:
timeout = send_freq - (now - last_send)
else:
timeout = 0
socks = dict(poller.poll(timeout))
if socks.get(in_sock) == zmq.POLLIN:
m_raw = in_sock.recv()
at = m_raw.find(' ')
m = tnetstring.loads(m_raw[at + 1:])
pub_id = m['id']
s = stats.get(pub_id)
if s is None:
s = dict()
stats[pub_id] = s
s['received'] = 0
s['rtimes'] = list()
if 'cur-id' in m and m['cur-id'] != cur_id:
cur_id = m['cur-id']
cur_body = m['cur-body']
# reset receive info on new id
for id, st in stats.iteritems():
st['received'] = 0
st['rtimes'] = list()
# who sent this?
pub_node = None
for id, node in nodes.iteritems():
if node['type'] != 'client':
continue
if node.get('manager-id') == pub_id:
pub_node = node
break
if 'latency' in m:
latency = m['latency']
else:
latency = 0
if pub_node is not None and 'ping' in pub_node:
latency += pub_node['ping'] / 2
s['total'] = m['total']
s['started'] = m['started']
s['errored'] = m['errored']
had_received = s['received']
s['received'] = m['received']
if 'cur-id' in m:
s['cur-id'] = m['cur-id']
now = int(time.time() * 1000)
if m['received'] > had_received:
s['rtimes'].append((now, m['received'] - had_received, latency))
elif m['received'] < had_received:
s['rtimes'] = list()
s['rtimes'].append((now, m['received'], latency))
updated = True
if updated:
now = int(time.time() * 1000)
if last_send + send_freq < now:
last_send = now
updated = False
total = 0
started = 0
received = 0
errored = 0
rtimes = list()
for id, s in stats.iteritems():
total += s['total']
started += s['started']
if 'cur-id' in s and s['cur-id'] == cur_id:
received += s['received']
errored += s['errored']
if len(s['rtimes']) > 0:
rtimes.extend(s['rtimes'])
s['rtimes'] = list()
if len(rtimes) > 0:
rcount = 0
latency_total = 0
for rtime in rtimes:
rcount += rtime[1]
latency_total += (now - rtime[0] + rtime[2]) * rtime[1]
latency = latency_total / rcount
else:
latency = None
m = dict()
m['total'] = total
m['started'] = started
m['received'] = received
m['errored'] = errored
if cur_id >= 0:
m['cur-id'] = cur_id
m['cur-body'] = cur_body
if latency is not None:
m['latency'] = latency
else:
m['latency'] = 0
out_sock.send('stats ' + tnetstring.dumps(m))
# this handler just outputs the request ID
import tnetstring
import zmq
ctx = zmq.Context()
sock = ctx.socket(zmq.PULL)
sock.connect('ipc://client-out')
while True:
m = sock.recv_multipart()
req = tnetstring.loads(m[0][1:])
print('{} {}'.format(req[b'from'].decode(), req[b'id'].decode()))
out_sock.send(s.to_address + ' T' + tnetstring.dumps(resp)) lock.release() class KeepAliveThread(threading.Thread): def run(self): while True: keepalive_worker() time.sleep(30) keepalive_thread = KeepAliveThread() keepalive_thread.daemon = True keepalive_thread.start() while True: m_raw = in_sock.recv() req = tnetstring.loads(m_raw[1:]) print 'IN %s' % req id = req['id'] if id in sessions: print 'session already active' continue if 'type' in req: print 'wrong packet type' continue s = Session() s.to_address = req['from'] lock.acquire() sessions[id] = s
if len(sys.argv) < 2:
print 'usage: %s [url]' % sys.argv[0]
sys.exit(1)
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('ipc:///tmp/zurl-req')
req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = 'GET'
req['uri'] = sys.argv[1]
#req['follow-redirects'] = True
#req['ignore-tls-errors'] = True
sock.send('T' + tnetstring.dumps(req))
resp = tnetstring.loads(sock.recv()[1:])
if 'type' in resp and resp['type'] == 'error':
print 'error: %s' % resp['condition']
sys.exit(1)
print 'code=%d reason=[%s]' % (resp['code'], resp['reason'])
for h in resp['headers']:
print '%s: %s' % (h[0], h[1])
if 'body' in resp:
print '\n%s' % resp['body']
else:
print '\n'
def test_roundtrip_format_unicode(self):
for _ in xrange(500):
v = get_random_object(unicode=True)
self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v,"utf8"),"utf8"))
self.assertEqual((v,""),tnetstring.pop(tnetstring.dumps(v,"utf16"),"utf16"))
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')
poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)
while True:
socks = dict(poller.poll())
if socks.get(in_sock) == zmq.POLLIN:
m_raw = in_sock.recv()
req = tnetstring.loads(m_raw[1:])
print 'IN %s' % req
id = req['id']
if id in sessions:
print 'session already active'
continue
if 'type' in req:
print 'wrong packet type'
continue
s = Session()
sessions[id] = s
s.file = open(filename, 'r')
'method': 'POST',
'uri': sys.argv[1],
'headers': [['Content-Length', str(len(file))]],
'stream': True,
'credits': 200000,
'more': True
}))
outseq += 1
raddr = None
while True:
buf = in_sock.recv()
at = buf.find(' ')
receiver = buf[:at]
data = tnetstring.loads(buf[at + 2:])
print 'IN: %s %s' % (receiver, data)
if ('type' in data
and data['type'] == 'error') or ('type' not in data
and 'more' not in data):
break
if 'type' in data and data['type'] == 'keep-alive':
odata = {
'id': rid,
'from': client_id,
'seq': outseq,
'type': 'keep-alive'
}
print 'OUT: %s' % odata
out_stream_sock.send_multipart(
[raddr, '', 'T' + tnetstring.dumps(odata)])
import zmq
if len(sys.argv) < 2:
print 'usage: %s [url]' % sys.argv[0]
sys.exit(1)
ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('ipc:///tmp/zurl-req')
req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = 'GET'
req['uri'] = sys.argv[1]
#req['ignore-tls-errors'] = True
sock.send('T' + tnetstring.dumps(req))
resp = tnetstring.loads(sock.recv()[1:])
if 'type' in resp and resp['type'] == 'error':
print 'error: %s' % resp['condition']
sys.exit(1)
print 'code=%d reason=[%s]' % (resp['code'], resp['reason'])
for h in resp['headers']:
print '%s: %s' % (h[0], h[1])
if 'body' in resp:
print '\n%s' % resp['body']
else:
print '\n'
def test_unicode_handling(self):
with self.assertRaises(ValueError):
tnetstring.dumps("hello")
self.assertEqual(tnetstring.dumps("hello".encode()),b"5:hello,")
self.assertEqual(type(tnetstring.loads(b"5:hello,")),bytes)
def stats_worker(c):
in_sock = zmq_context.socket(zmq.SUB)
in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
in_sock.setsockopt(zmq.SUBSCRIBE, 'update ')
in_sock.bind(update_spec)
c.ready()
out_sock = zmq_context.socket(zmq.PUB)
out_sock.linger = 0
out_sock.bind(stats_out_spec)
received = 0
rtimes = list()
receive_min = 0
receive_max = 0
receive_avg = 0
while True:
m_raw = in_sock.recv()
at = m_raw.find(' ')
mtype = m_raw[:at]
mreceived = None
if mtype == 'stats':
m = tnetstring.loads(m_raw[at + 1:])
logger.debug('stats: %s' % m)
mreceived = m['received']
mlatency = m['latency']
elif mtype == 'update':
utype = m_raw[at + 1:]
logger.debug('update: ' + utype)
count = db.config_get_count()
nodes = db.node_get_all()
edge_up = 0
client_up = 0
for id, node in nodes.iteritems():
if node['type'] == 'edge':
if 'ping' in node:
edge_up += 1
elif node['type'] == 'client':
if 'ping' in node:
client_up += 1
if count > 0:
edge_need = ((count - 1) / edge_capacity) + 1
client_need = ((count - 1) / client_capacity) + 1
else:
edge_need = 0
client_need = 0
ping_data = db.get_ping_data()
stats = dict()
stats['capacity'] = count
stats['edge-up'] = edge_up
stats['edge-total'] = edge_need
stats['client-up'] = client_up
stats['client-total'] = client_need
if 'min' in ping_data:
stats['ping-min'] = ping_data['min']
else:
stats['ping-min'] = 0
if 'max' in ping_data:
stats['ping-max'] = ping_data['max']
else:
stats['ping-max'] = 0
if 'avg' in ping_data:
stats['ping-avg'] = ping_data['avg']
else:
stats['ping-avg'] = 0
origin_info = db.get_origin_info()
if mreceived is not None:
latency = mlatency
if 'ping' in origin_info:
latency += origin_info['ping'] / 2
now = int(time.time() * 1000)
had_received = received
received = mreceived
if mreceived > had_received:
rtimes.append((now, mreceived - had_received, latency))
else:
rtimes = list()
rtimes.append((now, mreceived, latency))
# calculate delivery times
start_time = db.send_get_start()
rmin = None
rmax = None
rtotal = 0
rcount = 0
for rtime in rtimes:
t = rtime[0] - start_time - rtime[2]
if t < 0:
t = 0 # should not happen
if rmin is None or t < rmin:
rmin = t
if rmax is None or t > rmax:
rmax = t
rcount += rtime[1]
rtotal += t * rtime[1]
if rmin is not None:
receive_min = rmin
else:
receive_min = 0
if rmax is not None:
receive_max = rmax
else:
receive_max = 0
if rtotal > 0:
receive_avg = rtotal / rcount
else:
receive_avg = 0
stats['received'] = received
stats['receive-min'] = receive_min
stats['receive-max'] = receive_max
stats['receive-avg'] = receive_avg
curdata = db.get_stats_data()
if curdata is not None:
del curdata['id']
if curdata == stats:
# stats are the same as before. don't write/send
continue
stats['id'] = db.set_stats_data(stats)
out_sock.send('stats ' + json.dumps(stats))
def test_roundtrip_format_examples(self):
for data, expect in FORMAT_EXAMPLES.items():
self.assertEqual(expect, tnetstring.loads(data))
self.assertEqual(expect,
tnetstring.loads(tnetstring.dumps(expect)))
self.assertEqual((expect, ""), tnetstring.pop(data))
def _read_pkt (self, _list, timeout = None):
c = self._redis.blpop ([_list], timeout or 5)
if None is c:
return None
return tnetstring.loads (c [1], 'iso-8859-15')
# adapted from mongrel2's manual
import sys
import zmq
import tnetstring
from pprint import pprint
CTX = zmq.Context()
addr = sys.argv[1]
ctl = CTX.socket(zmq.REQ)
print 'CONNECTING'
ctl.connect(addr)
while True:
cmd = raw_input('> ')
# will only work with simple commands that have no arguments
ctl.send(tnetstring.dumps([cmd, {}]))
resp = ctl.recv()
pprint(tnetstring.loads(resp))
ctl.close()
def thrash_tnetstring():
for obj, tns, json in TESTS:
assert tnetstring.loads(tns) == obj
assert tnetstring.loads(tnetstring.dumps(obj)) == obj
in_sock = ctx.socket(zmq.SUB)
in_sock.setsockopt(zmq.SUBSCRIBE, client_id)
in_sock.connect('ipc:///tmp/zurl-out')
time.sleep(0.5)
rid = str(uuid.uuid4())
inseq = 0
outseq = 0
out_sock.send('T' + tnetstring.dumps({'from': client_id, 'id': rid, 'seq': outseq, 'method': 'GET', 'uri': sys.argv[1], 'stream': True, 'credits': 230000}))
outseq += 1
while True:
buf = in_sock.recv()
at = buf.find(' ')
receiver = buf[:at]
indata = tnetstring.loads(buf[at + 2:])
if indata['id'] != rid:
continue
print 'IN: %s' % indata
assert(indata['seq'] == inseq)
inseq += 1
if ('type' in indata and (indata['type'] == 'error' or indata['type'] == 'cancel')) or ('type' not in indata and 'more' not in indata):
break
raddr = indata['from']
if 'body' in indata and len(indata['body']) > 0:
outdata = {'id': rid, 'from': client_id, 'seq': outseq, 'type': 'credit', 'credits': len(indata['body'])}
print 'OUT: %s' % outdata
out_stream_sock.send_multipart([raddr, '', 'T' + tnetstring.dumps(outdata)])
outseq += 1
def test_unicode_handling(self):
with self.assertRaises(ValueError):
tnetstring.dumps("hello")
self.assertEqual(tnetstring.dumps("hello".encode()), b"5:hello,")
self.assertEqual(type(tnetstring.loads(b"5:hello,")), bytes)
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')
poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)
while True:
socks = dict(poller.poll())
if socks.get(in_sock) == zmq.POLLIN:
m_raw = in_sock.recv()
req = tnetstring.loads(m_raw[1:])
print 'IN %s' % req
id = req['id']
if id in sessions:
print 'session already active'
continue
s = Session()
s.id = id
headers = req.get('headers')
if headers:
for h in headers:
if h[0].lower() == 'content-type':
s.content_type = h[1]
break
import sys
import tnetstring
import zmq
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect(sys.argv[1])
sock.setsockopt(zmq.SUBSCRIBE, '')
while True:
m_raw = sock.recv()
at = m_raw.find(' ')
mtype = m_raw[:at]
mdata = m_raw[at + 1:]
if mdata[0] == 'T':
m = tnetstring.loads(mdata[1:])
else:
m = tnetstring.loads(mdata)
print '%s %s' % (mtype, m)
def test_roundtrip_format_examples(self):
for data, expect in FORMAT_EXAMPLES.items():
self.assertEqual(expect,tnetstring.loads(data))
self.assertEqual(expect,tnetstring.loads(tnetstring.dumps(expect)))
self.assertEqual((expect,""),tnetstring.pop(data))
if s is None:
raise ValueError('unknown sid')
for k, v in last_ids.iteritems():
s.last_ids[k] = v
def session_get_last_ids(sid):
s = sessions.get(sid)
if s is not None:
return s.last_ids
else:
return None
while True:
req = tnetstring.loads(sock.recv())
method = req['method']
args = req['args']
print 'IN %s %s' % (method, args)
try:
resp = None
ret = None
if method == 'session-detect-rules-set':
rule_data_list = args['rules']
rlist = list()
for rule_data in rule_data_list:
r = Rule()
r.domain = rule_data['domain']
r.path_prefix = rule_data['path-prefix']
r.sid_ptr = rule_data['sid-ptr']
r.json_param = rule_data.get('json-param')
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')
poller = zmq.core.poll.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)
while True:
socks = dict(poller.poll())
if socks.get(in_sock) == zmq.POLLIN:
m_raw = in_sock.recv()
req = tnetstring.loads(m_raw)
print 'IN %s' % req
id = req['id']
if id in sessions:
print 'session already active'
continue
if 'type' in req:
print 'wrong packet type'
continue
s = Session()
sessions[id] = s
s.file = open(filename, 'r')
def test_roundtrip_format_random(self):
for _ in xrange(500):
v = get_random_object()
self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v)))
self.assertEqual((v,""),tnetstring.pop(tnetstring.dumps(v)))