Exemplo n.º 1
0
def thrash_tnetstring():
    for obj, tns, json, msh in TESTS:
        if TEST_DUMP_ONLY:
            tnetstring.dumps(obj)
        elif TEST_LOAD_ONLY:
            assert tnetstring.loads(tns) == obj
        else:
            assert tnetstring.loads(tnetstring.dumps(obj)) == obj
Exemplo n.º 2
0
 def test_unicode_handling(self):
     self.assertRaises(ValueError,tnetstring.dumps,u"hello")
     self.assertEquals(tnetstring.dumps(u"hello","utf8"),"5:hello,")
     self.assertEquals(type(tnetstring.loads("5:hello,")),str)
     self.assertEquals(type(tnetstring.loads("5:hello,","utf8")),unicode)
     ALPHA = u"\N{GREEK CAPITAL LETTER ALPHA}lpha"
     self.assertEquals(tnetstring.dumps(ALPHA,"utf8"),"6:"+ALPHA.encode("utf8")+",")
     self.assertEquals(tnetstring.dumps(ALPHA,"utf16"),"12:"+ALPHA.encode("utf16")+",")
     self.assertEquals(tnetstring.loads("12:\xff\xfe\x91\x03l\x00p\x00h\x00a\x00,","utf16"),ALPHA)
Exemplo n.º 3
0
 def test_unicode_handling(self):
     self.assertRaises(ValueError, tnetstring.dumps, u"hello")
     self.assertEquals(tnetstring.dumps(u"hello", "utf8"), "5:hello,")
     self.assertEquals(type(tnetstring.loads("5:hello,")), str)
     self.assertEquals(type(tnetstring.loads("5:hello,", "utf8")), unicode)
     ALPHA = u"\N{GREEK CAPITAL LETTER ALPHA}lpha"
     self.assertEquals(tnetstring.dumps(ALPHA, "utf8"),
                       "6:" + ALPHA.encode("utf8") + ",")
     self.assertEquals(tnetstring.dumps(ALPHA, "utf16"),
                       "12:" + ALPHA.encode("utf16") + ",")
     self.assertEquals(
         tnetstring.loads("12:\xff\xfe\x91\x03l\x00p\x00h\x00a\x00,",
                          "utf16"), ALPHA)
Exemplo n.º 4
0
def process_data(mtype, mdata):
    """
    This method processes the data sent from Pushpin's socket
    """
    mdecode = tnetstring.loads(mdata[1:])
    print('received: %s, %s' % mtype % mdecode)
    if mtype == b'conn':
        # unavail = False
        c_id = 'none'
        conn_num = 0
        peer_ip = '0.0.0.0'
        unavail = b'unavailable' in mdecode
        if b'id' in mdecode:
            c_id = mdecode[b'id'].decode('UTF-8')
        ppstatconn = PushpinStat_Conn(unavailable=unavail, conn_id=c_id, conn_num=conn_num, peer_ip=peer_ip, type='ws')
        ppstatconn.save()
    elif mytpe == b'sub' or mtype == b'report' or mtype == b'activity' or mtype == b'message':
        channel = ''
        mode = ''
        sub_cnt = 0
        if b'channel' in mdecode:
            channel = mdecode[b'channel']
        if b'mode' in mdecode:
            mode = mdecode[b'mode']
        if b'subscribers' in mdecode:
            sub_cnt = mdecode[b'subscribers']
        unavailable = b'unavailable' in mdecode
        ppstatsub = PushpinStat_Sub(channel=channel, mode=mode, sub_cnt=sub_cnt, unavailable=unavailable)
        ppstatsub.save()
    else:
        print('not a valid type')
Exemplo n.º 5
0
 def test_roundtrip_format_unicode(self):
     for _ in xrange(500):
         v = get_random_object(unicode=True)
         self.assertEqual(
             v, tnetstring.loads(tnetstring.dumps(v, "utf8"), "utf8"))
         self.assertEqual((v, ""),
                          tnetstring.pop(tnetstring.dumps(v, "utf16"),
                                         "utf16"))
Exemplo n.º 6
0
    def get(self, sid):
        if not self.is_valid_key(sid):
            return self.new()

        key = self.get_session_key(sid)
        saved = self.redis.get(key)
        if saved:
            data = tnetstring.loads(saved)
            return self.session_class(data, sid, False)
Exemplo n.º 7
0
 def __init__(self, key, fstore, path, create=False):
     self.id = key
     self.store = fstore
     self.path = path
     try:
         self.data = tnetstring.loads(fstore.open(path).read())
     except IOError:
         if not create:
             raise KeyError('Item does not exist')
         self.data = {}
Exemplo n.º 8
0
def read_packets (filename):
    try:
        os.stat (filename)
    except OSError:
        print "No such file : %s"%filename
        sys.exit (1)
    pkts = open (filename).read ()
    pkts = tnetstring.loads (pkts, 'iso-8859-15')
    for data in pkts:
        yield data
Exemplo n.º 9
0
 def __init__(self, key, fstore, path, create=False):
     self.id = key
     self.store = fstore
     self.path = path
     try:
         self.data = tnetstring.loads(fstore.open(path).read())
     except IOError:
         if not create:
             raise KeyError("Item does not exist")
         self.data = {}
Exemplo n.º 10
0
def process_data(m_type, m_data):
    """
    This method processes the data sent from Pushpin's socket
    """
    m_decode = tnetstring.loads(m_data[1:])
    print('in process_data: m_type=', m_type)
    print('m_decode=', m_decode)

    # If there is new data, need to notify frontend that data is updated
    if decode_data(m_decode, m_type):
        send_update_notify()
Exemplo n.º 11
0
 def start_push(self):
     while True:
         m_raw = self.sock.recv().decode(
             "ISO-8859-1")  # .decode("ISO-8859-1")
         mtype, mdata = m_raw.split(" ", 1)
         if mdata[0] != "T":
             print("unsupported format")
             continue
         m = tnetstring.loads(mdata[1:].encode("ISO-8859-1"))
         print("%s %s" % (mtype, m))
         self.send_prom_push_gateway(msg_type=mtype, msg=m)
Exemplo n.º 12
0
 def _validate (self):
     if not self._raw_data or 2 is not len (self._raw_data):
         self._logger.error (
             "Invalid data from REDIS: '%s'"%self._raw_data)
         return False
     self._table, self._data = self._raw_data
     try:
         self._data = tnetstring.loads (self._data, 'iso-8859-15')
     except ValueError, e:
         self._logger.critical (
             "Invalid tnetstring data : '%s'"%self._data)
         return False
Exemplo n.º 13
0
	def call(self, method, args=dict()):
		req = dict()
		req['id'] = str(uuid.uuid4())
		req['method'] = ensure_utf8(method)
		req['args'] = ensure_utf8(args)
		req_raw = tnetstring.dumps(req)
		try:
			if not self.sock.poll(30000, zmq.POLLOUT):
				raise CallError('send-timeout')
			m_list = list()
			m_list.append('')
			m_list.append(req_raw)
			self.sock.send_multipart(m_list)
		except zmq.ZMQError as e:
			raise CallError('send-failed', e.message)
		start = int(time.clock() * 1000)
		while True:
			elapsed = max(int(time.clock() * 1000) - start, 0)
			try:
				if not self.sock.poll(max(30000 - elapsed, 0), zmq.POLLIN):
					raise CallError('receive-timeout')
				m_list = self.sock.recv_multipart()
			except zmq.ZMQError as e:
				raise CallError('receive-failed', e.message)
			if len(m_list) != 2:
				print "response has wrong number of parts, skipping"
				continue
			if len(m_list[0]) > 0:
				print "response first part is not empty, skipping"
				continue
			resp_raw = m_list[1]
			try:
				resp = tnetstring.loads(resp_raw)
			except:
				print "failed to parse response as tnetstring, skipping"
				continue
			if 'id' not in resp:
				print "response missing id field, skipping"
				continue
			if resp['id'] != req['id']:
				print "unexpected response id, skipping"
				continue
			break
		if 'success' not in resp:
			raise CallError('invalid-response', 'missing success field')
		if resp['success']:
			if 'value' not in resp:
				raise CallError('invalid-response', 'missing value field')
			return resp['value']
		else:
			if 'condition' not in resp:
				raise CallError('invalid-response', 'missing condition field')
			raise CallError(resp['condition'])
Exemplo n.º 14
0
	def call(self, method, args, timeout=10000):
		req = dict()
		req['id'] = str(uuid.uuid4())
		req['method'] = ensure_utf8(method)
		req['args'] = ensure_utf8(args)
		req_raw = tnetstring.dumps(req)
		start = int(time.clock() * 1000)
		try:
			if not self.sock.poll(timeout, zmq.POLLOUT):
				raise CallError('send-timeout')
			m_list = list()
			m_list.append('')
			m_list.append(req_raw)
			self.sock.send_multipart(m_list)
		except zmq.ZMQError as e:
			raise CallError('send-failed', e.message)
		while True:
			elapsed = max(int(time.clock() * 1000) - start, 0)
			try:
				if not self.sock.poll(max(timeout - elapsed, 0), zmq.POLLIN):
					raise CallError('receive-timeout')
				m_list = self.sock.recv_multipart()
			except zmq.ZMQError as e:
				raise CallError('receive-failed', e.message)
			if len(m_list) != 2:
				print "response has wrong number of parts, skipping"
				continue
			if len(m_list[0]) > 0:
				print "response first part is not empty, skipping"
				continue
			resp_raw = m_list[1]
			try:
				resp = tnetstring.loads(resp_raw)
			except:
				print "failed to parse response as tnetstring, skipping"
				continue
			if 'id' not in resp:
				print "response missing id field, skipping"
				continue
			if resp['id'] != req['id']:
				print "unexpected response id, skipping"
				continue
			break
		if 'success' not in resp:
			raise CallError('invalid-response', 'missing success field')
		if resp['success']:
			if 'value' not in resp:
				raise CallError('invalid-response', 'missing value field')
			return resp['value']
		else:
			if 'condition' not in resp:
				raise CallError('invalid-response', 'missing condition field')
			raise CallError(resp['condition'])
Exemplo n.º 15
0
 def pkt_read_parse (self, info, timeout = None, maxtry = 10,
                     reemission = False, sid = False):
     _reply = SIPpacket ()
     _r = self.pkt_read (info, timeout, sid) if not reemission else \
         self.pkt_read_retry (info, maxtry, sid)
     if not _r:
         return None
     _a = tnetstring.loads (_r [1], 'iso-8859-15')
     s_id, timestamp, addr, pkt = _a
     if self._debug:
         print "<=== received from %s at %s"%(addr, time.ctime (timestamp))
         print str (pkt) 
     pkt = self.pkt_parse (pkt)
     return (s_id, timestamp, addr, pkt)
Exemplo n.º 16
0
    def __init__(self, base, create=False):
        self.base = op.expandvars(op.expanduser(base))
        if not op.exists(self.base):
            if create:
                os.mkdir(self.base)
            else:
                raise ValueError("%s does not exist" % base)

        if not op.isdir(self.base):
            raise ValueError("%s is not a directory" % base)

        try:
            self.index = tnetstring.loads(self.open("index.gz").read())
        except IOError:
            self.index = {}
Exemplo n.º 17
0
    def __init__(self, base, create=False):
        self.base = op.expandvars(op.expanduser(base))
        if not op.exists(self.base):
            if create:
                os.mkdir(self.base)
            else:
                raise ValueError('%s does not exist' % base)

        if not op.isdir(self.base):
            raise ValueError('%s is not a directory' % base)

        try:
            self.index = tnetstring.loads(self.open('index.gz').read())
        except IOError:
            self.index = {}
Exemplo n.º 18
0
	def run(self, handler, data):
		poller = zmq.Poller()
		poller.register(self.control_sock, zmq.POLLIN)
		poller.register(self.rep_sock, zmq.POLLIN)

		while True:
			socks = dict(poller.poll())
			if socks.get(self.control_sock) == zmq.POLLIN:
				s = self.control_sock.recv()
				if s == 'stop':
					break
			elif socks.get(self.rep_sock) == zmq.POLLIN:
				req_raw = self.rep_sock.recv()
				try:
					req = tnetstring.loads(req_raw)
				except:
					self._respond_error('bad-request')
					continue

				self.req_id = req.get('id')

				method = req.get('method')
				if not method:
					self._respond_error('bad-request')
					continue

				args = req.get('args')
				if args is None:
					args = dict()
				if not isinstance(args, dict):
					self._respond_error('bad-request')
					continue

				try:
					ret = handler(method, args, data=data)
					self._respond(ret)
				except CallError as e:
					self._respond_error(e.condition)
				except:
					traceback.print_exc()
					self._respond_error('internal-server-error')

		self.rep_sock.linger = 0
		self.rep_sock.close()
		self.control_sock.send('finished')
		self.control_sock.close()
Exemplo n.º 19
0
	def _discover_uris(self):
		self._thread_cond.acquire()
		if self._discovery_in_progress:
			self._thread_cond.wait()
			self._thread_cond.release()
			self._verify_discovered_uris()
			return
		else:
			self._discovery_in_progress = True
			self._thread_cond.release()
		if (self.uri is None or self._discovery_completed or
				(self.pub_uri and self.push_uri)):
			self._end_discovery(False)
			return
		sock = self._context.socket(zmq.REQ)
		sock.linger = 0
		sock.connect(self.uri)
		start = int(timeit.default_timer() * 1000)
		if not sock.poll(3000, zmq.POLLOUT):
			sock.close()
			self._end_discovery(False)
			raise ValueError('uri discovery request failed: pollout timeout')
		req = {_ensure_utf8('method'): _ensure_utf8('get-zmq-uris')}
		sock.send(tnetstring.dumps(req))
		elapsed = max(int(timeit.default_timer() * 1000) - start, 0)
		if not sock.poll(max(3000 - elapsed, 0), zmq.POLLIN):
			sock.close()
			self._end_discovery(False)
			raise ValueError('uri discovery request failed: pollin timeout')
		resp = tnetstring.loads(sock.recv())
		sock.close()
		if (not resp.get(_ensure_utf8('success')) or
				not resp.get(_ensure_utf8('value'))):
			self._end_discovery(False)
			raise ValueError('uri discovery request failed: %s' % resp)
		self._set_discovered_uris(resp[_ensure_utf8('value')])
		self._end_discovery(True)
		self._verify_discovered_uris()
Exemplo n.º 20
0
def stream_rtp (pkt = None):
    # SDP
    body = pkt.body
    sdp = SDP (str (body))
    rtp_address = sdp.conndata.address
    rtp_port = sdp.mediadata.port
    print "sending RTP _to_ %s:%d"%(str(rtp_address), int (rtp_port))
    sys.stdout.flush ()
    # RTP
    rtp_pkt = open ('../rtp/rtp.out.g711.py').read ()
    rtp_pkt = tnetstring.loads (rtp_pkt, 'iso-8859-15')
    try:
        for data_rtp in rtp_pkt:
            print type (data_rtp)
            client.pkt_send (session_id ['server_id_rtp'], data_rtp,
                             rtp_address, int (rtp_port))
            time.sleep (config.rtp_framing)
            print ".",
            sys.stdout.flush ()
    except KeyboardInterrupt:
        print "Stop sending RTP... waiting for BYE"
    session_id ['stream'] = False
    return
Exemplo n.º 21
0
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect('ipc:///tmp/pushpin-stats')
sock.setsockopt(zmq.SUBSCRIBE, 'sub ')

poller = zmq.Poller()
poller.register(sock, zmq.POLLIN)

while True:
    socks = dict(poller.poll(1000))
    if socks.get(sock) == zmq.POLLIN:
        m_raw = sock.recv()
        if not m_raw.startswith('sub T'):
            continue
        m = tnetstring.loads(m_raw[5:])

        now = int(time.time())
        sub_key = (m['mode'], m['channel'])
        sub = subs.get(sub_key)
        if m.get('unavailable'):
            if sub:
                del subs[sub_key]
                print 'UNSUB mode=%s channel=%s' % (m['mode'], m['channel'])
        else:
            if not sub:
                sub = Subscription()
                subs[sub_key] = sub
                print 'SUB mode=%s channel=%s' % (m['mode'], m['channel'])
            sub.ttl = m['ttl']
            sub.last_refresh = now
Exemplo n.º 22
0
poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)

while True:
    socks = dict(poller.poll(None))

    if socks.get(in_sock) == zmq.POLLIN:
        m_raw = in_sock.recv()
    elif socks.get(in_stream_sock) == zmq.POLLIN:
        m_list = in_stream_sock.recv_multipart()
        m_raw = m_list[2]
    else:
        continue

    req = tnetstring.loads(m_raw[1:])
    print('IN {}'.format(req))

    if req.get(b'type'):
        # skip all non-data messages
        continue

    if req.get(b'uri', b'').startswith(b'ws'):
        resp = {}
        resp[b'from'] = instance_id
        resp[b'id'] = req[b'id']
        resp[b'seq'] = 0
        resp[b'code'] = 101
        resp[b'reason'] = b'Switching Protocols'
        resp[b'credits'] = 1024
Exemplo n.º 23
0
def session_update(sid, last_ids):
	s = sessions.get(sid)
	if s is None:
		raise ValueError('unknown sid')
	for k, v in last_ids.iteritems():
		s.last_ids[k] = v

def session_get_last_ids(sid):
	s = sessions.get(sid)
	if s is not None:
		return s.last_ids
	else:
		return None

while True:
	req = tnetstring.loads(sock.recv())
	method = req['method']
	args = req['args']
	print 'IN %s %s' % (method, args)
	try:
		resp = None
		ret = None
		if method == 'session-detect-rules-set':
			rule_data_list = args['rules']
			rlist = list()
			for rule_data in rule_data_list:
				r = Rule()
				r.domain = rule_data['domain']
				r.path_prefix = rule_data['path-prefix']
				r.sid_ptr = rule_data['sid-ptr']
				r.json_param = rule_data.get('json-param')
Exemplo n.º 24
0
 def test_roundtrip_format_random(self):
     for _ in xrange(500):
         v = get_random_object()
         self.assertEqual(v, tnetstring.loads(tnetstring.dumps(v)))
         self.assertEqual((v, ""), tnetstring.pop(tnetstring.dumps(v)))
Exemplo n.º 25
0
 def test_roundtrip_big_integer(self):
     i1 = math.factorial(30000)
     s = tnetstring.dumps(i1)
     i2 = tnetstring.loads(s)
     self.assertEquals(i1, i2)
Exemplo n.º 26
0
ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect('ipc:///tmp/pushpin-stats')
sock.setsockopt(zmq.SUBSCRIBE, 'sub ')

poller = zmq.Poller()
poller.register(sock, zmq.POLLIN)

while True:
	socks = dict(poller.poll(1000))
	if socks.get(sock) == zmq.POLLIN:
		m_raw = sock.recv()
		if not m_raw.startswith('sub T'):
			continue
		m = tnetstring.loads(m_raw[5:])

		now = int(time.time())
		sub_key = (m['mode'], m['channel'])
		sub = subs.get(sub_key)
		if m.get('unavailable'):
			if sub:
				del subs[sub_key]
				print 'UNSUB mode=%s channel=%s' % (m['mode'], m['channel'])
		else:
			if not sub:
				sub = Subscription()
				subs[sub_key] = sub
				print 'SUB mode=%s channel=%s' % (m['mode'], m['channel'])
			sub.ttl = m['ttl']
			sub.last_refresh = now
Exemplo n.º 27
0
import sys
import json
import tnetstring
import zmq

ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect(sys.argv[1])
sock.setsockopt(zmq.SUBSCRIBE, '')

while True:
    m_raw = sock.recv()
    at = m_raw.find(' ')
    mtype = m_raw[:at]
    mdata = m_raw[at + 1:]
    if mdata[0] == 'T':
        m = tnetstring.loads(mdata[1:])
    elif mdata[0] == 'J':
        m = json.loads(mdata[1:])
    else:
        m = mdata
    print '%s %s' % (mtype, m)
Exemplo n.º 28
0
pull_sock = zmq_context.socket(zmq.PULL)
pull_sock.bind(pull_spec)

pub_sock = zmq_context.socket(zmq.XPUB)
pub_sock.bind(pub_spec)

poller = zmq.Poller()
poller.register(pull_sock, zmq.POLLIN)
poller.register(pub_sock, zmq.POLLIN)

subs = set()

while True:
	socks = dict(poller.poll())
	if socks.get(pull_sock) == zmq.POLLIN:
		m = tnetstring.loads(pull_sock.recv())
		channel = m['channel']
		if channel in subs:
			del m['channel']
			pub_sock.send_multipart([channel, tnetstring.dumps(m)])
	elif socks.get(pub_sock) == zmq.POLLIN:
		m = pub_sock.recv()
		mtype = m[0]
		topic = m[1:]
		if mtype == '\x01':
			assert(topic not in subs)
			print 'subscribing [%s]' % topic
			subs.add(topic)
		elif mtype == '\x00':
			assert(topic in subs)
			print 'unsubscribing [%s]' % topic
Exemplo n.º 29
0
state = 0 # 0=connecting, 1=connected, 2=closing

m = dict()
m['from'] = client_id
m['id'] = rid
m['seq'] = outseq
m['uri'] = sys.argv[1]
m['credits'] = 200000
print 'OUT: %s' % m
out_sock.send('T' + tnetstring.dumps(m))
outseq += 1

while True:
	m_raw = in_sock.recv()
	at = m_raw.find(' ')
	m = tnetstring.loads(m_raw[at + 2:])
	if m['id'] != rid:
		continue
	print 'IN: %s' % m
	assert(m['seq'] == inseq)
	inseq += 1

	if state == 0:
		mtype = m.get('type')
		if mtype is not None:
			break

		# connected
		state = 1
		raddr = m['from']
Exemplo n.º 30
0
Arquivo: post.py Projeto: agfeo/zurl
            "method": "POST",
            "url": sys.argv[1],
            "headers": [["Content-Length", str(len(file))]],
            "stream": True,
            "credits": 200000,
            "more": True,
        }
    )
)
outseq += 1

while True:
    buf = in_sock.recv()
    at = buf.find(" ")
    receiver = buf[:at]
    data = tnetstring.loads(buf[at + 1 :])
    print receiver, data
    if "error" in data or ("body" in data and "more" not in data):
        break
    if "credits" in data:
        outcredits += data["credits"]
    raddr = data["reply-address"]
    if "body" in data and len(data["body"]) > 0:
        out_stream_sock.send_multipart(
            [raddr, "", tnetstring.dumps({"id": rid, "sender": client_id, "seq": outseq, "credits": len(data["body"])})]
        )
        outseq += 1
    if pos < len(file) and outcredits > 0:
        chunk = file[pos : pos + outcredits]
        outcredits -= len(chunk)
        pos += len(chunk)
Exemplo n.º 31
0
 def test_roundtrip_big_integer(self):
     i1 = math.factorial(30000)
     s = tnetstring.dumps(i1)
     i2 = tnetstring.loads(s)
     self.assertEquals(i1, i2)
Exemplo n.º 32
0
def nodestats_worker(c):
	in_sock = zmq_context.socket(zmq.SUB)
	in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
	connected_to = set()

	out_sock = zmq_context.socket(zmq.PUB)
	out_sock.linger = 0
	out_sock.connect(update_spec)

	stats = dict()
	cur_id = -1
	cur_body = ''
	updated = False
	last_send = 0
	send_freq = 100

	poller = zmq.Poller()
	poller.register(in_sock, zmq.POLLIN)
	while True:
		nodes = db.node_get_all()

		# remove any
		for id in connected_to:
			if id not in nodes:
				# lost a connection, start over
				in_sock.close()
				in_sock = zmq_context.socket(zmq.SUB)
				in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
				connected_to.clear()
				break

		# add any
		for id, node in nodes.iteritems():
			if node['type'] != 'client':
				continue
			if id not in connected_to and node.get('public-addr'):
				in_sock.connect('tcp://%s:10101' % node['public-addr'])
				connected_to.add(id)
				logger.info('stats connected to %s' % id)

		to_remove = list()
		for pub_id, s in stats.iteritems():
			found = False
			for id, node in nodes.iteritems():
				if node['type'] != 'client':
					continue
				if node.get('manager-id') == pub_id:
					found = True
					break
			if not found:
				to_remove.append(pub_id)
		for pub_id in to_remove:
			del stats[pub_id]

		timeout = 5000 # max timeout, so we can update stats connections
		if updated:
			now = int(time.time() * 1000)
			if last_send + send_freq > now:
				timeout = send_freq - (now - last_send)
			else:
				timeout = 0
		socks = dict(poller.poll(timeout))
		if socks.get(in_sock) == zmq.POLLIN:
			m_raw = in_sock.recv()
			at = m_raw.find(' ')
			m = tnetstring.loads(m_raw[at + 1:])
			pub_id = m['id']
			s = stats.get(pub_id)
			if s is None:
				s = dict()
				stats[pub_id] = s
				s['received'] = 0
				s['rtimes'] = list()
			if 'cur-id' in m and m['cur-id'] != cur_id:
				cur_id = m['cur-id']
				cur_body = m['cur-body']
				# reset receive info on new id
				for id, st in stats.iteritems():
					st['received'] = 0
					st['rtimes'] = list()

			# who sent this?
			pub_node = None
			for id, node in nodes.iteritems():
				if node['type'] != 'client':
					continue
				if node.get('manager-id') == pub_id:
					pub_node = node
					break

			if 'latency' in m:
				latency = m['latency']
			else:
				latency = 0
			if pub_node is not None and 'ping' in pub_node:
				latency += pub_node['ping'] / 2

			s['total'] = m['total']
			s['started'] = m['started']
			s['errored'] = m['errored']
			had_received = s['received']
			s['received'] = m['received']
			if 'cur-id' in m:
				s['cur-id'] = m['cur-id']
			now = int(time.time() * 1000)
			if m['received'] > had_received:
				s['rtimes'].append((now, m['received'] - had_received, latency))
			elif m['received'] < had_received:
				s['rtimes'] = list()
				s['rtimes'].append((now, m['received'], latency))
			updated = True

		if updated:
			now = int(time.time() * 1000)
			if last_send + send_freq < now:
				last_send = now
				updated = False

				total = 0
				started = 0
				received = 0
				errored = 0
				rtimes = list()
				for id, s in stats.iteritems():
					total += s['total']
					started += s['started']
					if 'cur-id' in s and s['cur-id'] == cur_id:
						received += s['received']
					errored += s['errored']
					if len(s['rtimes']) > 0:
						rtimes.extend(s['rtimes'])
					s['rtimes'] = list()

				if len(rtimes) > 0:
					rcount = 0
					latency_total = 0
					for rtime in rtimes:
						rcount += rtime[1]
						latency_total += (now - rtime[0] + rtime[2]) * rtime[1]
					latency = latency_total / rcount
				else:
					latency = None

				m = dict()
				m['total'] = total
				m['started'] = started
				m['received'] = received
				m['errored'] = errored
				if cur_id >= 0:
					m['cur-id'] = cur_id
					m['cur-body'] = cur_body
				if latency is not None:
					m['latency'] = latency
				else:
					m['latency'] = 0

				out_sock.send('stats ' + tnetstring.dumps(m))
Exemplo n.º 33
0
# this handler just outputs the request ID

import tnetstring
import zmq

ctx = zmq.Context()
sock = ctx.socket(zmq.PULL)
sock.connect('ipc://client-out')

while True:
    m = sock.recv_multipart()
    req = tnetstring.loads(m[0][1:])
    print('{} {}'.format(req[b'from'].decode(), req[b'id'].decode()))
Exemplo n.º 34
0
		out_sock.send(s.to_address + ' T' + tnetstring.dumps(resp))
	lock.release()

class KeepAliveThread(threading.Thread):
	def run(self):
		while True:
			keepalive_worker()
			time.sleep(30)

keepalive_thread = KeepAliveThread()
keepalive_thread.daemon = True
keepalive_thread.start()

while True:
	m_raw = in_sock.recv()
	req = tnetstring.loads(m_raw[1:])
	print 'IN %s' % req

	id = req['id']
	if id in sessions:
		print 'session already active'
		continue

	if 'type' in req:
		print 'wrong packet type'
		continue

	s = Session()
	s.to_address = req['from']
	lock.acquire()
	sessions[id] = s
Exemplo n.º 35
0
Arquivo: get.py Projeto: namezis/zurl
if len(sys.argv) < 2:
    print 'usage: %s [url]' % sys.argv[0]
    sys.exit(1)

ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('ipc:///tmp/zurl-req')

req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = 'GET'
req['uri'] = sys.argv[1]
#req['follow-redirects'] = True
#req['ignore-tls-errors'] = True
sock.send('T' + tnetstring.dumps(req))

resp = tnetstring.loads(sock.recv()[1:])
if 'type' in resp and resp['type'] == 'error':
    print 'error: %s' % resp['condition']
    sys.exit(1)

print 'code=%d reason=[%s]' % (resp['code'], resp['reason'])
for h in resp['headers']:
    print '%s: %s' % (h[0], h[1])

if 'body' in resp:
    print '\n%s' % resp['body']
else:
    print '\n'
Exemplo n.º 36
0
 def test_roundtrip_format_unicode(self):
     for _ in xrange(500):
         v = get_random_object(unicode=True)
         self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v,"utf8"),"utf8"))
         self.assertEqual((v,""),tnetstring.pop(tnetstring.dumps(v,"utf16"),"utf16"))
Exemplo n.º 37
0
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')

poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)

while True:
	socks = dict(poller.poll())
	if socks.get(in_sock) == zmq.POLLIN:
		m_raw = in_sock.recv()
		req = tnetstring.loads(m_raw[1:])
		print 'IN %s' % req

		id = req['id']
		if id in sessions:
			print 'session already active'
			continue

		if 'type' in req:
			print 'wrong packet type'
			continue

		s = Session()
		sessions[id] = s

		s.file = open(filename, 'r')
Exemplo n.º 38
0
Arquivo: post.py Projeto: namezis/zurl
        'method': 'POST',
        'uri': sys.argv[1],
        'headers': [['Content-Length', str(len(file))]],
        'stream': True,
        'credits': 200000,
        'more': True
    }))
outseq += 1

raddr = None

while True:
    buf = in_sock.recv()
    at = buf.find(' ')
    receiver = buf[:at]
    data = tnetstring.loads(buf[at + 2:])
    print 'IN: %s %s' % (receiver, data)
    if ('type' in data
            and data['type'] == 'error') or ('type' not in data
                                             and 'more' not in data):
        break
    if 'type' in data and data['type'] == 'keep-alive':
        odata = {
            'id': rid,
            'from': client_id,
            'seq': outseq,
            'type': 'keep-alive'
        }
        print 'OUT: %s' % odata
        out_stream_sock.send_multipart(
            [raddr, '', 'T' + tnetstring.dumps(odata)])
Exemplo n.º 39
0
import zmq

if len(sys.argv) < 2:
	print 'usage: %s [url]' % sys.argv[0]
	sys.exit(1)

ctx = zmq.Context()
sock = ctx.socket(zmq.REQ)
sock.connect('ipc:///tmp/zurl-req')

req = dict()
req['id'] = str(uuid.uuid4())
req['method'] = 'GET'
req['uri'] = sys.argv[1]
#req['ignore-tls-errors'] = True
sock.send('T' + tnetstring.dumps(req))

resp = tnetstring.loads(sock.recv()[1:])
if 'type' in resp and resp['type'] == 'error':
	print 'error: %s' % resp['condition']
	sys.exit(1)

print 'code=%d reason=[%s]' % (resp['code'], resp['reason'])
for h in resp['headers']:
	print '%s: %s' % (h[0], h[1])

if 'body' in resp:
	print '\n%s' % resp['body']
else:
	print '\n'
Exemplo n.º 40
0
 def test_unicode_handling(self):
     with self.assertRaises(ValueError):
         tnetstring.dumps("hello")
     self.assertEqual(tnetstring.dumps("hello".encode()),b"5:hello,")
     self.assertEqual(type(tnetstring.loads(b"5:hello,")),bytes)
Exemplo n.º 41
0
def stats_worker(c):
	in_sock = zmq_context.socket(zmq.SUB)
	in_sock.setsockopt(zmq.SUBSCRIBE, 'stats ')
	in_sock.setsockopt(zmq.SUBSCRIBE, 'update ')
	in_sock.bind(update_spec)
	c.ready()

	out_sock = zmq_context.socket(zmq.PUB)
	out_sock.linger = 0
	out_sock.bind(stats_out_spec)

	received = 0
	rtimes = list()
	receive_min = 0
	receive_max = 0
	receive_avg = 0

	while True:
		m_raw = in_sock.recv()
		at = m_raw.find(' ')
		mtype = m_raw[:at]

		mreceived = None
		if mtype == 'stats':
			m = tnetstring.loads(m_raw[at + 1:])
			logger.debug('stats: %s' % m)
			mreceived = m['received']
			mlatency = m['latency']
		elif mtype == 'update':
			utype = m_raw[at + 1:]
			logger.debug('update: ' + utype)

		count = db.config_get_count()

		nodes = db.node_get_all()
		edge_up = 0
		client_up = 0
		for id, node in nodes.iteritems():
			if node['type'] == 'edge':
				if 'ping' in node:
					edge_up += 1
			elif node['type'] == 'client':
				if 'ping' in node:
					client_up += 1

		if count > 0:
			edge_need = ((count - 1) / edge_capacity) + 1
			client_need = ((count - 1) / client_capacity) + 1
		else:
			edge_need = 0
			client_need = 0

		ping_data = db.get_ping_data()

		stats = dict()

		stats['capacity'] = count

		stats['edge-up'] = edge_up
		stats['edge-total'] = edge_need
		stats['client-up'] = client_up
		stats['client-total'] = client_need

		if 'min' in ping_data:
			stats['ping-min'] = ping_data['min']
		else:
			stats['ping-min'] = 0
		if 'max' in ping_data:
			stats['ping-max'] = ping_data['max']
		else:
			stats['ping-max'] = 0
		if 'avg' in ping_data:
			stats['ping-avg'] = ping_data['avg']
		else:
			stats['ping-avg'] = 0

		origin_info = db.get_origin_info()

		if mreceived is not None:
			latency = mlatency
			if 'ping' in origin_info:
				latency += origin_info['ping'] / 2

			now = int(time.time() * 1000)

			had_received = received
			received = mreceived
			if mreceived > had_received:
				rtimes.append((now, mreceived - had_received, latency))
			else:
				rtimes = list()
				rtimes.append((now, mreceived, latency))

			# calculate delivery times
			start_time = db.send_get_start()

			rmin = None
			rmax = None
			rtotal = 0
			rcount = 0
			for rtime in rtimes:
				t = rtime[0] - start_time - rtime[2]
				if t < 0:
					t = 0 # should not happen
				if rmin is None or t < rmin:
					rmin = t
				if rmax is None or t > rmax:
					rmax = t
				rcount += rtime[1]
				rtotal += t * rtime[1]
			if rmin is not None:
				receive_min = rmin
			else:
				receive_min = 0
			if rmax is not None:
				receive_max = rmax
			else:
				receive_max = 0
			if rtotal > 0:
				receive_avg = rtotal / rcount
			else:
				receive_avg = 0

		stats['received'] = received
		stats['receive-min'] = receive_min
		stats['receive-max'] = receive_max
		stats['receive-avg'] = receive_avg

		curdata = db.get_stats_data()
		if curdata is not None:
			del curdata['id']
			if curdata == stats:
				# stats are the same as before. don't write/send
				continue

		stats['id'] = db.set_stats_data(stats)
		out_sock.send('stats ' + json.dumps(stats))
Exemplo n.º 42
0
 def test_roundtrip_format_examples(self):
     for data, expect in FORMAT_EXAMPLES.items():
         self.assertEqual(expect, tnetstring.loads(data))
         self.assertEqual(expect,
                          tnetstring.loads(tnetstring.dumps(expect)))
         self.assertEqual((expect, ""), tnetstring.pop(data))
Exemplo n.º 43
0
 def _read_pkt (self, _list, timeout = None):
     c = self._redis.blpop ([_list], timeout or 5)
     if None is c:
         return None
     return tnetstring.loads (c [1], 'iso-8859-15')
Exemplo n.º 44
0
# adapted from mongrel2's manual

import sys
import zmq
import tnetstring
from pprint import pprint

CTX = zmq.Context()
addr = sys.argv[1]

ctl = CTX.socket(zmq.REQ)

print 'CONNECTING'
ctl.connect(addr)

while True:
    cmd = raw_input('> ')
    # will only work with simple commands that have no arguments
    ctl.send(tnetstring.dumps([cmd, {}]))

    resp = ctl.recv()

    pprint(tnetstring.loads(resp))

ctl.close()
Exemplo n.º 45
0
def thrash_tnetstring():
    for obj, tns, json in TESTS:
        assert tnetstring.loads(tns) == obj
        assert tnetstring.loads(tnetstring.dumps(obj)) == obj
Exemplo n.º 46
0
in_sock = ctx.socket(zmq.SUB)
in_sock.setsockopt(zmq.SUBSCRIBE, client_id)
in_sock.connect('ipc:///tmp/zurl-out')

time.sleep(0.5)

rid = str(uuid.uuid4())
inseq = 0
outseq = 0
out_sock.send('T' + tnetstring.dumps({'from': client_id, 'id': rid, 'seq': outseq, 'method': 'GET', 'uri': sys.argv[1], 'stream': True, 'credits': 230000}))
outseq += 1

while True:
	buf = in_sock.recv()
	at = buf.find(' ')
	receiver = buf[:at]
	indata = tnetstring.loads(buf[at + 2:])
	if indata['id'] != rid:
		continue
	print 'IN: %s' % indata
	assert(indata['seq'] == inseq)
	inseq += 1
	if ('type' in indata and (indata['type'] == 'error' or indata['type'] == 'cancel')) or ('type' not in indata and 'more' not in indata):
		break
	raddr = indata['from']
	if 'body' in indata and len(indata['body']) > 0:
		outdata = {'id': rid, 'from': client_id, 'seq': outseq, 'type': 'credit', 'credits': len(indata['body'])}
		print 'OUT: %s' % outdata
		out_stream_sock.send_multipart([raddr, '', 'T' + tnetstring.dumps(outdata)])
		outseq += 1
Exemplo n.º 47
0
 def test_unicode_handling(self):
     with self.assertRaises(ValueError):
         tnetstring.dumps("hello")
     self.assertEqual(tnetstring.dumps("hello".encode()), b"5:hello,")
     self.assertEqual(type(tnetstring.loads(b"5:hello,")), bytes)
Exemplo n.º 48
0
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')

poller = zmq.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)

while True:
    socks = dict(poller.poll())
    if socks.get(in_sock) == zmq.POLLIN:
        m_raw = in_sock.recv()
        req = tnetstring.loads(m_raw[1:])
        print 'IN %s' % req

        id = req['id']
        if id in sessions:
            print 'session already active'
            continue

        s = Session()
        s.id = id
        headers = req.get('headers')
        if headers:
            for h in headers:
                if h[0].lower() == 'content-type':
                    s.content_type = h[1]
                    break
Exemplo n.º 49
0
import sys
import tnetstring
import zmq

ctx = zmq.Context()
sock = ctx.socket(zmq.SUB)
sock.connect(sys.argv[1])
sock.setsockopt(zmq.SUBSCRIBE, '')

while True:
	m_raw = sock.recv()
	at = m_raw.find(' ')
	mtype = m_raw[:at]
	mdata = m_raw[at + 1:]
	if mdata[0] == 'T':
		m = tnetstring.loads(mdata[1:])
	else:
		m = tnetstring.loads(mdata)
	print '%s %s' % (mtype, m)
Exemplo n.º 50
0
 def test_roundtrip_format_examples(self):
     for data, expect in FORMAT_EXAMPLES.items():
         self.assertEqual(expect,tnetstring.loads(data))
         self.assertEqual(expect,tnetstring.loads(tnetstring.dumps(expect)))
         self.assertEqual((expect,""),tnetstring.pop(data))
Exemplo n.º 51
0
    if s is None:
        raise ValueError('unknown sid')
    for k, v in last_ids.iteritems():
        s.last_ids[k] = v


def session_get_last_ids(sid):
    s = sessions.get(sid)
    if s is not None:
        return s.last_ids
    else:
        return None


while True:
    req = tnetstring.loads(sock.recv())
    method = req['method']
    args = req['args']
    print 'IN %s %s' % (method, args)
    try:
        resp = None
        ret = None
        if method == 'session-detect-rules-set':
            rule_data_list = args['rules']
            rlist = list()
            for rule_data in rule_data_list:
                r = Rule()
                r.domain = rule_data['domain']
                r.path_prefix = rule_data['path-prefix']
                r.sid_ptr = rule_data['sid-ptr']
                r.json_param = rule_data.get('json-param')
Exemplo n.º 52
0
in_sock.connect('ipc:///tmp/zhttp-test-out')
in_stream_sock = ctx.socket(zmq.DEALER)
in_stream_sock.setsockopt(zmq.IDENTITY, client_id)
in_stream_sock.connect('ipc:///tmp/zhttp-test-out-stream')
out_sock = ctx.socket(zmq.PUB)
out_sock.connect('ipc:///tmp/zhttp-test-in')

poller = zmq.core.poll.Poller()
poller.register(in_sock, zmq.POLLIN)
poller.register(in_stream_sock, zmq.POLLIN)

while True:
	socks = dict(poller.poll())
	if socks.get(in_sock) == zmq.POLLIN:
		m_raw = in_sock.recv()
		req = tnetstring.loads(m_raw)
		print 'IN %s' % req

		id = req['id']
		if id in sessions:
			print 'session already active'
			continue

		if 'type' in req:
			print 'wrong packet type'
			continue

		s = Session()
		sessions[id] = s

		s.file = open(filename, 'r')
Exemplo n.º 53
0
 def test_roundtrip_format_random(self):
     for _ in xrange(500):
         v = get_random_object()
         self.assertEqual(v,tnetstring.loads(tnetstring.dumps(v)))
         self.assertEqual((v,""),tnetstring.pop(tnetstring.dumps(v)))