def test_receive_middle_of_header(self):
"""
This test concatenates two messsages (as they're sent over the network),
and lets DelugeTransferProtocol receive the data in two parts.
The first part contains the first message, plus two bytes of the next message.
The next part contains the rest of the message.
This is a special case, as DelugeTransferProtocol can't start parsing
a message until it has at least 5 bytes (the size of the header) to be able
to read and parse the size of the payload.
"""
two_concatenated = base64.b64decode(
self.msg1_expected_compressed_base64
) + base64.b64decode(self.msg2_expected_compressed_base64)
first_len = len(base64.b64decode(self.msg1_expected_compressed_base64))
# Now found the entire first message, and half the header of the next message (2 bytes into the header)
self.transfer.dataReceived(two_concatenated[: first_len + 2])
# Should be 1 message in the list
self.assertEqual(1, len(self.transfer.get_messages_in()))
# Send the rest
self.transfer.dataReceived(two_concatenated[first_len + 2 :])
# Should be 2 messages in the list
self.assertEqual(2, len(self.transfer.get_messages_in()))
# Get the data as sent by DelugeTransferProtocol
message1 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message1))
message2 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg2), rencode.dumps(message2))
def Send(self, data,id):
self.refTime = time.time()
if self.pingAdded != 0:
self.sid += 1
th_send = threading.Thread(target=self.ThSend,name="Send th"+str(self.sid),args=(data,id))
th_send.start()
else:
self.sendto(dumps({"content":dumps(data),"id":id}),(self.host,self.port))
def Send(self, data,id):
#print "UDP data sent!:",data,"as",id
self.refTime = time.time()
if self.pingAdded != 0:
self.sid += 1
th_send = threading.Thread(target=self.ThSend,name="Send th"+str(self.sid),args=(data,id))
th_send.start()
else:
self.sendto(dumps({"content":dumps(data),"id":id}),(self.host,self.port))
def ThSend(self,data,id):
#print float(self.pingAdded)/1000.0
percentaje = float(self.pingAdded) * float(self.irregularPercentaje) / 100.0
min_percentaje = -percentaje
max_percentaje = +percentaje
ping_error = random.uniform(min_percentaje,max_percentaje)
final_ping = self.pingAdded + ping_error
time.sleep(float(final_ping)/1000.0)
self.sendto(dumps({"content":dumps(data),"id":id}),(self.host,self.port))
def test_receive_one_message(self):
"""
Receive one message and test that it has been sent to the
method 'message_received'.
"""
self.transfer.dataReceived(
base64.b64decode(self.msg1_expected_compressed_base64)
)
# Get the data as sent by DelugeTransferProtocol
messages = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(messages))
def send_request(self, request):
data = (request.format(), )
payload = zlib.compress(rencode.dumps(data))
self.conn.sendall(payload)
buf = b''
while True:
data = self.conn.recv(1024)
if not data:
self.connected = False
break
buf += data
dobj = zlib.decompressobj()
try:
message = rencode.loads(dobj.decompress(buf))
except (ValueError, zlib.error, struct.error):
# Probably incomplete data, read more
continue
else:
buf = dobj.unused_data
yield message
def test_receive_two_concatenated_messages(self):
"""
This test simply concatenates two messsages (as they're sent over the network),
and lets DelugeTransferProtocol receive the data as one string.
"""
two_concatenated = base64.b64decode(
self.msg1_expected_compressed_base64
) + base64.b64decode(self.msg2_expected_compressed_base64)
self.transfer.dataReceived(two_concatenated)
# Get the data as sent by DelugeTransferProtocol
message1 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message1))
message2 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg2), rencode.dumps(message2))
def Send(self , data , addr):
if self.pingAdded == 0:
self.sendto(dumps(data),addr)
else:
self.sid += 1
th_send = threading.Thread(target=self.ThSend,name="Send th"+str(self.sid),args=(data,addr))
th_send.start()
def Send(self, data):
"""Returns the number of bytes sent after enoding."""
data['PID'] = id.next()
outgoing = dumps(data) + self.endchars
self.sendqueue.append(outgoing)
return len(outgoing)
def upload(obj, notification_format):
data = crypt.encrypt_string_to_string(rencode.dumps(obj))
request = urllib2.urlopen(config.url, data=data)
response_string = crypt.decrypt_stream_to_string(request)
response = rencode.loads(response_string)
url = response["url"]
pyperclip.copy(url)
subprocess.Popen(["notify-send", "--expire-time=2000", notification_format % url])
def Send(self, data, addr):
if self.pingAdded == 0:
self.sendto(dumps(data), addr)
else:
self.sid += 1
th_send = threading.Thread(target=self.ThSend,
name="Send th" + str(self.sid),
args=(data, addr))
th_send.start()
def test_receive_old_message(self):
"""
Receive an old message (with no header) and verify that the data is discarded.
"""
self.transfer.dataReceived(rencode.dumps(self.msg1))
self.assertEqual(len(self.transfer.get_messages_in()), 0)
self.assertEqual(self.transfer._message_length, 0)
self.assertEqual(len(self.transfer._buffer), 0)
def upload(obj, notification_format):
data = crypt.encrypt_string_to_string(rencode.dumps(obj))
request = urllib2.urlopen(config.url, data=data)
response_string = crypt.decrypt_stream_to_string(request)
response = rencode.loads(response_string)
url = response["url"]
pyperclip.copy(url)
subprocess.Popen(
["notify-send", "--expire-time=2000", notification_format % url])
def sendData(self, data):
"""
Sends the data to the client.
:param data: the object that is to be sent to the client. This should
be one of the RPC message types.
:type data: object
"""
self.transport.write(zlib.compress(rencode.dumps(data)))
def sendData(self, data):
"""
Sends the data to the client.
:param data: the object that is to be sent to the client. This should
be one of the RPC message types.
:type data: object
"""
self.transport.write(zlib.compress(rencode.dumps(data)))
def step(self):
with self.cv:
while self.data is None and not self.done:
self.cv.wait()
if self.done:
return True
data = self.data
self.data = None
try:
with open(self.path, mode="wb") as f:
f.write(rencode.dumps(data))
except:
log.exception("While writing state")
def transfer_message(self, data):
"""
Transfer the data.
The data will be serialized and compressed before being sent.
First a header is sent - containing the length of the compressed payload
to come as a signed integer. After the header, the payload is transfered.
:param data: data to be transfered in a data structure serializable by rencode.
"""
compressed = zlib.compress(rencode.dumps(data))
size_data = len(compressed)
# Store length as a signed integer (using 4 bytes). "!" denotes network byte order.
header = b"D" + struct.pack("!i", size_data)
self.transport.write(header)
self.transport.write(compressed)
def transfer_message(self, data):
"""
Transfer the data.
The data will be serialized and compressed before being sent.
First a header is sent - containing the length of the compressed payload
to come as a signed integer. After the header, the payload is transfered.
:param data: data to be transfered in a data structure serializable by rencode.
"""
compressed = zlib.compress(rencode.dumps(data))
size_data = len(compressed)
# Store length as a signed integer (using 4 bytes). "!" denotes network byte order.
header = b"D" + struct.pack("!i", size_data)
self.transport.write(header)
self.transport.write(compressed)
def transfer_message(self, data):
"""
Transfer the data.
:param data: data to be transferred in a data structure serializable by rencode.
"""
body = zlib.compress(rencode.dumps(data))
body_len = len(body)
message = struct.pack(
'{}{}s'.format(MESSAGE_HEADER_FORMAT, body_len),
PROTOCOL_VERSION,
body_len,
body,
)
self._bytes_sent += len(message)
self.transport.write(message)
def send_request(self, request):
"""
Sends a RPCRequest to the server.
:param request: RPCRequest
"""
# Store the DelugeRPCRequest object just in case a RPCError is sent in
# response to this request. We use the extra information when printing
# out the error for debugging purposes.
self.__rpc_requests[request.request_id] = request
#log.debug("Sending RPCRequest %s: %s", request.request_id, request)
# Send the request in a tuple because multiple requests can be sent at once
data = zlib.compress(rencode.dumps((request.format_message(),)))
self.factory.bytes_sent += len(data)
self.transport.write(data)
def send_request(self, request):
"""
Sends a RPCRequest to the server.
:param request: RPCRequest
"""
# Store the DelugeRPCRequest object just in case a RPCError is sent in
# response to this request. We use the extra information when printing
# out the error for debugging purposes.
self.__rpc_requests[request.request_id] = request
#log.debug("Sending RPCRequest %s: %s", request.request_id, request)
# Send the request in a tuple because multiple requests can be sent at once
data = zlib.compress(rencode.dumps((request.format_message(), )))
self.factory.bytes_sent += len(data)
self.transport.write(data)
def do_POST(self):
data_str = crypt.decrypt_stream_to_string(self.rfile, limit=int(self.headers["Content-Length"])).strip()
data = dict(rencode.loads(data_str))
data["time"] = time.time()
while True:
item_id = config.generate_name()
if db.hlen(item_id) == 0:
# todo: stop ignoring this race condition
db.hmset(item_id, data)
break
self.send_response(200)
self.end_headers()
response = {
"item": item_id,
"url": config.url + "/" + item_id,
}
crypt.encrypt_string_to_stream(rencode.dumps(response), self.wfile)
def start():
print "Starting WiiMote driver daemon"
server = socketipc.IPCServer(port=31307)
while True:
connection = server.accept()
try:
while True:
msg = rencode.loads(connection.receive())
cmd = msg[0]
data = msg[1:]
print 'command = ', cmd
resp = respond(cmd, data)
print 'response = ', resp
connection.send(rencode.dumps(resp))
except RuntimeError, e:
# They closed the connection, keep going
pass
def test_receive_three_messages_in_parts(self):
"""
This test concatenates three messsages (as they're sent over the network),
and lets DelugeTransferProtocol receive the data in multiple parts.
"""
msg_bytes = (
base64.b64decode(self.msg1_expected_compressed_base64)
+ base64.b64decode(self.msg2_expected_compressed_base64)
+ base64.b64decode(self.msg1_expected_compressed_base64)
)
packet_size = 40
one_message_byte_count = len(
base64.b64decode(self.msg1_expected_compressed_base64)
)
two_messages_byte_count = one_message_byte_count + len(
base64.b64decode(self.msg2_expected_compressed_base64)
)
three_messages_byte_count = two_messages_byte_count + len(
base64.b64decode(self.msg1_expected_compressed_base64)
)
for d in self.receive_parts_helper(msg_bytes, packet_size):
bytes_received = self.transfer.get_bytes_recv()
if bytes_received >= three_messages_byte_count:
expected_msgs_received_count = 3
elif bytes_received >= two_messages_byte_count:
expected_msgs_received_count = 2
elif bytes_received >= one_message_byte_count:
expected_msgs_received_count = 1
else:
expected_msgs_received_count = 0
# Verify that the expected number of complete messages has arrived
self.assertEqual(
expected_msgs_received_count, len(self.transfer.get_messages_in())
)
# Get the data as received by DelugeTransferProtocol
message1 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message1))
message2 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg2), rencode.dumps(message2))
message3 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message3))
def send(self):
if self.debug:
logging.info(
"Sending %s to %s " % ( len(self.data), str(self.address) )
)
''' sending data to server '''
outgoing = dumps(self.data)
if self.debug:
logging.info(
"Outgoing data: %s " % str(outgoing)
)
self.socket.sendall(outgoing)
''' receiving data from server '''
self.received_data = self.socket.recv(self.BUFFER)
self.received_data = loads(self.received_data)
if not len(self.received_data):
if self.debug:
logging.error("No data received from server.")
return
if self.debug:
logging.info(
"Received data: %s " % str(self.received_data)
)
def call(self, method, *args, **kwargs):
self.request_id += 1
request = ((self.request_id, method, args, kwargs), )
comp_request = zlib.compress(rencode.dumps(request))
# send
self._socket.send(comp_request)
# recv
data_queue = deque()
while True:
try:
d = self._socket.recv(64)
data_queue.append(d)
except ssl.SSLError:
raise
try:
data = zlib.decompress(b''.join(data_queue))
except zlib.error:
if not d:
raise
continue
break
data = list(rencode.loads(data))
msg_type = data.pop(0)
request_id = data.pop(0)
print('request_id = {}'.format(request_id))
if msg_type == 2:
exception_type, exception_msg, traceback = data[0]
print(exception_type, exception_msg, traceback, sep=', ')
raise Exception
elif msg_type == 1:
return data[0]
def pack(data): '''packs data for socket ''' return dumps(data)
def encode(self, data, compress=False):
# encode(and possibly compress) the data with rencode
return rencode.dumps(data, compress)
def pack_data(self, packet):
return dumps(packet)
def do_rencode(v):
return dumps(v), FLAGS_RENCODE
def serialize(x):
return rencode.dumps(x)
def encode(self, data):
# encode the data with rencode
# return rencode.dumps(data)
return rencode.dumps(data)
def encode(self, data, compress=False):
# encode(and possibly compress) the data with rencode
return rencode.dumps(data, compress)
def serialize(x):
return rencode.dumps(x)
def Send(self, data): self.sendqueue.append(dumps(data) + self.endchars)
def ThSend(self, data, addr):
time.sleep(float(self.pingAdded) / 1000.0)
self.sendto(dumps(data), addr)
def Send(self, data):
"""Returns the number of bytes sent after encoding."""
outgoing = dumps(data) + self.endchars
self.sendqueue.append(outgoing)
return len(outgoing)
import rencode
if __name__ == '__main__':
#common
chars = NetPool()
#client
state = {
1: [2, 3],
2: {
'type': 1,
0: ('(empty)', [0.1, 0.2, 0.3]),
1: 'charname'
}
}
strstate = rencode.dumps(state)
print len(strstate)
print strstate
state = rencode.loads(strstate)
NetEnt.setState(state)
#server
#for i in range(10):
# chars.add(Character())
print NetEnt.entities
print NetEnt.types
x = NetEnt.getState()
print len(rencode.dumps(x))
print x
def _test_rencode_fail_protocol(self):
"""
This test tries to test the protocol that relies on errors from rencode.
"""
msg_bytes = (
base64.b64decode(self.msg1_expected_compressed_base64)
+ base64.b64decode(self.msg2_expected_compressed_base64)
+ base64.b64decode(self.msg1_expected_compressed_base64)
)
packet_size = 149
one_message_byte_count = len(
base64.b64decode(self.msg1_expected_compressed_base64)
)
two_messages_byte_count = one_message_byte_count + len(
base64.b64decode(self.msg2_expected_compressed_base64)
)
three_messages_byte_count = two_messages_byte_count + len(
base64.b64decode(self.msg1_expected_compressed_base64)
)
print()
print(
'Msg1 size:',
len(base64.b64decode(self.msg1_expected_compressed_base64)) - 4,
)
print(
'Msg2 size:',
len(base64.b64decode(self.msg2_expected_compressed_base64)) - 4,
)
print(
'Msg3 size:',
len(base64.b64decode(self.msg1_expected_compressed_base64)) - 4,
)
print('one_message_byte_count:', one_message_byte_count)
print('two_messages_byte_count:', two_messages_byte_count)
print('three_messages_byte_count:', three_messages_byte_count)
for d in self.receive_parts_helper(
msg_bytes, packet_size, self.transfer.data_received_old_protocol
):
bytes_received = self.transfer.get_bytes_recv()
if bytes_received >= three_messages_byte_count:
expected_msgs_received_count = 3
elif bytes_received >= two_messages_byte_count:
expected_msgs_received_count = 2
elif bytes_received >= one_message_byte_count:
expected_msgs_received_count = 1
else:
expected_msgs_received_count = 0
# Verify that the expected number of complete messages has arrived
if expected_msgs_received_count != len(self.transfer.get_messages_in()):
print(
'Expected number of messages received is %d, but %d have been received.'
% (
expected_msgs_received_count,
len(self.transfer.get_messages_in()),
)
)
# Get the data as received by DelugeTransferProtocol
message1 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message1))
message2 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg2), rencode.dumps(message2))
message3 = self.transfer.get_messages_in().pop(0)
self.assertEqual(rencode.dumps(self.msg1), rencode.dumps(message3))
def connectionMade(self):
self.transport.write(rencode.dumps(self.factory.args))
self.transport.loseConnection()
def Send(self, data):
"""Returns the number of bytes sent after enoding."""
outgoing = dumps(data) + self.endchars
self.sendqueue.append(outgoing)
return len(outgoing)
def ThSend(self,data,addr):
time.sleep(float(self.pingAdded)/1000.0)
self.sendto(dumps(data),addr)
int16 = 32767
int32 = 2147483647
int64 = 9223372036854775807
hugeInt = 922337203685477580700
float32 = 45.5
smallString = "wtf"
bigString = hs.compl("bigString", 100)
bigDict = {}
for i in range(0, 100):
bigDict[str(i)] = i
bigList = [13] * 100
nestedList = [bigList, 1, 2] + bigList
testCases = [12, -11, int8, int16, int32, int64, hugeInt,
float32, smallString, bigString, [1, 2, 3, True, False], bigList,
{'x': 1, 'y': 2}, bigDict, nestedList]
i = 0
for t in testCases:
t = rencode.dumps(t)
print t
print open('rencodeSamples/' + (str(i)),'w').write(t)
i += 1
def connectionMade(self):
self.transport.write(rencode.dumps(self.factory.args))
self.transport.loseConnection()
def send_messages(self, *messages):
assert self.transport
log().debug("sending: %s", messages)
data = zlib.compress(rencode.dumps(messages))
self.transport.write(data)
def _write_socket(socket, data):
encoded = rencode.dumps(data)
socket.sendall(struct.pack("I", len(encoded)))
socket.sendall(encoded)
def Send(self, data):
self.sendqueue.append(dumps(data) + self.endchars)
def encode_msg(self, msg):
return rencode.dumps(msg)
dataDict[0] = None
dataDict[1] = self.points
return dataDict
def setState(self, weightOld, dataOld, weightNew, dataDictNew):
self.char = NetEnt.entities[dataDictNew[0]]
self.points = dataDictNew[1]
NetEnt.registerSubclass(User)
import rencode
if __name__ == '__main__':
#common
chars = NetPool()
#client
state = {1:[2,3], 2:{'type':1, 0:('(empty)',[0.1,0.2,0.3]), 1:'charname'}}
strstate = rencode.dumps(state)
print len(strstate)
print strstate
state = rencode.loads(strstate)
NetEnt.setState(state)
#server
#for i in range(10):
# chars.add(Character())
print NetEnt.entities
print NetEnt.types
x = NetEnt.getState()
print len(rencode.dumps(x))
print x