def _process_send(self, conn, data):
from_name = data[0]
to_name = data[1]
msg = data[2]
from_user = self.conn2user(conn)
error = ''
if from_user is None:
error = 'attempting to send a message before logging in, for message: from: %s | to: %s | body: %s' % (from_name, to_name, msg)
elif from_name != from_user.username:
error = 'specified from name is not the sender. Got username %s, current username is %s, for message: from: %s | to: %s | body: %s' % (from_name, from_user.username, from_name, to_name, msg)
elif to_name not in [u.username for u in self.users]:
error = 'specific recipient %s does not exist, for message: from: %s | to: %s | body: %s' % (to_name, from_name, to_name, msg)
if error:
conn.send_buffer += WireProtocol.data_to_bytes(CMD.RESPONSE, error)
return
to_user = [u for u in self.users if u.username == to_name][0]
# Send to recipient immediately if they are online
if to_user.connection is not None:
to_user.connection.send_buffer += WireProtocol.data_to_bytes(CMD.SEND, *data)
else:
# If they're not online, add to undelivered message list
to_user.undelivered_messages.append(data)
# In any event, send back to the sender for printing
from_user.connection.send_buffer += WireProtocol.data_to_bytes(CMD.SEND, *data)
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.connected_peers = set()
self._seen_peers = set() # (host, port, node_id)
self._stopped = False
self.local_address = () # host, port
self.lock = threading.Lock()
self.wire = WireProtocol(self, config)
def test_parse_bytes_multiarg():
wp = WireProtocol()
msg_bytes = b'\x00\x02' + b'\x00' * 39 + b'\x20' + b'toname|||fromname|||hello world!'
output = wp.parse_incoming_bytes(msg_bytes)
assert output == True
assert wp.version == 0
assert wp.command == 2
assert wp.data_len == 32
assert wp.data_buffer == b'toname|||fromname|||hello world!'
def test_msg_data_parse():
wp = WireProtocol()
msg_bytes = b'\x00\x02' + b'\x00' * 39 + b'\x20' + b'toname|||fromname|||hello world!'
wp.parse_incoming_bytes(msg_bytes)
data_out = wp.parse_data()
assert len(data_out) == 3
assert data_out[0] == 'toname'
assert data_out[1] == 'fromname'
assert data_out[2] == 'hello world!'
def test_parse_bytes_singlearg():
wp = WireProtocol()
msg_bytes = b'\x00\x00' + b'\x00' * 39 + b'\x08' + b'testname'
output = wp.parse_incoming_bytes(msg_bytes)
assert output == True
assert wp.version == 0
assert wp.command == 0
assert wp.data_len == 8
print(wp.data_buffer)
assert wp.data_buffer == b'testname'
def test_parse_bytes_multiarg_partial():
wp = WireProtocol()
msg_bytes = b'\x00\x02' + b'\x00' * 39 + b'\x20' + b'toname|||fro'
output = wp.parse_incoming_bytes(msg_bytes)
assert output == False
assert wp.version == 0
assert wp.command == 2
assert wp.data_len == 32
assert wp.data_buffer == b''
assert wp.tmp_buffer == b'toname|||fro'
def test_emptyedge():
wp = WireProtocol()
msg_bytes = b''
output = wp.parse_incoming_bytes(msg_bytes)
assert output == False
assert wp.version == -1
assert wp.command == -1
assert wp.data_len == -1
assert wp.data_buffer == b''
assert wp.tmp_buffer == b''
def test_send_notexists():
server = Server(host=None)
setup_server(server)
conn = server.connections[1]
server._process_send(
conn, [server.users[1].username, 'asdfasdfadf', 'test message'])
wp = WireProtocol()
wp.parse_incoming_bytes(conn.send_buffer)
assert wp.command == CMD.RESPONSE # indicates error
def test_login_alreadylogged():
server = Server(host=None)
setup_server(server)
conn = Connection('newconn', None)
server._process_login(conn, server.users[2].username)
wp = WireProtocol()
wp.parse_incoming_bytes(conn.send_buffer)
assert wp.command == CMD.RESPONSE
assert wp.data_len == len('user is already logged in')
assert wp.data_buffer == b'user is already logged in'
def test_send_notlogged():
server = Server(host=None)
setup_server(server)
conn = Connection('newconn', None)
server._process_send(
conn,
[server.users[1].username, server.users[2].username, 'test message'])
wp = WireProtocol()
wp.parse_incoming_bytes(conn.send_buffer)
assert wp.command == CMD.RESPONSE # indicates error
def test_login_noaccount():
server = Server(host=None)
setup_server(server)
server.users[2].logout()
conn = Connection('newconn', None)
server._process_login(conn, server.users[2].username + 'asdfasdf')
wp = WireProtocol()
wp.parse_incoming_bytes(conn.send_buffer)
assert wp.command == CMD.RESPONSE
assert wp.data_len == len('username does not exist')
assert wp.data_buffer == b'username does not exist'
def test_deliver_order():
server = Server(host=None)
setup_server(server)
server.users[2].logout()
# first send from u2 to u3
conn = server.connections[1]
server._process_send(
conn,
[server.users[1].username, server.users[2].username, 'test message!'])
server._process_send(conn, [
server.users[1].username, server.users[2].username,
'another test message'
])
# then login as u3 and get all the messages
conn = server.connections[2]
server.users[2].login(conn)
assert len(server.users[2].undelivered_messages) == 2
server._process_deliver(conn, None)
wp = WireProtocol()
wp.parse_incoming_bytes(conn.send_buffer)
assert wp.command == CMD.SEND
assert wp.data_buffer == b'u2|||u3|||test message!'
remaining_bytes = wp.tmp_buffer
wp.reset_buffers()
wp.parse_incoming_bytes(remaining_bytes)
assert wp.command == CMD.SEND
assert wp.data_buffer == b'u2|||u3|||another test message'
def test_data_to_bytes_multiarg():
command = CMD.SEND
to_str = 'toname' # 6 bytes
from_str = 'fromname' # 8 bytes
msg = 'hello world!' # 12 bytes
res = WireProtocol.data_to_bytes(command, to_str, from_str,
msg) # total len is 26 + 3*2 = 32
assert res == b'\x00\x02' + b'\x00' * 39 + b'\x20' + b'toname|||fromname|||hello world!'
def _process_deliver(self, conn, data):
print('conn uuid: %s' % str(conn.uuid))
for u in self.users:
print(u.username)
print(u.connection)
if u.connection:
print(u.connection.uuid)
from_user = self.conn2user(conn)
error = ''
if from_user is None:
error = 'attempting to deliver undelivered messages before logging in'
conn.send_buffer += WireProtocol.data_to_bytes(CMD.RESPONSE, error)
return
for msg in from_user.undelivered_messages:
conn.send_buffer += WireProtocol.data_to_bytes(CMD.SEND, *msg)
from_user.undelivered_messages = []
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.connected_peers = set()
self._seen_peers = set() # (host, port, node_id)
self._stopped = False
self.local_address = () # host, port
self.lock = threading.Lock()
self.wire = WireProtocol(self, config)
def _process_create(self, conn, data):
username = data
error = ''
if username in [u.username for u in self.users]:
error = 'username already exists'
elif CMD.DELIM.decode('ascii') in username:
error = 'illegal character sequence %s in username' % CMD.DELIM.decode('ascii')
else:
user = User(username)
user.login(conn)
self.users.append(user)
# send response with potential error
conn.send_buffer += WireProtocol.data_to_bytes(CMD.RESPONSE, error)
def _process_login(self, conn, data):
username = data
error = ''
if username not in [u.username for u in self.users]:
error = 'username does not exist'
else:
user = [u for u in self.users if u.username == username][0]
# check if user is already logged in
if user.connection is not None:
error = 'user is already logged in'
else:
user.login(conn)
print('user logged in, username: %s, conn %s' % (user.username, user.connection))
# send response with potential error
conn.send_buffer += WireProtocol.data_to_bytes(CMD.RESPONSE, error)
def listen_for_messages(q, socket, buffer_size):
# sets up
wp = WireProtocol()
while True:
# indicates that the socket has been closed from the main process
if socket.fileno() == -1:
break
data = socket.recv(buffer_size)
if not data:
q.put(
[None, None]
) # None indicates server cut connection (uncontrolled failure mode)
break
# if parse_incoming_bytes is True, an entire message has been received
while wp.parse_incoming_bytes(data):
# print('received full message, command: %d, args: %s' % (wp.command, str(wp.parse_data())))
q.put([wp.command, wp.parse_data()])
data = wp.tmp_buffer # Save any leftover bytes
wp.reset_buffers()
def test_data_to_bytes_singlearg():
command = CMD.CREATE
username = '******'
res = WireProtocol.data_to_bytes(command, username)
assert res == b'\x00\x00' + b'\x00' * 39 + b'\x08' + b'testname'
def __init__(self, uuid, socket):
self.uuid = uuid
self.wp = WireProtocol()
self.socket = socket
self.send_buffer = b''
def _process_list(self, conn, data):
# check if we need to filter by anything
names = [u.username for u in self.users]
if data is not None:
names = [n for n in names if fnmatch.fnmatch(n, data)]
conn.send_buffer += WireProtocol.data_to_bytes(CMD.LISTRESPONSE, *names)
class PeerManager(threading.Thread):
max_silence = 5 # how long before pinging a peer
max_ping_wait = 1. # how long to wait before disconenctiong after ping
max_ask_for_peers_elapsed = 30 # how long before asking for peers
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.connected_peers = set()
self._seen_peers = set() # (host, port, node_id)
self._stopped = False
self.local_address = () # host, port
self.lock = threading.Lock()
self.wire = WireProtocol(self, config)
def get_peer_by_id(self, peer_id):
for peer in self.connected_peers:
if peer_id == peer.id():
return peer
return None
def add_peer_address(self, ip, port, node_id):
ipn = (ip, port, node_id)
with self.lock:
if ipn not in self._seen_peers:
self._seen_peers.add(ipn)
def get_known_peer_addresses(self):
# fixme add self
return set(self._seen_peers).union(self.get_connected_peer_addresses())
def get_connected_peer_addresses(self):
"get peers, we connected and have a port"
return set((p.ip, p.port, p.node_id) for p in self.connected_peers
if p.port)
def stop(self):
with self.lock:
if not self._stopped:
for peer in self.connected_peers:
peer.stop()
self._stopped = True
def stopped(self):
with self.lock:
return self._stopped
def add_peer(self, peer):
with self.lock:
self.connected_peers.add(peer)
def remove_peer(self, peer):
peer.stop()
with self.lock:
self.connected_peers.remove(peer)
# connect new peers if there are no candidates
def connect_peer(self, host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.settimeout(1)
logger.debug('connecting {0}:{1}'.format(host, port))
try:
sock.connect((host, port))
except Exception as e:
logger.debug(
'Conencting {0}:{1} failed, {2}'.format(host, port, str(e)))
return False
sock.settimeout(.1)
ip, port = sock.getpeername()
logger.debug('connected {0}:{1}'.format(ip, port))
peer = Peer(self.wire, sock, ip, port)
self.add_peer(peer)
peer.start()
# Send Hello
peer.wire.send_Hello(peer)
peer.wire.send_GetPeers(peer)
return True
def _peer_candidates(self):
candidates = self.get_known_peer_addresses().difference(
self.get_connected_peer_addresses())
candidates = [
ipn for ipn in candidates if not ipn[:2] == self.local_address]
return candidates
def _poll_more_candidates(self):
for peer in list(self.connected_peers):
with peer.lock:
peer.wire.send_GetPeers(peer)
def _connect_peers(self):
num_peers = self.config.getint('network', 'num_peers')
candidates = self._peer_candidates()
if len(self.connected_peers) < num_peers:
logger.debug('not enough peers: {0}'.format(
len(self.connected_peers)))
logger.debug('num candidates: {0}'.format(len(candidates)))
if len(candidates):
ip, port, node_id = candidates.pop()
self.connect_peer(ip, port)
# don't use this node again in case of connect error > remove
self._seen_peers.remove((ip, port, node_id))
else:
self._poll_more_candidates()
def _check_alive(self, peer):
now = time.time()
dt_ping = now - peer.last_pinged
dt_seen = now - peer.last_valid_packet_received
# if ping was sent and not returned within last second
if dt_ping < dt_seen and dt_ping > self.max_ping_wait:
logger.debug(
'{0} last ping: {1} last seen: {2}'
.format(peer, dt_ping, dt_seen))
logger.debug(
'{0} did not respond to ping, disconnecting {1}:{2}'
.format(peer, peer.ip, peer.port))
self.remove_peer(peer)
elif min(dt_seen, dt_ping) > self.max_silence:
# ping silent peer
logger.debug('pinging silent peer {0}'.format(peer))
with peer.lock:
peer.wire.send_Ping(peer)
peer.last_pinged = now
def manage_connections(self):
self._connect_peers()
for peer in list(self.connected_peers):
if peer.stopped():
self.remove_peer(peer)
continue
self._check_alive(peer)
now = time.time()
# ask for peers
if now - peer.last_asked_for_peers >\
self.max_ask_for_peers_elapsed:
with peer.lock:
peer.wire.send_GetPeers(peer)
peer.last_asked_for_peers = now
def run(self):
while not self.stopped():
self.manage_connections()
self.wire.process_chainmanager_queue()
time.sleep(0.1)
def test_malformed():
wp = WireProtocol()
msg_bytes = b'this is very wrong!'
wp.parse_incoming_bytes(msg_bytes)
with pytest.raises(ValueError):
data_out = wp.parse_data()
def send_to_server(self, command, *args):
msg = WireProtocol.data_to_bytes(command, *args)
i = 0
while i < len(msg):
sent = self.socket.send(msg[i:])
i += sent
class PeerManager(threading.Thread):
max_silence = 5 # how long before pinging a peer
max_ping_wait = 1. # how long to wait before disconenctiong after ping
max_ask_for_peers_elapsed = 30 # how long before asking for peers
def __init__(self, config):
threading.Thread.__init__(self)
self.config = config
self.connected_peers = set()
self._seen_peers = set() # (host, port, node_id)
self._stopped = False
self.local_address = () # host, port
self.lock = threading.Lock()
self.wire = WireProtocol(self, config)
def get_peer_by_id(self, peer_id):
for peer in self.connected_peers:
if peer_id == peer.id():
return peer
return None
def add_peer_address(self, ip, port, node_id):
ipn = (ip, port, node_id)
with self.lock:
if ipn not in self._seen_peers:
self._seen_peers.add(ipn)
def get_known_peer_addresses(self):
# fixme add self
return set(self._seen_peers).union(self.get_connected_peer_addresses())
def get_connected_peer_addresses(self):
"get peers, we connected and have a port"
return set(
(p.ip, p.port, p.node_id) for p in self.connected_peers if p.port)
def stop(self):
with self.lock:
if not self._stopped:
for peer in self.connected_peers:
peer.stop()
self._stopped = True
def stopped(self):
with self.lock:
return self._stopped
def add_peer(self, peer):
with self.lock:
self.connected_peers.add(peer)
def remove_peer(self, peer):
peer.stop()
with self.lock:
self.connected_peers.remove(peer)
# connect new peers if there are no candidates
def connect_peer(self, host, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.settimeout(1)
logger.debug('connecting {0}:{1}'.format(host, port))
try:
sock.connect((host, port))
except Exception as e:
logger.debug('Conencting {0}:{1} failed, {2}'.format(
host, port, str(e)))
return False
sock.settimeout(.1)
ip, port = sock.getpeername()
logger.debug('connected {0}:{1}'.format(ip, port))
peer = Peer(self.wire, sock, ip, port)
self.add_peer(peer)
peer.start()
# Send Hello
peer.wire.send_Hello(peer)
peer.wire.send_GetPeers(peer)
return True
def _peer_candidates(self):
candidates = self.get_known_peer_addresses().difference(
self.get_connected_peer_addresses())
candidates = [
ipn for ipn in candidates if not ipn[:2] == self.local_address
]
return candidates
def _poll_more_candidates(self):
for peer in list(self.connected_peers):
with peer.lock:
peer.wire.send_GetPeers(peer)
def _connect_peers(self):
num_peers = self.config.getint('network', 'num_peers')
candidates = self._peer_candidates()
if len(self.connected_peers) < num_peers:
logger.debug('not enough peers: {0}'.format(
len(self.connected_peers)))
logger.debug('num candidates: {0}'.format(len(candidates)))
if len(candidates):
ip, port, node_id = candidates.pop()
self.connect_peer(ip, port)
# don't use this node again in case of connect error > remove
self._seen_peers.remove((ip, port, node_id))
else:
self._poll_more_candidates()
def _check_alive(self, peer):
now = time.time()
dt_ping = now - peer.last_pinged
dt_seen = now - peer.last_valid_packet_received
# if ping was sent and not returned within last second
if dt_ping < dt_seen and dt_ping > self.max_ping_wait:
logger.debug('{0} last ping: {1} last seen: {2}'.format(
peer, dt_ping, dt_seen))
logger.debug(
'{0} did not respond to ping, disconnecting {1}:{2}'.format(
peer, peer.ip, peer.port))
self.remove_peer(peer)
elif min(dt_seen, dt_ping) > self.max_silence:
# ping silent peer
logger.debug('pinging silent peer {0}'.format(peer))
with peer.lock:
peer.wire.send_Ping(peer)
peer.last_pinged = now
def manage_connections(self):
self._connect_peers()
for peer in list(self.connected_peers):
if peer.stopped():
self.remove_peer(peer)
continue
self._check_alive(peer)
now = time.time()
# ask for peers
if now - peer.last_asked_for_peers >\
self.max_ask_for_peers_elapsed:
with peer.lock:
peer.wire.send_GetPeers(peer)
peer.last_asked_for_peers = now
def run(self):
while not self.stopped():
self.manage_connections()
self.wire.send_chain_out_cmd()
time.sleep(0.1)