def predecessor_dead():
msg = Message(constants.message_type["survival"],
(address[0], tcp.predecessor_port))
msg_parser = MessageParser()
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, ('', tcp.predecessor_port))
pass
def store(request):
global db
data = request.data
data_id = sha1(data.split(',')[0].lstrip('(').strip("'")).hexdigest()
if successor is None:
with open(constants.DB_FILE, "w+") as db:
data = str(data) + '\n'
print data
db.write(data)
db.close()
print "Data stored"
else:
print "here"
successor_id = sha1(str(successor[1])).hexdigest()
msg_parser = MessageParser()
if id < data_id < successor_id or successor == (root_address[0],
root_address[1] + 2):
# store on self node
print "Storing ", id, data_id, successor_id
with open(constants.DB_FILE, "w+") as db:
data = str(data) + '\n'
db.write(data)
db.close()
print "Data stored"
else:
request.sender = address
request = msg_parser.pack_msg(request)
tcp.successor_socket.sendto(request, successor)
def run(self):
try:
while not self.stopped():
request, addr = tcp.socket.recvfrom(constants.BUFFER_SIZE)
msg_parser = MessageParser()
request = msg_parser.unpack_msg(request)
request_handler(request)
except Exception as e:
print str(e)
def run(self):
msg_parser = MessageParser()
while not self.stopped():
time.sleep(constants.SOCKET_TIMEOUT)
msg = Message(constants.message_type["heartbeat"],
(address[0], tcp.successor_port))
msg = msg_parser.pack_msg(msg)
tcp.successor_socket.sendto(msg, successor)
msg = Message(constants.message_type["heartbeat"],
(address[0], tcp.predecessor_port))
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, predecessor)
pass
def distribute_keys():
global db
msg_parser = MessageParser()
successor_id = sha1(str(successor[1])).hexdigest()
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
print key, key_id, successor_id
if key_id >= successor_id:
msg = Message(constants.message_type["save"], address, line)
msg = msg_parser.pack_msg(msg)
tcp.successor_socket.sendto(msg, successor)
db.close()
def run(self):
global predecessor_dead_flag
global successor_dead_flag
msg_parser = MessageParser()
timer_p = 0
timer_s = 0
while not self.stopped():
time.sleep(constants.SOCKET_TIMEOUT - 2)
msg, addr = tcp.predecessor_socket.recvfrom(constants.BUFFER_SIZE)
msg = msg_parser.unpack_msg(msg)
msg_parser.validate_message_type(msg)
if msg.type == constants.message_type["heartbeat"]:
timer_p = 0
pass
if timer_p >= 2:
predecessor_dead_flag = True
predecessor_dead()
msg, addr = tcp.successor_socket.recvfrom(constants.BUFFER_SIZE)
msg = msg_parser.unpack_msg(msg)
msg_parser.validate_message_type(msg)
if msg.type == constants.message_type["heartbeat"]:
timer_s = 0
pass
if timer_s >= 2:
successor_dead_flag = True
timer_p += 1
timer_s += 1
pass
def run(self):
print "Predecessor thread started"
while not self.stopped():
try:
print "Waiting for new predecessor at: ", tcp.predecessor_port
while True:
request, addr = tcp.predecessor_socket.recvfrom(
constants.BUFFER_SIZE)
print "recd from ", addr
msg_parser = MessageParser()
request = msg_parser.unpack_msg(request)
if request.type != constants.message_type["heartbeat"]:
request_handler(request)
except Exception as e:
print str(e)
def join_handler(request):
try:
global successor
global predecessor
msg_parser = MessageParser()
destination = request.destination
if successor is None and predecessor is None:
# send msg to request.sender their successor and predecessor
msg = Message(constants.message_type["set_predecessor"], address,
(address[0], tcp.successor_port), destination)
msg = msg_parser.pack_msg(msg)
tcp.socket.sendto(msg, destination)
msg = Message(constants.message_type["set_successor"], address,
(address[0], tcp.predecessor_port), destination)
msg = msg_parser.pack_msg(msg)
tcp.socket.sendto(msg, destination)
successor = (request.data[0], request.data[1] + 2)
predecessor = (request.data[0], request.data[1] + 1)
successor_thread.start()
predecessor_thread.start()
heartbeat = HeartBeatSendThread()
heartbeat.start()
heartbeat_listen = HeartBeatListenThread()
heartbeat_listen.start()
else:
peer_id = sha1(str(request.data[1])).hexdigest()
successor_id = sha1(str(successor[1])).hexdigest()
if id > successor_id:
if peer_id > successor_id and successor[0] != root_address[0]:
# send the request to successor
send_req_to_successor(request, msg_parser)
else:
# add the node in between self and successor
add_peer(request, msg_parser, destination)
else:
if peer_id < successor_id or successor[0] == root_address[0]:
# add node in between self and successor
add_peer(request, msg_parser, destination)
else:
# send the request to successor
send_req_to_successor(request, msg_parser)
pass
except Exception as e:
print "Join handler failed" + str(e)
def run(self):
while not self.stopped():
try:
# tcp.successor_socket.listen(constants.BACKLOG_FOR_TCP_SOCKET)
print "Waiting for new successor at: ", tcp.successor_port
# tcp.successor_connection, tcp.useless = tcp.successor_socket.accept()
# print "New successor connected: ", successor
while True:
request, addr = tcp.successor_socket.recvfrom(
constants.BUFFER_SIZE)
print "recd from ", addr
msg_parser = MessageParser()
request = msg_parser.unpack_msg(request)
if request.type != constants.message_type["heartbeat"]:
request_handler(request)
except Exception as e:
print str(e)
def main():
# Init colorama
colorama.init(autoreset=True)
# Load the messages
messages = []
messages += MessageParser.read('corpus/2013-09-16 tot 2014-01-24.messages')
messages += MessageParser.read('corpus/2014-01-26 tot 2014-02-12.messages')
messages += MessageParser.read('corpus/2014-02-07 tot 2014-03-03.messages')
messages += MessageParser.read('corpus/2014-03-03 tot 2014-05-03.messages')
messages += MessageParser.read('corpus/2014-05-03 tot 2014-09-20.messages')
messages += MessageParser.read('corpus/2014-09-20 tot 2016-06-13.messages')
messages += MessageParser.read('corpus/2016-06-13 tot 2018-01-23.messages')
# Analyze the messages
analyzer = Analyzer(messages)
analyzer.analyze()
# Use the prompt
prompt = Prompt(messages,analyzer)
prompt.prompt = '> '
prompt.cmdloop('\nDone loading! List available commands with "help" or detailed help with "help cmd".')
# De-init colorama
colorama.deinit()
def recursive_retrieve(request):
global db
data = request.data
data_id = sha1(data).hexdigest()
if successor is None:
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
if data_id == key_id:
db.close()
return line
db.close()
return -1
else:
successor_id = sha1(str(successor[1])).hexdigest()
msg_parser = MessageParser()
if id < data_id < successor_id or successor == (root_address[0],
root_address[1] + 2):
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
if data_id == key_id:
if root:
db.close()
return line
else:
msg = Message(constants.message_type["ack"],
address, line)
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, request.sender)
db.close()
if root:
return -1
else:
msg = Message(constants.message_type["not_found"])
msg = msg_parser.pack_msg(msg)
tcp.predocessor_socket.sendto(msg, request.sender)
else:
msg = Message(constants.message_type["recursive_retrieve"],
address, data)
msg = msg_parser.pack_msg(msg)
tcp.successor_socket.sendto(msg, successor)
response, addr = tcp.socket.recvfrom(constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
msg_parser.validate_message_type(response)
if root:
return response.data
else:
tcp.predecessor_socket.sendto(response, request.sender)
def join():
try:
msg = Message(constants.message_type["join"], address, address,
address)
msg_parser = MessageParser()
msg = msg_parser.pack_msg(msg)
tcp.socket.sendto(msg, root_address)
response, addr = tcp.socket.recvfrom(constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
request_handler(response)
response, addr = tcp.socket.recvfrom(constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
request_handler(response)
print "Node joined successfully at: ", successor, predecessor
successor_thread.start()
predecessor_thread.start()
msg = Message(constants.message_type["distribute_keys"],
(address[0], tcp.predecessor_port), str(address[1]))
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, predecessor)
heartbeat.start()
heartbeat_listen.start()
except Exception as e:
print str(e)
def test_message_parser(self):
# Tests in case of two messages
bytestring = b'\x00\x00\x00\x03\x05\xff\x80\x00\x00\x00\x01\x01'
m = MessageParser()
messages = m(bytestring)
message1 = next(messages)
message2 = next(messages)
print(message1)
self.assertEqual(message1.type, MessageType.BITFIELD)
self.assertEqual(message1.payload, b'\xff\x80')
self.assertEqual(message2.type, MessageType.UNCHOKE)
# Test in case of Keep Alive
ka = m(b'\x00\x00\x00\x00')
self.assertEqual(next(ka).type, MessageType.KEEP_ALIVE)
# Test in case of incomplete message and later fulfillment
message_half1 = b'\x00\x00\x00\x03'
message_half2 = b'\x05\xff\x80'
inc = MessageParser()
attempt1 = inc(message_half1)
answers = []
for m in attempt1:
answers.append(m)
self.assertFalse(answers, "Shouldn't be any answers.")
attempt2 = inc(message_half2)
for m in attempt2:
answers.append(m)
self.assertTrue(len(answers) == 1, "Should be one complete message.")
self.assertEqual(answers[0].type, MessageType.BITFIELD)
self.assertEqual(answers[0].payload, b'\xff\x80')
def connect_to_root(self):
self.client.tcp.start(self.client.address)
msg = Message(msg_type=constants.message_type["client_connect"],
destination=self.client.address)
msg_parser = MessageParser()
msg = msg_parser.pack_msg(msg)
self.client.tcp.socket.sendto(msg, self.root)
response, addr = self.client.tcp.socket.recvfrom(constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
msg_parser.validate_message_type(response)
print response.data
def iterative_retrieve(request):
global db
data = request.data
print data
data_id = sha1(data).hexdigest()
if successor is None:
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
if data_id == key_id:
db.close()
return line
db.close()
return -1
else:
successor_id = sha1(str(successor[1])).hexdigest()
msg_parser = MessageParser()
if id < data_id < successor_id or successor == (root_address[0],
root_address[1] + 2):
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
if data_id == key_id:
db.close()
return line
db.close()
return -1
else:
temp_successor = successor
print temp_successor
while temp_successor != (address[0], address[1] + 2):
msg = Message(constants.message_type["exist"], address, data)
msg = msg_parser.pack_msg(msg)
tcp.successor_socket.sendto(
msg, (temp_successor[0], temp_successor[1]))
response, addr = tcp.socket.recvfrom(constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
msg_parser.validate_message_type(response)
print constants.message_dictionary[response.type]
if response.type == constants.message_type["ack"]:
return response.data
pass
elif response.type == constants.message_type["nack"]:
temp_successor = response.data
elif response.type == constants.message_type["not_found"]:
print "Not Found"
return -1
pass
def __init__(self, peer_id, ip, port, torrent):
self.peer_id = peer_id
self.ip = ip
self.port = port
self.info_hash = torrent.info_hash
self.torrent = torrent
self.hands_shook = False
self.choked = False
self.is_choking = True
self.interested = False
self.is_interested = False
self.pieces = set()
self.messages_from_peer = MessageQueue()
self.messages_to_peer = MessageQueue()
self.working_on_piece = False
self._connection_thread = None
self._peer_listener_thread = None
self._client_listener_thread = None
self._message_parser = MessageParser()
def request_handler(request):
global predecessor
global successor
global db
msg_parser = MessageParser()
msg_parser.validate_message_type(request)
destination = request.destination
if request.type == constants.message_type["join"]:
join_handler(request)
elif request.type == constants.message_type["set_predecessor"]:
if request.destination == address:
print "setting predecessor to: ", request.data
predecessor = request.data
else:
request.sender = address
request = msg_parser.pack_msg(request)
tcp.predecessor_socket.sendto(request, predecessor)
elif request.type == constants.message_type["set_successor"]:
if request.destination == address:
print "setting successor to: ", request.data
successor = request.data
else:
request.sender = address
request = msg_parser.pack_msg(request)
tcp.predecessor_socket.sendto(request, predecessor)
elif request.type == constants.message_type["update_predecessor"]:
print "Updating predecessor: ", request.data
predecessor_thread.stop()
heartbeat_listen.stop()
heartbeat.stop()
predecessor = request.data
predecessor_thread._stop_event.clear()
heartbeat_listen._stop_event.clear()
heartbeat._stop_event.clear()
predecessor_thread.run()
heartbeat.run()
heartbeat_listen.run()
elif request.type == constants.message_type["store"]:
store(request)
elif request.type == constants.message_type["client_connect"]:
print "Client connecting"
msg = Message(constants.message_type["greeting"], address,
constants.GREETING_MESSAGE)
msg = msg_parser.pack_msg(msg)
tcp.socket.sendto(msg, destination)
elif request.type == constants.message_type["iterative_retrieve"]:
data = iterative_retrieve(request)
msg = Message(constants.message_type["iterative_retrieve"], address,
data, destination)
msg = msg_parser.pack_msg(msg)
tcp.socket.sendto(msg, destination)
pass
elif request.type == constants.message_type["recursive_retrieve"]:
data = recursive_retrieve(request)
msg = Message(constants.message_type["recursive_retrieve"], address,
data, destination)
msg = msg_parser.pack_msg(msg)
if root:
tcp.socket.sendto(msg, destination)
else:
tcp.predecessor_socket.sendto(msg, predecessor)
pass
elif request.type == constants.message_type["exist"]:
data_id = request.data
data_id = sha1(data_id).hexdigest()
successor_id = sha1(str(successor[1])).hexdigest()
if id < data_id < successor_id or (root_address[0],
root_address[1] + 2) == successor:
with open(constants.DB_FILE, "r") as db:
for line in db:
key = line.split(',')[0].lstrip('(').strip("'")
key_id = sha1(key).hexdigest()
print key, key_id, data_id
if data_id == key_id:
msg = Message(constants.message_type["ack"], address,
line)
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, request.sender)
db.close()
break
db.close()
msg = Message(constants.message_type["not_found"], address)
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, request.sender)
else:
msg = Message(constants.message_type["nack"], address, successor)
msg = msg_parser.pack_msg(msg)
tcp.predecessor_socket.sendto(msg, request.sender)
elif request.type == constants.message_type["distribute_keys"]:
distribute_keys()
elif request.type == constants.message_type["save"]:
with open(constants.DB_FILE, "w+") as db:
print "writing to db: ", request.data
db.write(str(request.data) + '\n')
db.close()
elif request.type == constants.message_type["survival"]:
if successor_dead_flag:
msg = Message(constants.message_type["update_predecessor"],
data=(address[0], tcp.successor_port))
msg = msg_parser.pack_msg(msg)
tcp.successor_socket.sendto(msg, request.sender)
heartbeat_listen.stop()
heartbeat.stop()
successor_thread.stop()
successor = request.sender
successor_thread._stop_event.clear()
successor_thread.run()
heartbeat.run()
heartbeat_listen.run()
def show_menu(self):
msg_parser = MessageParser()
while True:
print "Enter a command:\ns --> Store\ni --> Iterative Search\n" \
"r --> Recursive Search\ne --> exit\n"
command = raw_input()
if command == "s":
print "Enter key:"
key = raw_input()
print "Enter value:"
val = raw_input()
data = (key, val)
msg = Message(constants.message_type["store"],
self.client.address, str(data), self.root)
msg = msg_parser.pack_msg(msg)
self.client.tcp.socket.sendto(msg, self.root)
pass
elif command == "i":
print "You have selected Iterative retrieve. Enter the key:"
key = raw_input()
msg = Message(constants.message_type["iterative_retrieve"],
self.client.address, str(key),
self.client.address)
msg = msg_parser.pack_msg(msg)
self.client.tcp.socket.sendto(msg, self.root)
response, addr = self.client.tcp.socket.recvfrom(
constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
msg_parser.validate_message_type(response)
print response.data
pass
elif command == "r":
print "You have selected Iterative retrieve. Enter the key:"
key = raw_input()
msg = Message(constants.message_type["recursive_retrieve"],
self.client.address, str(key),
self.client.address)
msg = msg_parser.pack_msg(msg)
self.client.tcp.socket.sendto(msg, self.root)
response, addr = self.client.tcp.socket.recvfrom(
constants.BUFFER_SIZE)
response = msg_parser.unpack_msg(response)
msg_parser.validate_message_type(response)
print response.data
pass
elif command == "e":
sys.exit(0)
else:
print "Enter correct instruction"