def upload_completed(file_name, chunk_id, is_success):
global my_ip, my_port
request = our_proto.UploadCompleteFileInfo()
request.fileName = file_name
request.chunkUploadInfo.chunkId = chunk_id
request.chunkUploadInfo.uploadedDatacenter.ip = my_ip
request.chunkUploadInfo.uploadedDatacenter.port = my_port
request.isSuccess = is_success
while True:
try:
random_raft = get_raft_node()
with grpc.insecure_channel(random_raft["ip"] + ':' +
random_raft["port"]) as channel:
stub = our_proto_rpc.RaftServiceStub(channel)
stub.FileUploadCompleted(request, timeout=GRPC_TIMEOUT)
print("Upload completed sent to raft ip :", random_raft["ip"],
",port :", random_raft["port"], ", success:", is_success)
break
except grpc.RpcError:
log_info("Could not sent upload complete to raft ip :",
random_raft["ip"], ",port :", random_raft["port"],
", success:", is_success)
time.sleep(0.1)
def send_request_to_other_raft_nodes(other_node, return_set):
try:
with grpc.insecure_channel(other_node["ip"] + ':' + other_node["port"]) as channel:
stub = file_transfer_proto_rpc.DataTransferServiceStub(channel)
files = stub.ListFiles(request, timeout=1)
for file_name in files.lstFileNames:
return_set.add(file_name)
except:
log_info("Fail to connect to: ", other_node["ip"], other_node["port"])
def _ask_for_vote():
log_info("Asking for vote...", Globals.CURRENT_CYCLE)
candidacy = raft_proto.Candidacy()
candidacy.cycle_number = Globals.CURRENT_CYCLE
candidacy.port = Globals.MY_PORT
candidacy.ip = Globals.MY_IP
candidacy.log_length = len(Tables.FILE_LOGS)
for client in Globals.LST_RAFT_CLIENTS:
client._RequestVote(candidacy)
def _process_heartbeat(client, table, heartbeat_counter, call_future):
log_info("_process_heartbeat:", client.server_port)
with ThreadPoolExecutorStackTraced(max_workers=10) as executor:
try:
call_future.result()
Globals.RAFT_HEARTBEAT_ACK_DICT[(client.server_address, client.server_port)] = (heartbeat_counter, True)
except:
Globals.RAFT_HEARTBEAT_ACK_DICT[(client.server_address, client.server_port)] = (heartbeat_counter, False)
log_info("Raft node not available!!", client.server_address, client.server_port)
return
def run(raft_ip, raft_port):
with grpc.insecure_channel(raft_ip + ':' + raft_port) as channel:
stub = file_transfer_rpc.DataTransferServiceStub(channel)
request = file_transfer.RequestFileList()
request.isClient = True
response = stub.ListFiles(request)
log_info("Got list of files: ")
pprint.pprint(response)
def AddFileLog(self, request, context):
if Globals.NODE_STATE == NodeState.LEADER:
request.log_index = Globals.get_next_log_index()
Tables.FILE_LOGS.append(request)
log_info("LOG ADDED")
# Update Table_log and File_info_table
Tables.set_table_log(Tables.FILE_LOGS)
ack = raft_proto.Ack()
ack.id = 1
return ack
def start_server(username, my_port):
# create a gRPC server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
server_object = ChatServer(username)
raft_proto_rpc.add_RaftServiceServicer_to_server(server_object, server)
file_transfer_proto_rpc.add_DataTransferServiceServicer_to_server(server_object, server)
log_info('Starting server. Listening...', my_port)
server.add_insecure_port('[::]:' + str(my_port))
server.start()
# Server starts in background (another thread) so keep waiting
while True:
time.sleep(64 * 64 * 100)
def _random_timeout():
log_info("_random_timeout: ", Globals.NODE_STATE, Globals.CURRENT_CYCLE)
if Globals.NODE_STATE == NodeState.FOLLOWER:
log_info("Standing for Election: ", Globals.MY_PORT)
Globals.NODE_STATE = NodeState.CANDIDATE
_increment_cycle_and_reset()
_ask_for_vote()
elif Globals.NODE_STATE == NodeState.LEADER:
pass
elif Globals.NODE_STATE == NodeState.CANDIDATE:
_increment_cycle_and_reset()
_ask_for_vote()
random_timer.reset()
def file_upload_iterator(file_path, file_name, chunk_num):
seq_max = file_utils.get_max_file_seqs_per_chunk(file_path, chunk_num)
cur_seq_num = 0
for chunk_buffer in file_utils.get_file_seqs_per_chunk(
file_path, chunk_num):
request = file_transfer.FileUploadData()
request.fileName = file_name
request.chunkId = chunk_num
request.seqMax = seq_max
request.seqNum = cur_seq_num
request.data = chunk_buffer
cur_seq_num += 1
log_info("Sending... Chunk: ", chunk_num, ", Seq: ", cur_seq_num)
yield request
def download_chunk(file_name, chunk_num, start_seq_num, proxy_address, proxy_port, downloads_folder="Downloads"):
log_info("requesting for :", file_name, "chunk no :", chunk_num, "from", proxy_address, ":", proxy_port)
global next_sequence_to_download
global maximum_number_of_sequences
chunk_data = bytes()
with grpc.insecure_channel(proxy_address + ':' + proxy_port) as channel:
stub = rpc.DataTransferServiceStub(channel)
request = file_transfer.ChunkInfo()
request.fileName = file_name
request.chunkId = chunk_num
request.startSeqNum = start_seq_num
try:
for response in stub.DownloadChunk(request):
log_info("Response received: Chunk", response.chunkId, "Sequence:", response.seqNum, "/",
response.seqMax)
next_sequence_to_download[chunk_num] = response.seqNum + 1
maximum_number_of_sequences[chunk_num] = response.seqMax
chunk_data += response.data
except grpc.RpcError:
log_info("Failed to connect to data center..Retrying !!")
write_file_chunks(response, os.path.join(os.path.dirname(os.path.realpath(__file__)), downloads_folder), chunk_data)
log_info("request completed for :", file_name, "chunk no :", chunk_num, "from", proxy_address, ":", proxy_port,
"last seq :", next_sequence_to_download[chunk_num], "max seq :",
maximum_number_of_sequences[chunk_num])
def ReplicationInitiate(self, request, context):
from_datacenter_ip = request.fromDatacenter.ip
from_datacenter_port = request.fromDatacenter.port
chunk = request.chunkId
filename = request.fileName
global FOLDER
log_info("Initiating replication of :", filename, "chunk :", chunk,
"from ip:", from_datacenter_ip, ",port :",
from_datacenter_port)
threading.Thread(target=start_download_as_client,
args=(filename, chunk, FOLDER, from_datacenter_ip,
from_datacenter_port)).start()
reply = our_proto.Ack()
reply.id = 1
return reply
def DownloadChunk(self, request, context):
file_name = request.fileName
chunk_id = request.chunkId
start_seq_num = request.startSeqNum
print("Download request received for", file_name, "chunk", chunk_id,
"seq", start_seq_num)
chunk_path = os.path.join(FOLDER, file_name, str(chunk_id))
current_seq = 0
if os.path.isfile(chunk_path):
total_seq = get_max_file_seqs(chunk_path)
with open(chunk_path, 'rb') as f:
while True:
piece = f.read(SEQUENCE_SIZE)
if not piece:
break
if current_seq >= start_seq_num:
reply = common_proto.FileMetaData()
reply.fileName = file_name
reply.chunkId = chunk_id
reply.data = piece
reply.seqNum = current_seq
reply.seqMax = total_seq
log_info("Sent...", file_name, "chunk", chunk_id,
"seq", current_seq)
current_seq += 1
# time.sleep(1)
yield reply
else:
current_seq += 1
else:
reply = common_proto.FileMetaData()
reply.fileName = file_name
reply.chunkId = chunk_id
reply.data = str.encode("")
reply.seqNum = 0
reply.seqMax = 0
log_info("Could not find", file_name, "chunk", chunk_id, "seq",
start_seq_num)
return reply
print("Download request completed for", file_name, "chunk", chunk_id,
"seq", start_seq_num)
def _process_request_for_vote(client, Candidacy, call_future):
with ThreadPoolExecutorStackTraced(max_workers=10) as executor:
try:
candidacy_response = call_future.result()
except:
log_info("Exception Error !!", client.server_port)
return
if candidacy_response.voted == raft_proto.YES and candidacy_response.cycle_number == Globals.CURRENT_CYCLE:
Globals.NUMBER_OF_VOTES += 1
log_info("Got Vote:", Globals.NUMBER_OF_VOTES)
if Globals.NUMBER_OF_VOTES / MAX_RAFT_NODES > 0.5 and Globals.NODE_STATE == NodeState.CANDIDATE:
Globals.NODE_STATE = NodeState.LEADER
Globals.LEADER_PORT = Globals.MY_PORT
Globals.LEADER_IP = Globals.MY_IP
_send_heartbeat()
random_timer.reset()
def RequestFileUpload(self, request, context):
if Globals.NODE_STATE == NodeState.LEADER:
my_reply = file_transfer_proto.ProxyList()
file_name = request.fileName
file_size = request.fileSize
total_chunks = math.ceil(file_size / CHUNK_SIZE)
if Tables.is_file_exists(file_name):
return my_reply
dcs = Tables.get_all_available_dc()
if not _send_heartbeat_to_check_majority_consensus():
return my_reply
for chunk_id in range(total_chunks):
random_dcs = [get_rand_hashing_node(dcs, file_name, chunk_id)]
Tables.insert_file_chunk_info_to_file_log(file_name, chunk_id, random_dcs, raft_proto.UploadRequested)
_send_heartbeat()
# pprint.pprint("TABLE_FILE_INFO")
# pprint.pprint(Tables.TABLE_FILE_INFO)
# pprint.pprint(Tables.TABLE_DC_INFO)
lst_proxies = Tables.get_all_available_proxies()
lst_proxy_info = []
for ip, port in lst_proxies:
proxy_info = file_transfer_proto.ProxyInfo()
proxy_info.ip = ip
proxy_info.port = port
lst_proxy_info.append(proxy_info)
log_info("LST_PROXIES:")
log_info(my_reply.lstProxy)
my_reply.lstProxy.extend(lst_proxy_info)
log_info("Replied to :")
# pprint.pprint(request)
# pprint.pprint(my_reply)
log_info("############################")
return my_reply
else:
client = get_leader_client()
if client:
my_reply = client._RequestFileUpload(request)
return my_reply
else:
return file_transfer_proto.ProxyList()
def FileUploadCompleted(self, request, context):
log_info(
"################################### FILE_UPLOAD_COMPLETED_ARRIVED!!! #################################")
# pprint.pprint(request)
log_info("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$")
if Globals.NODE_STATE == NodeState.LEADER:
chunk_id = request.chunkUploadInfo.chunkId
lst_dc = [(request.chunkUploadInfo.uploadedDatacenter.ip, request.chunkUploadInfo.uploadedDatacenter.port)]
if not _send_heartbeat_to_check_majority_consensus():
return
Tables.insert_file_chunk_info_to_file_log(request.fileName, chunk_id, lst_dc,
raft_proto.Uploaded if request.isSuccess else raft_proto.UploadFaied)
_send_heartbeat()
# pprint.pprint(Tables.TABLE_FILE_INFO)
log_info("###########################################################################")
else:
client = get_leader_client()
if client:
my_reply = client._FileUploadCompleted(request)
return my_reply
else:
return raft_proto.Empty()
return raft_proto.Empty()
def get_file_location(stub, request):
file_location_info = stub.RequestFileInfo(request)
log_info("Response received: ")
# pprint.pprint(file_location_info)
log_info(file_location_info.maxChunks)
log_info("is file found :", file_location_info.isFileFound)
return file_location_info
def GetFileLocation(self, request, context):
my_reply = file_transfer_proto.FileLocationInfo()
file_name = request.fileName
is_file_found = True
if file_name not in Tables.TABLE_FILE_INFO.keys():
# is_file_found = False
return my_reply
max_chunks = len(Tables.TABLE_FILE_INFO[file_name].keys())
log_info("max_chunks from raft ", max_chunks)
lst_proxies = Tables.get_all_available_proxies()
lst_proxy_info = []
for ip, port in lst_proxies:
proxy_info = file_transfer_proto.ProxyInfo()
proxy_info.ip = ip
proxy_info.port = port
lst_proxy_info.append(proxy_info)
my_reply.fileName = file_name
my_reply.maxChunks = max_chunks
my_reply.lstProxy.extend(lst_proxy_info)
my_reply.isFileFound = is_file_found
return my_reply
def UploadFile(self, request_iterator, context):
file_name = ""
chunk_id = None
seq_num = 0
seq_max = float('inf')
chunk_data = bytes()
for request in request_iterator:
file_name = request.fileName
chunk_id = request.chunkId
seq_num = request.seqNum
seq_max = request.seqMax
file_path = os.path.join(FOLDER, file_name)
log_info("Received... File:", file_name, "Chunk:", chunk_id,
", Seq: ", seq_num, "/", seq_max)
if seq_num == 0:
log_info("Upload request received for", file_name, "chunk",
chunk_id)
if os.path.isfile(os.path.join(file_path, str(chunk_id))):
os.remove(os.path.join(file_path, str(chunk_id)))
chunk_data += request.data
write_file_chunks(request, FOLDER, chunk_data)
my_reply = common_proto.FileInfo()
my_reply.fileName = file_name
if chunk_id is not None:
if seq_num == seq_max - 1:
# full chunk received
threading.Thread(target=upload_completed,
args=(file_name, chunk_id, True))
else:
# full chunk not received
threading.Thread(target=upload_completed,
args=(file_name, chunk_id, False))
return my_reply
def request_file_info_from_other_raft_nodes(request):
for node in other_raft_nodes:
try:
with grpc.insecure_channel(node["ip"] + ':' + node["port"]) as channel:
stub = file_transfer_proto_rpc.DataTransferServiceStub(channel)
file_location_info = stub.GetFileLocation(request)
log_info("Response received From other Raft: ")
# pprint.pprint(file_location_info)
log_info(file_location_info.maxChunks)
log_info("is file found in other Raft:", file_location_info.isFileFound)
if file_location_info.isFileFound:
return file_location_info
except:
log_info("Fail to connect to: ", node["ip"], node["port"])
file_location_info = file_transfer_proto.FileLocationInfo()
file_location_info.isFileFound = False
return file_location_info
def RaftHeartbeat(self, request, context):
log_info("heartbeat arrived: ", len(Tables.FILE_LOGS))
ack = raft_proto.Ack()
if len(request.tableLog) > len(Tables.FILE_LOGS):
Globals.NODE_STATE = NodeState.FOLLOWER
Globals.CURRENT_CYCLE = request.cycle_number
Globals.HAS_CURRENT_VOTED = False
Globals.NUMBER_OF_VOTES = 0
Globals.LEADER_PORT = request.leader_port
Globals.LEADER_IP = request.leader_ip
elif len(request.tableLog) == len(Tables.FILE_LOGS) and request.cycle_number > Globals.CURRENT_CYCLE:
Globals.NODE_STATE = NodeState.FOLLOWER
Globals.CURRENT_CYCLE = request.cycle_number
Globals.HAS_CURRENT_VOTED = False
Globals.NUMBER_OF_VOTES = 0
Globals.LEADER_PORT = request.leader_port
Globals.LEADER_IP = request.leader_ip
elif request.leader_ip != Globals.LEADER_IP or request.leader_port != Globals.LEADER_PORT:
ack.id = -1
return ack
random_timer.reset()
# print("MY Leader: ", Globals.LEADER_IP, Globals.LEADER_PORT, len(Tables.FILE_LOGS))
# Update Table_log and File_info_table
Tables.set_table_log(request.tableLog)
ack.id = len(Tables.FILE_LOGS)
# log_info("Tables.TABLE_FILE_INFO")
# pprint.pprint(Tables.TABLE_FILE_INFO)
log_info("Tables.TABLE_PROXY_INFO")
pprint.pprint(Tables.TABLE_PROXY_INFO)
log_info("Tables.TABLE_DC_INFO")
pprint.pprint(Tables.TABLE_DC_INFO)
log_info("###########################################################")
return ack
def _raft_heartbeat_timeout():
log_info("FILE_INFO_TABLE:")
# pprint.pprint(Tables.TABLE_FILE_INFO)
if Globals.NODE_STATE == NodeState.FOLLOWER:
pass
elif Globals.NODE_STATE == NodeState.LEADER:
log_info("_heartbeat_timeout: ", Globals.NODE_STATE, Globals.CURRENT_CYCLE)
# print("Leader !!")
_send_heartbeat()
elif Globals.NODE_STATE == NodeState.CANDIDATE:
log_info("_heartbeat_timeout: ", Globals.NODE_STATE, Globals.CURRENT_CYCLE)
_ask_for_vote()
_send_heartbeat()
raft_heartbeat_timer.reset()
