def replicate(self, dest):
f = self.file
print(f, dest)
port = KEEPERS_TO_KEEPERS_PORT
keeperSocket = zhelper.newSocket(zmq.REQ, dest["nodeIP"], (port, ))
uploader = Uploader()
uploader.upload(keeperSocket, f["userID"], f["fileName"])
def upload(self, socket, token = None, filePath = "client/vtest.mp4"):
ip, port = self.requestUpload(socket, token)
if (not ip):
return False
print("your file will be uploaded to ", ip + ":" + port)
uploadSocket = zhelper.newSocket(zmq.REQ, ip, (port,))
data = readVideo(filePath)
filename = filePath.split("/")[-1]
payload = {
"function": "upload",
"filename": filename,
"numChunks": len(data),
}
if token:
payload["token"] = token
uploadSocket.send_json(payload)
uploadSocket.recv()
print("uploading start")
with progressbar.ProgressBar(max_value=len(data)) as bar:
for i in range(len(data)):
#? may want to send the token here also, network overhead
uploadSocket.send(data[i])
uploadSocket.recv()
#update progress bar
bar.update(i)
def replicator_process(self):
listOfFiles = self.db.listAllFilesAllUsers()
listOfNodes = self.db.listAliveNodes()
print("listOfFiles", listOfFiles)
listOfNodes = sorted(listOfNodes,
key=lambda k: k["numFiles"]) #, reverse=True)
print("listOfNodes=", listOfNodes)
for file in listOfFiles:
print("handling file:", file)
# instances = (sum(value["fileName"] == file["fileName"] for value in filesLocations))
# TODO: we don't need this function: we will get the number of instances from getting the len(src)
instances = self.db.getInstancesOfFile(file["userID"],
file["fileName"])
src = next(
(item
for item in listOfNodes if item["nodeIP"] == file["nodeIP"]),
None)
# instances = len(src)
if not src:
print(
"OH MY GOD, this file can't be replicated as the whole src nodes are died"
)
continue
requiredDestNum = MIN_REPLICATION - instances
if (requiredDestNum <= 0):
continue
print("src = ", src,
f"should be transferred to {requiredDestNum} machines")
destinations = [
item for item in listOfNodes
if (item != src) and not self.db.isFileOnNode(
file["fileName"], item["nodeIP"], file["userID"])
]
availableNodesNum = len(destinations)
if (availableNodesNum < requiredDestNum):
print(
f"there is only {availableNodesNum} nodes, but {requiredDestNum} required"
)
selectedMachines = destinations[:min(availableNodesNum,
requiredDestNum)]
print("selectedMachines=", selectedMachines)
srcSocket = zhelper.newSocket(zmq.REQ, src["nodeIP"],
KEEPERS_PORTS)
srcSocket.send_json({"dests": selectedMachines, "file": file})
srcSocket.recv() # ack that replication has been done
def main(port):
#? TODO: when to use udp protocol
trackerSocket = zhelper.newSocket(zmq.REQ, TRACKER_IP,
TRACKER_PORTS_KEEPERS)
downloadUploadSocket = zhelper.newServerSocket(zmq.REP, getCurrMachineIp(),
port)
downloader = Downloader(downloadUploadSocket, trackerSocket, port)
uploader = Uploader(downloadUploadSocket, trackerSocket, port)
while True:
request = downloadUploadSocket.recv_json()
print("request received:", request)
username = authenticate(downloadUploadSocket, request.get("token"))
downloadUploadSocket.send_string("ACK")
if not username:
continue
del request["token"]
request["username"] = username
if request.get("function") == "download":
uploader.upload(request)
elif request.get("function") == "upload":
downloader.download(request)
def __init__(self):
self.socket = zhelper.newSocket(zmq.REQ, TRACKER_IP, TRACKER_PORTS)
self.downloader = Downloader()
self.uploader = Uploader()
self.fileExplorer = FileExplorer()
def __init__(self, port):
# self.trackerSocket = zhelper.newSocket()
self.mysocket = zhelper.newServerSocket(zmq.REP, "*", port)
trackerSocket = zhelper.newSocket(zmq.REQ, TRACKER_IP,
TRACKER_PORTS_KEEPERS)
self.downloader = Downloader(self.mysocket, trackerSocket, port)