def changeRole(groupName, targetPeerID, action): """ Change the role of a peer in a group (Master peers only) :param groupName: is the name of the group on which the change will be applied :param targetPeerID: is the peerID of the peer target of the change :param action: can be ADD_MASTER, CHANGE_MASTER, TO_RW, TO_RO :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "ROLE {} {} {}".format(action.upper(), targetPeerID, groupName) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: role not changed print('Received from the tracker :', answer) return False else: # tracker successfully changed role if action.upper() == "CHANGE_MASTER": # set the peer itself (former master) to RW groupsList[groupName]["role"] = "RW" return True
def getChunksList(file, peerAddr): """ Retrives the chunks list for a file from another active peer. :param file: File object :param peerAddr: IP address and port of the destination peer :return: chunksList (list() of integer, each one represents a chunkID) """ # connect to remote peer s = networking.createConnection(peerAddr) if s is None: return None try: # send request and get the answer message = str(peerCore.peerID) + " " + \ "CHUNKS_LIST {} {} {}".format(file.groupName, file.treePath, file.timestamp) networking.mySend(s, message) data = networking.myRecv(s) networking.closeConnection(s, peerCore.peerID) except (socket.timeout, RuntimeError, ValueError): print("Error while getting chunks list") networking.closeConnection(s, peerCore.peerID) return None if str(data).split()[0] == "ERROR": # remote peer answered with an error string print('Received from the peer :', data) return None else: # success: return chunksList chunksList = eval(str(data)) return chunksList
def disconnectGroup(groupName): """ Disconnect the peer from the group (the group becomes restorable). Stop eventual running sync threads. :param groupName: name of the group from which the peer want to disconnect :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "DISCONNECT {}".format(groupName) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return False print('Received from the tracker :', answer) return False else: # stop every synchronization thread working on file of the group # and remove group related tasks from the queue syncScheduler.stopSyncThreadsByGroup(groupName, syncScheduler.SYNC_STOPPED) syncScheduler.removeGroupTasks(groupName) groupsList[groupName]["status"] = "RESTORABLE" return True
def retrieveGroups(): """ Retrieves groups from the tracker and update local groups list. In case of error return immediately without updating local groups. :return: boolean (True for success, False for any error) """ global groupsList s = networking.createConnection(trackerZTAddr) if s is None: return try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "GROUPS" networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: group not restored print('Received from the tracker :', answer) return False else: # set the local groups list equals to the retrieved one # split operation in order to skip the initial 'OK -' groupsList = eval(answer.split(" ", 2)[2]) return True
def startPeer(): """ Load previous session information about files. Start a server thread on a free port. Finally send coordinates to central tracker in order to be reachable from other peers. :return: tracker object """ # load local file tree with previous session information (if any) global localFileTree localFileTree = fileSystem.getFileStatus(previousSessionFile) if localFileTree is None: return None # create and start the scheduler thread schedulerThread = Thread(target=syncScheduler.scheduler, args=()) schedulerThread.daemon = True schedulerThread.start() # join the ZeroTier virtual network zeroTierIP = networking.joinNetwork() # retrieve ZT IP address of the tracker getTrackerZTAddr() # create a server thread which will ask on all the IP addresses # including the ZeroTier IP address # port will be choose among available ones server = peerServer.Server() server.daemon = True server.start() # wait for serverStart in order to retrieve the choosen port number while not server.serverStart: pass # get peer server port number global myPortNumber myPortNumber = server.port s = networking.createConnection(trackerZTAddr) if s is None: return None # send my IP address and port number to tracker # in order to be reachable from other peers try: message = str(peerID) + " " + "HERE {} {}".format(zeroTierIP, myPortNumber) networking.mySend(s, message) __ = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return None return server
def peerExit(): """ Disconnect peer from all the active groups by sending a request to the tracker. Furthermore, stop all the working synchronization thread. :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "EXIT" networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return False print('Received from the tracker :', answer) return False else: # stop scheduler syncScheduler.stopScheduler() syncScheduler.removeAllTasks() # stop every working synchronization thread syncScheduler.stopAllSyncThreads(syncScheduler.SYNC_STOPPED) # leave ZetoTier network networking.leaveNetwork() # wait for eventual sync threads termination time.sleep(3) # save session status fileSystem.saveFileStatus(localFileTree, previousSessionFile) return True
def createGroup(groupName, groupTokenRW, groupTokenRO): """ Create a new group by sending a request to the tracker with all the necessary information :param groupName: name of the group :param groupTokenRW: token for Read&Write access :param groupTokenRO: token for ReadOnly access :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False # encrypt the two tokens using the md5 algorithm encryptedTokenRW = hashlib.md5(groupTokenRW.encode()) encryptedTokenRO = hashlib.md5(groupTokenRO.encode()) try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "CREATE {} {} {}".format(groupName, encryptedTokenRW.hexdigest(), encryptedTokenRO.hexdigest()) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: group not created print('Received from the tracker :', answer) return False else: # group successfully created: add it to my local groups list groupsList[groupName] = dict() groupsList[groupName]["name"] = groupName groupsList[groupName]["status"] = "ACTIVE" groupsList[groupName]["total"] = 1 groupsList[groupName]["active"] = 1 groupsList[groupName]["role"] = "MASTER" localFileTree.addGroup(fileSystem.Node(groupName, True)) return True
def run(self): """ Run the thread socketServer and manage the incoming communication. :return: void """ # print("[Thr {}] SocketServerThread starting with peer {}".format(self.number, self.clientAddr)) while not self.__stop: if self.clientSock: # Check if the client is still connected and if data is available: try: rdyRead, __, __ = select.select([ self.clientSock, ], [ self.clientSock, ], [], 5) except select.error: print('[Thr {}] Select() failed on socket with {}'.format( self.number, self.clientAddr)) self.stop() return if len(rdyRead) > 0: # read request readData = networking.myRecv(self.clientSock) # Check if socket has been closed if len(readData) == 0: # print('[Thr {}] {} closed the socket.'.format(self.number, self.clientAddr)) self.stop() else: # Strip newlines just for output clarity message = readData.rstrip() messageFields = message.split(' ', 1) peerID = messageFields[0] message = messageFields[1] self.manageRequest(message, peerID) else: print( "[Thr {}] No peer is connected, SocketServer can't receive data" .format(self.number)) self.stop() self.close()
def joinGroup(groupName, token): """ Join a group by sending a request to the tracker. :param groupName: is the name of the group that peer want to join :param token: it's the access token (password) :return: boolean (True for success, False for any error) (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False # encrypt the token using the md5 algorithm encryptedToken = hashlib.md5(token.encode()) try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "JOIN {} {}".format(groupName, encryptedToken.hexdigest()) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: group not joined print('Received from the tracker :', answer) return False else: # group successfully joined: set group status to ACTIVE groupsList[groupName]["status"] = "ACTIVE" if localFileTree.getGroup(groupName) is None: localFileTree.addGroup(fileSystem.Node(groupName, True)) # initialize file list for the group startGroupSync(groupName) return True
def startGroupSync(groupName): """ Starts eventual required synchronization in a specific group. First of all, it retrieves files information from the tracker. Information retrieved are compared to local information belonging to a previous session of the group (if any) in order to detect added/removed/updated files, reacting properly to these events :param groupName: selected group :return: void """ s = networking.createConnection(trackerZTAddr) if s is None: return # retrieves file information from the tracker try: message = str(peerID) + " " + "GET_FILES {}".format(groupName) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return immediately print("Received from the tracker: ", answer) return else: # split operation in order to skip the initial 'OK -' updatedFileList = eval(answer.split(" ", 2)[2]) # call the function that evaluates the information retrieved from the tracker # comparing them with local information about a previous session (if it exists) updateLocalGroupTree(groupName, localFileTree.getGroup(groupName), updatedFileList)
def retrievePeers(groupName, selectAll): """ Retrieve a list containg all the peers of a group. If selectAll = False retrieve only ACTIVE peers :param groupName: name of the group :param selectAll: boolean (True for success, False for any error) value :return: list of peers """ s = networking.createConnection(trackerZTAddr) if s is None: return None if selectAll: tmp = "ALL" else: tmp = "ACTIVE" try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "PEERS {} {} ".format(groupName, tmp) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return None if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return None print('Received from the tracker :', answer) peersList = None else: # split operation in order to skip the initial 'OK -' peersList = eval(answer.split(" ", 2)[2]) return peersList
def restoreGroup(groupName): """ Restore a group by sending a request to the tracker. In case of success update my local groups list setting the group status to ACTIVE. :param groupName: is the name of the group that I want to restore :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "RESTORE {}".format(groupName) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: group not restored print('Received from the tracker :', answer) return False else: # group successfully restored: set group status to ACTIVE groupsList[groupName]["status"] = "ACTIVE" if localFileTree.getGroup(groupName) is None: localFileTree.addGroup(fileSystem.Node(groupName, True)) # initialize file list for the group startGroupSync(groupName) return True
def getTrackerZTAddr(): """ Retrives the ZeroTier IP address from the tracker and set the associated global variable trackerZTAddr :return: void """ global trackerZTAddr s = networking.createConnection(trackerAddr) if s is None: return try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "INFO" networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False trackerZTAddr = eval(answer)
def updateFiles(groupName, files): """ Update a list of file in the synchronization group, making them ready to be acknowledged from other peers. :param groupName: name of the group from which files will be updated :param files: list of tuples (fileObject, timestamp) :return: void """ s = networking.createConnection(trackerZTAddr) if s is None: return False # collect information required for the update operation filesInfo = list() for f in files: file = f[0] fileInfo = dict() fileInfo["treePath"] = file.treePath fileInfo["filesize"] = file.filesize fileInfo["timestamp"] = file.timestamp filesInfo.append(fileInfo) try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "UPDATED_FILES {} {}".format(groupName, str(filesInfo)) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return False print('Received from the tracker :', answer) return False else: for f in files: file = f[0] timestamp = f[1] key = file.groupName + "_" + file.treePath # stop possible synchronization thread syncScheduler.stopSyncThread(key, syncScheduler.FILE_UPDATED) # make the peer ready to upload chunks if file.syncLock.acquire(blocking=False): # file not used by any sinchronization process file.status = "S" file.initSeed() file.syncLock.release() else: # file is currently in synchronization # create a thread which will wait under the end of the synchronization # and then it will update file state t = Thread(target=waitSyncAndUpdate, args=(file, timestamp)) t.daemon = True t.start() # retrieve the list of active peers for the file activePeers = retrievePeers(groupName, selectAll=False) # notify other active peers for peer in activePeers: s = networking.createConnection(peer["address"]) if s is None: continue try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "UPDATED_FILES {} {}".format(groupName, str(filesInfo)) networking.mySend(s, message) __ = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) continue return True
def removeFiles(groupName, treePaths): """ Remove a list of file from the synchronization group. :param groupName: name of the group from which files will be removed :param treePaths: list of treePaths of the files that will be removed :return: boolean (True for success, False for any error) """ s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "REMOVED_FILES {} {}".format(groupName, str(treePaths)) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return False print('Received from the tracker :', answer) return False else: # remove files from the local file list groupTree = localFileTree.getGroup(groupName) for treePath in treePaths: key = groupName + "_" + treePath if key in syncScheduler.syncThreads: groupTree.removeNode(treePath, False) syncScheduler.stopSyncThread(key, syncScheduler.FILE_REMOVED) else: groupTree.removeNode(treePath, True) # retrieve the list of active peers for the file activePeers = retrievePeers(groupName, selectAll=False) # notify other active peers for peer in activePeers: s = networking.createConnection(peer["address"]) if s is None: continue try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "REMOVED_FILES {} {}".format(groupName, str(treePaths)) networking.mySend(s, message) __ = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) continue return True
def addFiles(groupName, filepaths, directory): """ Add a list of files to a synchronization group. :param groupName: name of the group in which files will be added :param filepaths: filepaths list of the files that will be added :param directory: directory path (empty if the file doesn't belong a directory) :return: boolean (True for success, False for any error) """ # list of dictionary, where each dict contains all the info required # from the tracker to add a file in the group filesInfo = list() # WP stands for Without FilePath: it's a copy of filesInfo but without filepaths # because I don't need to send them to the tracker filesInfoWFP = list() if directory == "": for filepath in filepaths: fileInfo = dict() # split filepath into directoryPath and filename, saving only the latter __, fileInfo["treePath"] = os.path.split(filepath) fileInfo["filepath"] = filepath fileInfo["filesize"], fileInfo["timestamp"] = fileManagement.getFileStat(filepath) filesInfo.append(fileInfo) fileInfoWFP = fileInfo.copy() del fileInfoWFP["filepath"] filesInfoWFP.append(fileInfoWFP) else: # it's a directory for filepath in filepaths: fileInfo = dict() # remove dirPath from the filename # e.g. filepath: C://home/Desktop/file.txt # directory: C://home/Desktop # dirPath: C://home # filename: Desktop/file.txt dirPath, __ = os.path.split(directory) fileInfo["treePath"] = filepath.replace(dirPath, "")[1:] fileInfo["filepath"] = filepath fileInfo["filesize"], fileInfo["timestamp"] = fileManagement.getFileStat(filepath) filesInfo.append(fileInfo) fileInfoWFP = fileInfo.copy() del fileInfoWFP["filepath"] filesInfoWFP.append(fileInfoWFP) s = networking.createConnection(trackerZTAddr) if s is None: return False try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "ADDED_FILES {} {}".format(groupName, str(filesInfoWFP)) networking.mySend(s, message) answer = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) return False if answer.split(" ", 1)[0] == "ERROR": # tracker replied with an error message: return False print("Received from the tracker: ", answer) return False else: groupTree = localFileTree.getGroup(groupName) for fileInfo in filesInfo: # add file to the personal list of files of the peer filename = fileInfo["treePath"].split("/")[-1] treePath = fileInfo["treePath"] file = fileManagement.File(groupName=groupName, treePath=treePath, filename=filename, filepath=fileInfo["filepath"], filesize=fileInfo["filesize"], timestamp=fileInfo["timestamp"], status="S", previousChunks=list()) groupTree.addNode(treePath, file) file.initSeed() # retrieve the list of active peers for the file activePeers = retrievePeers(groupName, selectAll=False) # notify other active peers for peer in activePeers: s = networking.createConnection(peer["address"]) if s is None: continue try: # send request message and wait for the answer, then close the socket message = str(peerID) + " " + "ADDED_FILES {} {}".format(groupName, str(filesInfoWFP)) networking.mySend(s, message) __ = networking.myRecv(s) networking.closeConnection(s, peerID) except (socket.timeout, RuntimeError, ValueError): networking.closeConnection(s, peerID) continue return True
def getChunks(dl, file, peer, tmpDirPath): """ This function allows to retrieve chunks from another active peer. Chunks are selected from the dl.rarestFirstChunksList and requested. The function iterate its behavior until the download is complete. :param dl: download information and data :param file: File object :param peer: peer information, it's a dictionary :param tmpDirPath: path of the directory where chunk will be stored :return: void """ # get peer address peerAddr = peer["address"] if len(file.availableChunks) + len(dl.scheduledChunks) >= COMPLETION_RATE * file.chunksNumber \ or len(file.missingChunks) <= MAX_CHUNKS: # don't use random discard if the number of missing chunks is smaller than a certain amount threshold = 1 else: # use random discard threshold = INITIAL_TRESHOLD # connect to remote peer s = networking.createConnection(peerAddr) if s is None: return while not dl.complete: if file.stopSync: break # list of chunks that the function will retrieve in a single iteration chunksList = list() dl.lock.acquire() for chunk in dl.rarestFirstChunksList: if len(chunksList) >= MAX_CHUNKS: # chunksList full break if len(file.missingChunks) > MAX_CHUNKS \ and len(file.availableChunks) + len(dl.scheduledChunks) <= COMPLETION_RATE * file.chunksNumber \ and random() > threshold: # randomly discard this chunk from the request list continue if peer in dl.chunksToPeers[chunk]: # add chunk to request list and scheduled list chunksList.append(chunk) dl.scheduledChunks.add(chunk) # remove scheduled chunks from main list # this avoid that other threads request scheduled chunks for chunk in chunksList: dl.rarestFirstChunksList.remove(chunk) dl.lock.release() # decrease discard probability for next round threshold += TRESHOLD_INC_STEP if len(chunksList) == 0: # if no chunks can be request to the peer # sleep 5 seconds and then try again to compute chunksList for i in range(0, 5): if file.stopSync or dl.complete: break else: time.sleep(1) else: errors = 0 for i in range(0, len(chunksList)): chunkID = chunksList[i] if errors == MAX_ERRORS: networking.closeConnection(s, peerCore.peerID) # try to re-establish connection s = networking.createConnection(peerAddr) if s is None: # connection broken or peer disconnected for j in range(i, len(chunksList)): # reload in the main list all the scheduled chunks chunk = chunksList[j] errorOnGetChunk(dl, chunk) return # check eventual stop forced from main thread if file.stopSync: break try: # send request and wait for a string response message = str(peerCore.peerID) + " " + \ "CHUNK {} {} {} {}".format(file.groupName, file.treePath, file.timestamp, chunkID) networking.mySend(s, message) answer = networking.myRecv(s) except (socket.timeout, RuntimeError, ValueError): print("Error receiving message about chunk {}".format( chunkID)) errorOnGetChunk(dl, chunkID) errors += 1 continue if answer.split()[0] == "ERROR": # error: consider next chunks print("Received:", answer) errorOnGetChunk(dl, chunkID) continue try: # success: get the chunk # evaluate chunks size if chunkID == file.chunksNumber - 1: chunkSize = file.lastChunkSize else: chunkSize = CHUNK_SIZE data = networking.recvChunk(s, chunkSize) except (socket.timeout, RuntimeError): print("Error receiving chunk {}".format(chunkID)) errorOnGetChunk(dl, chunkID) errors += 1 continue try: peerCore.pathCreationLock.acquire() if not os.path.exists(tmpDirPath): # print("Creating the path: " + tmpDirPath) os.makedirs(tmpDirPath) peerCore.pathCreationLock.release() chunkPath = tmpDirPath + "chunk" + str(chunkID) # write chunk in a new file f = open(chunkPath, 'wb') f.write(data) f.close() try: file.missingChunks.remove(chunkID) file.availableChunks.append(chunkID) dl.scheduledChunks.remove(chunkID) except ValueError: pass except FileNotFoundError: errorOnGetChunk(dl, chunkID) continue networking.closeConnection(s, peerCore.peerID)