def downloadPart(url: str, start: int, end: int, filename: str) -> (bool, str): """Returns statusCode Helps to download a segment of a data as filename. Uses tqdm to display progressbar """ headers = {'Range' : 'bytes=%d-%d' % (start, end)} try: r = requests.get(url, headers = headers, stream = True) r.raise_for_status() except requests.exceptions.HTTPError as err: logger.fatal(err) return (False, None) # sys.exit(1) logger.info("Initiating download of %s" % filename) fileSegmentSize = int(r.headers.get('Content-Length')) chunkSize = 1024 with open(os.path.join(HOME, filename), "wb") as fp: pbar = tqdm(unit = "B", total = fileSegmentSize) for chunk in r.iter_content(chunk_size = chunkSize): if chunk: # filter-out keep-alive new chunks; needs investigation pbar.update(len(chunk)) fp.write(chunk) logger.info("Download completed for %s" % filename) return (True, filename) """obsolete with open(filename, "r+b") as fp: fp.seek(start) currentPoint = fp.tell() fp.write(r.content) """ #return True pass
def checkAcceptRange(url: str) -> (bool, str): """Return a tuple, (bool, fileSize) Validates if partial download is possible over the url. Also returns the Content-Length, and Content-Type if true """ status = (False, None) try: r = requests.head(url) r.raise_for_status() except requests.exceptions.HTTPError as err: logger.error(err) # sys.exit(1) return status if r.headers.get('Accept-Ranges'): if r.headers.get('Accept-Ranges') == "bytes": logger.info("Supports partial download") # Parse the file size from the header fileSize = int(r.headers.get('Content-Length')) return (True, fileSize) else: # Handle unsupported partial download logger.fatal("Unsupported partial Download for file") else: logger.warn("File not suitable for download") return status
def start(cls): """ Start the File Server """ # python = python3 FIXME: Decide universally cls.fileServer = Popen(['python3', 'httpServer.py'], stdout=PIPE, stderr=PIPE) logger.info("Initiating local file server at %s" % (HOME))
def getPart(filename: str, partNumber: int, hostIP: str) -> (bool, str): hostPort = 11112 #NOTE: fixed port for all #None # ?? from config file partitionFilename = filename + ".part" + (str(partNumber).rjust(3, '0')) url = "http://" + ":".join([hostIP, str(hostPort) ]) + "/" + partitionFilename try: r = requests.get(url, stream=True) r.raise_for_status() except requests.exceptions.HTTPError as err: logger.fatal(err) return (False, None) logger.info("Getting part %s from user %s" % (partitionFilename, hostIP)) fileSegmentSize = int(r.headers.get('Content-Length')) chunkSize = 1024 with open(os.path.join(HOME, partitionFilename), "wb") as fp: pbar = tqdm(unit="B", total=fileSegmentSize) for chunk in r.iter_content(chunk_size=chunkSize): if chunk: # filter-out keep-alive chunks pbar.update(len(chunk)) fp.write(chunk) logger.info("Downloading complete for %s from %s" % (partitionFilename, hostIP)) return (True, partitionFilename)
def startDownload(url: str, user_count: int, user_id: int): logger.info("Initiating Splitfire") status, fileSize = checkAcceptRange(url) if status == True: success, filename = partitionManager(url, fileSize, user_count, user_id) return success, filename
def __del__(self): """Release all the users, and kill the room Currently not clearing the Room.activeUsers (just deleting object clears the air) """ for user in self.activeUsers: user.leaveRoom(self) logger.info("Successfully deleted room %s" % self.roomName)
def getActiveUsers(self): """Returns the list of active users in the Room""" if self.activeUserCount == 0: logger.info("Empty room %s" % self.roomName) return list() userList = list() for user in self.activeUsers: userList.append(user.userName) return userList
def checkDownloadDirectory(path: str=HOME) -> bool: """ Verify if the download path exist. Else create new directory""" if os.path.isdir(HOME): return True try: os.makedirs(HOME) logger.info("Created Dragonload directory at %s" % HOME) except Exception as err: logger.fatal("Unable to create download home directory: %s" % err) return False return True
def CreateRoom(self, request, context): room = decodeRoom(request) # Check conflicting names for _room in roomList: if _room.roomName == room.roomName: # Room already exist return dragonvault_pb2.Ack( status=False, msg="Another room with name '{}' already exist".format( room.roomName)) # Create a new room roomList.append(room) logger.info("Room Created- {} ".format(room.roomName)) return dragonvault_pb2.Ack(status=True, msg="Room created Successfully")
def leaveRoom(self, roomObject) -> bool: """ Leave the specified room. Arguments: roomObject: object of class Room Return: True <- Successfully exited Room roomObject False <- Unable to leave the Room roomObject """ if self.room == roomObject: self.room = None logger.info("User %s exited room %s" % (self.userName, roomObject.roomName)) return True logger.error("User %s unable to exit room %s" % (self.userName, roomObject.roomName)) return False
def terminateRoom(self): """Force quit the users from the room Return: (status <bool>, remaining users <int>) (True, 0) if no users in the room (True, count) if room is deleted, and count users are still left (False, count) if room is not deleted """ if self.activeUserCount == 0: logger.info('Successfully deleted room %s' % (self.roomName)) return True, self.activeUserCount # Handle active users for user in self.activeUsers: user.leaveRoom(self) self.activeUserCount -= 1 remainingUsers = self.activeUserCount del self # Delete the class return True, remainingUsers
def joinRoom(self, roomObject): """ Join the specified Room. Arguments: roomObject: object of class Room Return: None <- Unsuccessful roomObject<- Successfull """ if not isinstance(roomObject, Room): # Format string %s is error; FIXME later logger.error( "User %s cannot join Room: %s is not a valid Room object" % (self.userName, roomObject)) return None self.room = roomObject logger.info("User %s joined room %s" % (self.userName, roomObject.roomName)) return self.room
def mergePartitions(filename: str, total_partitions: int) -> (bool, str): CHUNK_SIZE = 1024 * 1024 # make sure that all the parts exists fileExistStatus = True for part in range(total_partitions): partitionFilename = os.path.join( HOME, filename) + ".part" + (str(part).rjust(3, '0')) fileExistStatus = fileExistStatus and checkFilesExist( partitionFilename) if not fileExistStatus: logger.critical( "Not all parts available in the directory: Initiating Chainrain again" ) return False, None """ Merger all the partitions together into one output file """ outputFile = open(os.path.join(HOME, filename), 'ab') for part in range(total_partitions): partitionFilename = os.path.join( HOME, filename) + ".part" + (str(part).rjust(3, '0')) with open(partitionFilename, 'rb') as inputFile: # Think why you scraped the idea of mmap #shutil.copyfileobj(inputFile, outputFile, CHUNK_SIZE * 10) while True: inputBuffer = inputFile.read(CHUNK_SIZE * 10) if not inputBuffer: break outputFile.write(inputBuffer) logger.info("Completed merging part %d of %d" % (part, total_partitions)) outputFile.close() logger.info("Complted merge procedure") assert checkFilesExist(os.path.join(HOME, filename)) return True, filename
def changeStatus(self, statusCode): """Change the status of the Room to statusCode Returns: True <- if successfull False <- if unsuccessfull """ backup_status = self.status try: self.status = statusCode self.status_message = Room.statusCodes[self.status] logger.info("Room %s switched to status: %s " % (self.roomName, self.status)) except Exception as err: logger.error( "Error changing room %s status from %s to %s ;; %s"\ %(self.roomName, backup_status, statusCode, err) ) self.status = backup_status self.status_message = Room.statusCodes[self.status] return False return True
def partitionManager(url: str, fileSize: int, user_count: int, user_id: int) -> bool: """Returns the segmentRanges for the partitions. user_id: the id of current_user, always < user_count This is supposed to be a multiple of activeParties - Manages File Names - Defines Byte Ranges - Divides Download Tasks among Parties """ checkDownloadDirectory(HOME) filename = url.split('/')[-1] #total_partitions = TRESHOLD * user_count total_partitions = user_count byteRanges = calculateByteRange(fileSize, total_partitions) downloadStatus = list() # InitiateDownload for counter, (start, end) in enumerate(byteRanges): # Download only parts assigned to current_user # Currently implemented to download - counter % user_id if not counter % user_count == user_id: continue # Each partition is specified by three digit representation partitionFilename = filename + ".part" + (str(counter).rjust(3, '0')) status, successFile = downloadPart(url, start, end, partitionFilename) if not status: logger.fatal('Download Failed for %s; retrying now' % partitionFilename) status, successFile = downloadPart(url, start, end, partitionFilename) downloadStatus.append((status, successFile)) statusList, _ = zip(*downloadStatus) if all(status == True for status in statusList): logger.info("splitfire Successfull") return True, filename
def startTransfer(filename: str, user_count: int, user_id: int, user_list): logger.info("Initializing chainrain") transferManager(filename, user_count, user_id, user_list) return True
def LogUser(self, request, context): user = decodeUser(request) # NOTE: Fix me in next iteration. add me to database instead. userList.append(user) logger.info("User Added- {} : {}".format(user.userName, user.ip_addr)) return dragonvault_pb2.Ack(status=True, msg="User logged successfully")
if _room.roomName == room.roomName: _live_room = _room break return _live_room def SubmitUrl(self, request, context): pass def StartDownload(self, request, context): pass # Initialize gRPC server server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) # Add DragonvaultServicer to function dragonvault_pb2_grpc.add_DragonvaultServicer_to_server(DragonvaultServicer(), server) logger.info("Starting dragonvault server, Listening on port 50051...") server.add_insecure_port('[::]:50051') server.start() try: while True: time.sleep(86400) except KeyboardInterrupt: server.stop(0) logger.info("Dragonvault terminated. Please restart the server manually")
def terminate(self): try: self.httpd.server_close() logger.info("Terminated the File Server") except Exception as err: logger.error("Unable to terminate the File Server: %s" % err)
def startDragonload(url: str, user_count: int, user_id: int, user_list): logger.info("Starting Dragonload!") status, filename = splitfire.startDownload(url, user_count, user_id) chainrain.startTransfer(filename, user_count, user_id, user_list)
def stop(cls): """ Stop the File Server """ cls.fileServer.kill() logger.info("Terminated local file server at %s" % (HOME))
def run(self): with socketserver.TCPServer(("0.0.0.0", self.PORT), self.Handler) as self.httpd: logger.info("File Server serving at port " + str(PORT)) self.httpd.serve_forever() logger.info("Terminated the File Server")