def downloadPart(url: str, start: int, end: int, filename: str) -> (bool, str):
"""Returns statusCode
Helps to download a segment of a data as filename. Uses tqdm to display progressbar
"""
headers = {'Range' : 'bytes=%d-%d' % (start, end)}
try:
r = requests.get(url, headers = headers, stream = True)
r.raise_for_status()
except requests.exceptions.HTTPError as err:
logger.fatal(err)
return (False, None)
# sys.exit(1)
logger.info("Initiating download of %s" % filename)
fileSegmentSize = int(r.headers.get('Content-Length'))
chunkSize = 1024
with open(os.path.join(HOME, filename), "wb") as fp:
pbar = tqdm(unit = "B", total = fileSegmentSize)
for chunk in r.iter_content(chunk_size = chunkSize):
if chunk: # filter-out keep-alive new chunks; needs investigation
pbar.update(len(chunk))
fp.write(chunk)
logger.info("Download completed for %s" % filename)
return (True, filename)
"""obsolete
with open(filename, "r+b") as fp:
fp.seek(start)
currentPoint = fp.tell()
fp.write(r.content)
"""
#return True
pass
def checkAcceptRange(url: str) -> (bool, str):
"""Return a tuple, (bool, fileSize)
Validates if partial download is possible over the url.
Also returns the Content-Length, and Content-Type if true
"""
status = (False, None)
try:
r = requests.head(url)
r.raise_for_status()
except requests.exceptions.HTTPError as err:
logger.error(err)
# sys.exit(1)
return status
if r.headers.get('Accept-Ranges'):
if r.headers.get('Accept-Ranges') == "bytes":
logger.info("Supports partial download")
# Parse the file size from the header
fileSize = int(r.headers.get('Content-Length'))
return (True, fileSize)
else:
# Handle unsupported partial download
logger.fatal("Unsupported partial Download for file")
else:
logger.warn("File not suitable for download")
return status
def start(cls):
""" Start the File Server """
# python = python3 FIXME: Decide universally
cls.fileServer = Popen(['python3', 'httpServer.py'],
stdout=PIPE,
stderr=PIPE)
logger.info("Initiating local file server at %s" % (HOME))
def getPart(filename: str, partNumber: int, hostIP: str) -> (bool, str):
hostPort = 11112 #NOTE: fixed port for all #None # ?? from config file
partitionFilename = filename + ".part" + (str(partNumber).rjust(3, '0'))
url = "http://" + ":".join([hostIP, str(hostPort)
]) + "/" + partitionFilename
try:
r = requests.get(url, stream=True)
r.raise_for_status()
except requests.exceptions.HTTPError as err:
logger.fatal(err)
return (False, None)
logger.info("Getting part %s from user %s" % (partitionFilename, hostIP))
fileSegmentSize = int(r.headers.get('Content-Length'))
chunkSize = 1024
with open(os.path.join(HOME, partitionFilename), "wb") as fp:
pbar = tqdm(unit="B", total=fileSegmentSize)
for chunk in r.iter_content(chunk_size=chunkSize):
if chunk: # filter-out keep-alive chunks
pbar.update(len(chunk))
fp.write(chunk)
logger.info("Downloading complete for %s from %s" %
(partitionFilename, hostIP))
return (True, partitionFilename)
def startDownload(url: str, user_count: int, user_id: int):
logger.info("Initiating Splitfire")
status, fileSize = checkAcceptRange(url)
if status == True:
success, filename = partitionManager(url, fileSize, user_count,
user_id)
return success, filename
def __del__(self):
"""Release all the users, and kill the room
Currently not clearing the Room.activeUsers (just deleting object clears the air)
"""
for user in self.activeUsers:
user.leaveRoom(self)
logger.info("Successfully deleted room %s" % self.roomName)
def getActiveUsers(self):
"""Returns the list of active users in the Room"""
if self.activeUserCount == 0:
logger.info("Empty room %s" % self.roomName)
return list()
userList = list()
for user in self.activeUsers:
userList.append(user.userName)
return userList
def checkDownloadDirectory(path: str=HOME) -> bool:
""" Verify if the download path exist. Else create new directory"""
if os.path.isdir(HOME):
return True
try:
os.makedirs(HOME)
logger.info("Created Dragonload directory at %s" % HOME)
except Exception as err:
logger.fatal("Unable to create download home directory: %s" % err)
return False
return True
def CreateRoom(self, request, context):
room = decodeRoom(request)
# Check conflicting names
for _room in roomList:
if _room.roomName == room.roomName:
# Room already exist
return dragonvault_pb2.Ack(
status=False,
msg="Another room with name '{}' already exist".format(
room.roomName))
# Create a new room
roomList.append(room)
logger.info("Room Created- {} ".format(room.roomName))
return dragonvault_pb2.Ack(status=True,
msg="Room created Successfully")
def leaveRoom(self, roomObject) -> bool:
""" Leave the specified room.
Arguments:
roomObject: object of class Room
Return:
True <- Successfully exited Room roomObject
False <- Unable to leave the Room roomObject
"""
if self.room == roomObject:
self.room = None
logger.info("User %s exited room %s" %
(self.userName, roomObject.roomName))
return True
logger.error("User %s unable to exit room %s" %
(self.userName, roomObject.roomName))
return False
def terminateRoom(self):
"""Force quit the users from the room
Return: (status <bool>, remaining users <int>)
(True, 0) if no users in the room
(True, count) if room is deleted, and count users are still left
(False, count) if room is not deleted
"""
if self.activeUserCount == 0:
logger.info('Successfully deleted room %s' % (self.roomName))
return True, self.activeUserCount
# Handle active users
for user in self.activeUsers:
user.leaveRoom(self)
self.activeUserCount -= 1
remainingUsers = self.activeUserCount
del self # Delete the class
return True, remainingUsers
def joinRoom(self, roomObject):
""" Join the specified Room.
Arguments:
roomObject: object of class Room
Return:
None <- Unsuccessful
roomObject<- Successfull
"""
if not isinstance(roomObject, Room):
# Format string %s is error; FIXME later
logger.error(
"User %s cannot join Room: %s is not a valid Room object" %
(self.userName, roomObject))
return None
self.room = roomObject
logger.info("User %s joined room %s" %
(self.userName, roomObject.roomName))
return self.room
def mergePartitions(filename: str, total_partitions: int) -> (bool, str):
CHUNK_SIZE = 1024 * 1024
# make sure that all the parts exists
fileExistStatus = True
for part in range(total_partitions):
partitionFilename = os.path.join(
HOME, filename) + ".part" + (str(part).rjust(3, '0'))
fileExistStatus = fileExistStatus and checkFilesExist(
partitionFilename)
if not fileExistStatus:
logger.critical(
"Not all parts available in the directory: Initiating Chainrain again"
)
return False, None
""" Merger all the partitions together into one output file """
outputFile = open(os.path.join(HOME, filename), 'ab')
for part in range(total_partitions):
partitionFilename = os.path.join(
HOME, filename) + ".part" + (str(part).rjust(3, '0'))
with open(partitionFilename, 'rb') as inputFile:
# Think why you scraped the idea of mmap
#shutil.copyfileobj(inputFile, outputFile, CHUNK_SIZE * 10)
while True:
inputBuffer = inputFile.read(CHUNK_SIZE * 10)
if not inputBuffer:
break
outputFile.write(inputBuffer)
logger.info("Completed merging part %d of %d" %
(part, total_partitions))
outputFile.close()
logger.info("Complted merge procedure")
assert checkFilesExist(os.path.join(HOME, filename))
return True, filename
def changeStatus(self, statusCode):
"""Change the status of the Room to statusCode
Returns:
True <- if successfull
False <- if unsuccessfull
"""
backup_status = self.status
try:
self.status = statusCode
self.status_message = Room.statusCodes[self.status]
logger.info("Room %s switched to status: %s " %
(self.roomName, self.status))
except Exception as err:
logger.error(
"Error changing room %s status from %s to %s ;; %s"\
%(self.roomName, backup_status, statusCode, err)
)
self.status = backup_status
self.status_message = Room.statusCodes[self.status]
return False
return True
def partitionManager(url: str, fileSize: int, user_count: int, user_id: int) -> bool:
"""Returns the segmentRanges for the partitions.
user_id: the id of current_user, always < user_count
This is supposed to be a multiple of activeParties
- Manages File Names
- Defines Byte Ranges
- Divides Download Tasks among Parties
"""
checkDownloadDirectory(HOME)
filename = url.split('/')[-1]
#total_partitions = TRESHOLD * user_count
total_partitions = user_count
byteRanges = calculateByteRange(fileSize, total_partitions)
downloadStatus = list()
# InitiateDownload
for counter, (start, end) in enumerate(byteRanges):
# Download only parts assigned to current_user
# Currently implemented to download - counter % user_id
if not counter % user_count == user_id:
continue
# Each partition is specified by three digit representation
partitionFilename = filename + ".part" + (str(counter).rjust(3, '0'))
status, successFile = downloadPart(url, start, end, partitionFilename)
if not status:
logger.fatal('Download Failed for %s; retrying now' % partitionFilename)
status, successFile = downloadPart(url, start, end, partitionFilename)
downloadStatus.append((status, successFile))
statusList, _ = zip(*downloadStatus)
if all(status == True for status in statusList):
logger.info("splitfire Successfull")
return True, filename
def startTransfer(filename: str, user_count: int, user_id: int, user_list):
logger.info("Initializing chainrain")
transferManager(filename, user_count, user_id, user_list)
return True
def LogUser(self, request, context):
user = decodeUser(request)
# NOTE: Fix me in next iteration. add me to database instead.
userList.append(user)
logger.info("User Added- {} : {}".format(user.userName, user.ip_addr))
return dragonvault_pb2.Ack(status=True, msg="User logged successfully")
if _room.roomName == room.roomName:
_live_room = _room
break
return _live_room
def SubmitUrl(self, request, context):
pass
def StartDownload(self, request, context):
pass
# Initialize gRPC server
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
# Add DragonvaultServicer to function
dragonvault_pb2_grpc.add_DragonvaultServicer_to_server(DragonvaultServicer(),
server)
logger.info("Starting dragonvault server, Listening on port 50051...")
server.add_insecure_port('[::]:50051')
server.start()
try:
while True:
time.sleep(86400)
except KeyboardInterrupt:
server.stop(0)
logger.info("Dragonvault terminated. Please restart the server manually")
def terminate(self):
try:
self.httpd.server_close()
logger.info("Terminated the File Server")
except Exception as err:
logger.error("Unable to terminate the File Server: %s" % err)
def startDragonload(url: str, user_count: int, user_id: int, user_list):
logger.info("Starting Dragonload!")
status, filename = splitfire.startDownload(url, user_count, user_id)
chainrain.startTransfer(filename, user_count, user_id, user_list)
def stop(cls):
""" Stop the File Server """
cls.fileServer.kill()
logger.info("Terminated local file server at %s" % (HOME))
def run(self):
with socketserver.TCPServer(("0.0.0.0", self.PORT),
self.Handler) as self.httpd:
logger.info("File Server serving at port " + str(PORT))
self.httpd.serve_forever()
logger.info("Terminated the File Server")
