def uploadFile(self, filePath, folder) -> bool:
fd = dict()
fd["fullName"] = os.path.normpath(filePath)
fd["name"] = os.path.basename(filePath)
if os.path.getsize(filePath) == 0:
print(
f"Couldn't upload: {fd['fullName']}\nBecause the filesize is equal to 0."
)
return False
else:
fd["size"] = os.path.getsize(filePath)
fd["type"] = mimetypes.guess_type(filePath)[0]
# fd["type"] = "application/octet-stream"
'''
Check first if the file exists already in the metadata
-> If yes skip all of this
'''
metadataToCheckIn = self.getFolderData(folder=folder)
for file in metadataToCheckIn["metadata"].files:
if file.name == fd["name"]:
print("File: {} already exists".format(fd["name"]))
return
else:
print("Uploading file: {}".format(fd["name"]))
metaData = FileMetaData(fd)
uploadSize = Helper.GetUploadSize(fd["size"])
endIndex = Helper.GetEndIndex(uploadSize, metaData.p)
handle = Helper.GenerateFileKeys()
hashBytes = handle[0:32]
keyBytes = handle[32:]
metaDataJson = Helper.GetJson(metaData.getDict())
encryptedMetaData = AesGcm256.encryptString(metaDataJson, keyBytes)
handleHex = handle.hex()
fileId = hashBytes.hex()
requestBody = dict()
requestBody["fileHandle"] = fileId
requestBody["fileSizeInByte"] = uploadSize
requestBody["endIndex"] = endIndex
requestBodyJson = Helper.GetJson(requestBody)
payload = self.SignPayloadForm(requestBodyJson,
{"metadata": encryptedMetaData})
with requests.Session() as s:
response = s.post(self._baseUrl + "init-upload", files=payload)
if response.status_code != 200:
raise Exception("Error during init-upload\n{}".format(
response.content.decode()))
'''
Uploading Parts
'''
# start_time = time.time()
Parallel(n_jobs=8)(
delayed(self.uploadPart)(fd, metaData, handle, index, endIndex)
for index in range(endIndex))
# print("--- %s seconds ---" % (time.time() - start_time))
# for index in range(endIndex):
# #start_time = time.time()
# print("Uploading file %s part %d/%d" % (fd["name"], index, endIndex))
# self.uploadPart(fd, metaData, handle, index, endIndex)
# #print("--- %s seconds ---" % (time.time() - start_time))
'''
Verify Upload & Retry missing parts
'''
requestBody = dict()
requestBody["fileHandle"] = fileId
requestBodyJson = Helper.GetJson(requestBody)
payload = self.signPayloadDict(requestBodyJson)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "upload-status",
data=payloadJson)
retries = 3
content = json.loads(response.content.decode())
if content["status"] != 'File is uploaded':
if content["status"] == 'chunks missing':
missing_parts = content["missingIndexes"]
while len(missing_parts) > 0 and retries > 0:
amount = content["endIndex"]
for missingPart in missing_parts:
print("Trying to re-upload part {} out of {}".format(
missingPart, amount))
self.uploadPart(fd, metaData, handle, missingPart - 1,
endIndex)
with requests.Session() as s:
response = s.post(self._baseUrl + "upload-status",
data=payloadJson)
retries -= 1
content = json.loads(response.content.decode())
if content["status"] == "File is uploaded":
break
else:
if retries == 0:
print(
f"Failed to upload the {fd['name']}\nReason: Too many retries"
)
return
missing_parts = content["missingIndexes"]
else:
raise AssertionError("Unknown status of upload-status")
'''
Add file to the metadata
'''
fileInfo = FolderMetaFile()
fileInfo.name = fd["name"]
fileInfo.created = int(os.path.getctime(fd["fullName"]) * 1000)
fileInfo.modified = int(os.path.getmtime(fd["fullName"]) * 1000)
# fileInfo.created = Helper.GetUnixMilliseconds()
# fileInfo.modified = Helper.GetUnixMilliseconds()
# fileInfo.type = "file"
fileInfo.versions.append(
FolderMetaFileVersion(
size=fd["size"],
handle=handleHex,
modified=fileInfo.modified,
created=fileInfo.created,
# modified=Helper.GetUnixMilliseconds(),
# created=Helper.GetUnixMilliseconds()
# modified=int(os.path.getmtime(filePath)),
# created=int(os.path.getctime(filePath))
))
try:
self.AddFileToFolderMetaData(folder, fileInfo, isFile=True)
print("Uploaded file: {}".format(fd["name"]))
except Exception as e:
print(
"Failed to attach the file to the folder\nFilehandle: {}\nFolder: {}\nReason: {}"
.format(handleHex, folder, e))
