def checkAccountStatus(self):
'''
fetches the Account data from opacity and returns an status object
'''
requestBody = dict()
requestBody["timestamp"] = Helper.GetUnixMilliseconds()
rawPayload = Helper.GetJson(requestBody)
payload = self.signPayloadDict(rawPayload)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "account-data", data=payloadJson)
if response.status_code == 404:
raise AttributeError("The provided account handle is invalid!")
else:
accountData = response.content.decode("utf-8")
return AccountStatus.ToObject(accountData)
def AddFileToFolderMetaData(self,
folder,
fileOrFolder,
isFile=False,
isFolder=False):
metadata = self.getFolderData(folder=folder)
keyString = metadata["keyString"]
folderMetaData = metadata["metadata"]
if isFile:
folderMetaData.files.append(fileOrFolder)
elif isFolder:
folderMetaData.folders.append(fileOrFolder)
else:
raise EnvironmentError("neither file nor folder")
## clean out bug deleted files
# folderMetaData.files = [temp for temp in folderMetaData.files if len(temp.versions)>0]
folderMetaDataString = folderMetaData.toString()
encryptedFolderMetaData = AesGcm256.encryptString(
folderMetaDataString, bytearray.fromhex(keyString))
encryptedFolderMetaDataBase64 = base64.b64encode(
encryptedFolderMetaData).decode("utf-8")
AesGcm256.decrypt(encryptedFolderMetaData,
bytearray.fromhex(keyString))
metaReqDict = {
"timestamp": Helper.GetUnixMilliseconds(),
"metadataKey": metadata["metadataKey"],
"metadata": encryptedFolderMetaDataBase64
}
metaReqDictJson = Helper.GetJson(metaReqDict)
payload = self.signPayloadDict(metaReqDictJson)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "metadata/set", data=payloadJson)
return response
def uploadPart(self, fileInfo, metaData, handle, currentIndex, lastIndex):
print("Uploading part {} out of {}".format(currentIndex + 1,
lastIndex))
#output.put("Uploading part {} out of {}".format(currentIndex + 1, lastIndex))
# start_time = time.time()
try:
hashBytes = handle[0:32]
keyBytes = handle[32:]
fileId = hashBytes.hex()
partSize = metaData.p.partSize
rawpart = Helper.GetPartial(fileInfo, partSize, currentIndex)
numChunks = math.ceil(len(rawpart) / metaData.p.blockSize)
encryptedBlob = bytearray(0)
for chunkIndex in range(numChunks):
remaining = len(rawpart) - (chunkIndex * metaData.p.blockSize)
if (remaining <= 0):
break
chunkSize = min(remaining, metaData.p.blockSize)
encryptedChunkSize = chunkSize + Constants.BLOCK_OVERHEAD
# chunk = bytearray(chunkSize)
chunk = rawpart[chunkIndex * metaData.p.blockSize:chunkIndex *
metaData.p.blockSize + chunkSize]
encryptedChunk = AesGcm256.encrypt(chunk, keyBytes)
if (encryptedChunkSize != len(encryptedChunk)):
breakpoint()
encryptedBlob += encryptedChunk
requestBody = dict()
requestBody["fileHandle"] = fileId
requestBody["partIndex"] = currentIndex + 1
requestBody["endIndex"] = lastIndex
requestBodyJson = Helper.GetJson(requestBody)
payload = self.SignPayloadForm(requestBodyJson,
{"chunkData": encryptedBlob})
with requests.Session() as s:
response = s.post(self._baseUrl + "upload", files=payload)
except Exception as e:
print(
f"Failed upload of part {currentIndex + 1} out of {lastIndex}\nError: {e.args}"
)
def delete(self,
folderPath,
handle,
skipGetMetadata=False,
metadata=None,
deleteFiles=True):
if skipGetMetadata is False:
metadata = self.getFolderData(folderPath)
if len(handle) == 128: # file
requestBody = dict()
requestBody["fileID"] = handle[:64]
rawPayload = Helper.GetJson(requestBody)
payload = self.signPayloadDict(rawPayload)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "delete", data=payloadJson)
response = response.content.decode()
# successful delete
if response == "{}":
folderMetaData = metadata["metadata"]
fileToDelete = [
file for file in folderMetaData.files
if file.versions[0].handle == handle
][0]
folderMetaData.files = [
file for file in folderMetaData.files
if file.versions[0].handle != handle
]
# metadata changes it files list because folderMetaData is a shallow copy of the metadata
self.setMetadata(metadata)
#print(Fore.GREEN, "Successfully deleted the file: {}".format(fileToDelete.name))
print("Successfully deleted the file: {}".format(
fileToDelete.name))
return folderMetaData
else: # file doesn't exist
#print(Fore.LIGHTRED_EX, "Error:\n{}".format(response))
print("Error:\n{}".format(response))
elif len(handle) == 64: # folder
# delete subdirectories aswell as subfiles first
folderMetaData = metadata["metadata"]
folderToDelete = [
folder for folder in folderMetaData.folders
if folder.handle == handle
][0]
folderToDeletePath = posixpath.join(folderPath,
folderToDelete.name)
print("Starting to delete {}".format(folderToDeletePath))
folderToDeleteMetadata = self.getFolderData(folderToDeletePath)
folders = folderToDeleteMetadata["metadata"].folders
if len(folders) > 0:
for folder in folders:
self.delete(folderToDeletePath, folder.handle, True,
folderToDeleteMetadata, deleteFiles)
if deleteFiles:
files = folderToDeleteMetadata["metadata"].files
if len(files) > 0:
for file in files:
self.delete("", file.versions[0].handle, True,
folderToDeleteMetadata)
# delte the folder itself
response = self.deleteMetaData(handle)
response = json.loads(response.content.decode())
if response["status"] == "metadata successfully deleted":
folderMetaData.folders = [
folder for folder in folderMetaData.folders
if folder.handle != handle
]
response = self.setMetadata(metadata)
#print(Fore.GREEN, "Finished deleting: {}".format(folderToDeletePath))
print("Finished deleting: {}".format(folderToDeletePath))
else:
#print(Fore.LIGHTRED_EX, "Error:\n{}".format(response))
print("Error:\n{}".format(response))
else:
#print(Fore.LIGHTRED_EX, "Handle hasn't the length of 64 or 128")
print("Handle hasn't the length of 64 or 128")
def uploadFile(self, filePath, folder) -> bool:
fd = dict()
fd["fullName"] = os.path.normpath(filePath)
fd["name"] = os.path.basename(filePath)
if os.path.getsize(filePath) == 0:
print(
f"Couldn't upload: {fd['fullName']}\nBecause the filesize is equal to 0."
)
return False
else:
fd["size"] = os.path.getsize(filePath)
fd["type"] = mimetypes.guess_type(filePath)[0]
# fd["type"] = "application/octet-stream"
'''
Check first if the file exists already in the metadata
-> If yes skip all of this
'''
metadataToCheckIn = self.getFolderData(folder=folder)
for file in metadataToCheckIn["metadata"].files:
if file.name == fd["name"]:
print("File: {} already exists".format(fd["name"]))
return
else:
print("Uploading file: {}".format(fd["name"]))
metaData = FileMetaData(fd)
uploadSize = Helper.GetUploadSize(fd["size"])
endIndex = Helper.GetEndIndex(uploadSize, metaData.p)
handle = Helper.GenerateFileKeys()
hashBytes = handle[0:32]
keyBytes = handle[32:]
metaDataJson = Helper.GetJson(metaData.getDict())
encryptedMetaData = AesGcm256.encryptString(metaDataJson, keyBytes)
handleHex = handle.hex()
fileId = hashBytes.hex()
requestBody = dict()
requestBody["fileHandle"] = fileId
requestBody["fileSizeInByte"] = uploadSize
requestBody["endIndex"] = endIndex
requestBodyJson = Helper.GetJson(requestBody)
payload = self.SignPayloadForm(requestBodyJson,
{"metadata": encryptedMetaData})
with requests.Session() as s:
response = s.post(self._baseUrl + "init-upload", files=payload)
if response.status_code != 200:
raise Exception("Error during init-upload\n{}".format(
response.content.decode()))
'''
Uploading Parts
'''
# start_time = time.time()
Parallel(n_jobs=8)(
delayed(self.uploadPart)(fd, metaData, handle, index, endIndex)
for index in range(endIndex))
# print("--- %s seconds ---" % (time.time() - start_time))
# for index in range(endIndex):
# #start_time = time.time()
# print("Uploading file %s part %d/%d" % (fd["name"], index, endIndex))
# self.uploadPart(fd, metaData, handle, index, endIndex)
# #print("--- %s seconds ---" % (time.time() - start_time))
'''
Verify Upload & Retry missing parts
'''
requestBody = dict()
requestBody["fileHandle"] = fileId
requestBodyJson = Helper.GetJson(requestBody)
payload = self.signPayloadDict(requestBodyJson)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "upload-status",
data=payloadJson)
retries = 3
content = json.loads(response.content.decode())
if content["status"] != 'File is uploaded':
if content["status"] == 'chunks missing':
missing_parts = content["missingIndexes"]
while len(missing_parts) > 0 and retries > 0:
amount = content["endIndex"]
for missingPart in missing_parts:
print("Trying to re-upload part {} out of {}".format(
missingPart, amount))
self.uploadPart(fd, metaData, handle, missingPart - 1,
endIndex)
with requests.Session() as s:
response = s.post(self._baseUrl + "upload-status",
data=payloadJson)
retries -= 1
content = json.loads(response.content.decode())
if content["status"] == "File is uploaded":
break
else:
if retries == 0:
print(
f"Failed to upload the {fd['name']}\nReason: Too many retries"
)
return
missing_parts = content["missingIndexes"]
else:
raise AssertionError("Unknown status of upload-status")
'''
Add file to the metadata
'''
fileInfo = FolderMetaFile()
fileInfo.name = fd["name"]
fileInfo.created = int(os.path.getctime(fd["fullName"]) * 1000)
fileInfo.modified = int(os.path.getmtime(fd["fullName"]) * 1000)
# fileInfo.created = Helper.GetUnixMilliseconds()
# fileInfo.modified = Helper.GetUnixMilliseconds()
# fileInfo.type = "file"
fileInfo.versions.append(
FolderMetaFileVersion(
size=fd["size"],
handle=handleHex,
modified=fileInfo.modified,
created=fileInfo.created,
# modified=Helper.GetUnixMilliseconds(),
# created=Helper.GetUnixMilliseconds()
# modified=int(os.path.getmtime(filePath)),
# created=int(os.path.getctime(filePath))
))
try:
self.AddFileToFolderMetaData(folder, fileInfo, isFile=True)
print("Uploaded file: {}".format(fd["name"]))
except Exception as e:
print(
"Failed to attach the file to the folder\nFilehandle: {}\nFolder: {}\nReason: {}"
.format(handleHex, folder, e))
