def createMetadata(self, folder):
dictionary = self.createMetadatakeyAndKeystring(folder=folder)
requestBody = dict()
requestBody["timestamp"] = Helper.GetUnixMilliseconds()
requestBody["metadataKey"] = dictionary["metadataKey"]
rawPayload = Helper.GetJson(requestBody)
payload = self.signPayloadDict(rawPayload)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "metadata/create",
data=payloadJson)
if response.status_code == 403:
print(
"The folder: {} already exists! -> Will use that folder instead"
.format(folder))
return {
"metadataKey": dictionary["metadataKey"],
"addFolder": False
}
else:
# set empty foldermetadata
newfolderMetadata = FolderMetaData()
newfolderMetadata.name = os.path.basename(folder)
newfolderMetadata.created = Helper.GetUnixMilliseconds()
newfolderMetadata.modified = Helper.GetUnixMilliseconds()
dictionary["metadata"] = newfolderMetadata
self.setMetadata(dictionary)
return {
"metadataKey": dictionary["metadataKey"],
"addFolder": True
}
def createMetadatakeyAndKeystring(self, folder):
folder = folder
folderKey = Helper.getFolderHDKey(self._masterKey, folder)
metaDataKey = Helper.getMetaDataKey(folderKey)
keyString = keccak.new(data=bytearray(folderKey.private_hex, "utf-8"),
digest_bits=256).hexdigest()
return {"metadataKey": metaDataKey, "keyString": keyString}
def setMetadata(self, metadata):
keyString = metadata["keyString"]
folderMetaDataString = metadata["metadata"].toString()
encryptedFolderMetaData = AesGcm256.encryptString(
folderMetaDataString, bytearray.fromhex(keyString))
encryptedFolderMetaDataBase64 = base64.b64encode(
encryptedFolderMetaData).decode("utf-8")
AesGcm256.decrypt(encryptedFolderMetaData,
bytearray.fromhex(keyString))
metaReqDict = {
"timestamp": Helper.GetUnixMilliseconds(),
"metadataKey": metadata["metadataKey"],
"metadata": encryptedFolderMetaDataBase64
}
metaReqDictJson = Helper.GetJson(metaReqDict)
payload = self.signPayloadDict(metaReqDictJson)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "metadata/set", data=payloadJson)
folderMetaData = self.decryptMetaData(response, keyString)
metadata["metadata"] = folderMetaData
return metadata
def checkAccountStatus(self):
requestBody = dict()
requestBody["timestamp"] = Helper.GetUnixMilliseconds()
rawPayload = Helper.GetJson(requestBody)
payload = self.signPayloadDict(rawPayload)
payloadJson = Helper.GetJson(payload)
with requests.Session() as s:
response = s.post(self._baseUrl + "account-data", data=payloadJson)
if response.status_code == 404:
raise AttributeError("The provided account handle is invalid!")
else:
accountData = response.content.decode("utf-8")
return AccountStatus.ToObject(accountData)
def getFolderData(self, folder, recreate_if_nonexistent=True):
folderKey = Helper.getFolderHDKey(self._masterKey, folder)
metaDataKey = Helper.getMetaDataKey(folderKey)
keyString = keccak.new(data=bytearray(folderKey.private_hex, "utf-8"),
digest_bits=256).hexdigest()
try:
folderMetaData = self.GetFolderMetaData(metaDataKey, keyString)
self._metaData = folderMetaData
except TypeError as e:
print(f"Error: {e}")
print(f"Folder does not exist; must have been deleted already")
if recreate_if_nonexistent:
folderMetaData = FolderMetaData()
else:
return False
return {
"metadata": folderMetaData,
"keyString": keyString,
"metadataKey": metaDataKey
}
def AddFileToFolderMetaData(self,
folder,
fileOrFolder,
isFile=False,
isFolder=False):
metadata = self.getFolderData(folder=folder)
keyString = metadata["keyString"]
folderMetaData = metadata["metadata"]
if isFile:
folderMetaData.files.append(fileOrFolder)
elif isFolder:
folderMetaData.folders.append(fileOrFolder)
else:
raise EnvironmentError("neither file nor folder")
folderMetaDataString = folderMetaData.toString()
encryptedFolderMetaData = AesGcm256.encryptString(
folderMetaDataString, bytearray.fromhex(keyString))
encryptedFolderMetaDataBase64 = base64.b64encode(
encryptedFolderMetaData).decode("utf-8")
AesGcm256.decrypt(encryptedFolderMetaData,
bytearray.fromhex(keyString))
metaReqDict = {
"timestamp": Helper.GetUnixMilliseconds(),
"metadataKey": metadata["metadataKey"],
"metadata": encryptedFolderMetaDataBase64
}
metaReqDictJson = Helper.GetJson(metaReqDict)
payload = self.signPayloadDict(metaReqDictJson)
payloadJson = Helper.GetJson(payload)
try:
with requests.Session() as s:
response = s.post(self._baseUrl + "metadata/set",
data=payloadJson)
except:
raise
return response
def GetFolderMetaData(self, metaDataKey, keyString):
timestamp = Helper.GetUnixMilliseconds()
payload = dict({"timestamp": timestamp, "metadataKey": metaDataKey})
payloadJson = Helper.GetJson(payload)
payloadMeta = self.signPayloadDict(payloadJson)
payloadMetaJson = Helper.GetJson(payloadMeta)
with requests.Session() as s:
response = s.post(self._baseUrl + "metadata/get",
data=payloadMetaJson)
resultMetaDataEncrypted = response.content.decode("utf-8")
resultMetaDataEncryptedJson = json.loads(resultMetaDataEncrypted)
stringbytes = bytes(resultMetaDataEncryptedJson["metadata"], "utf-8")
stringDecoded = base64.b64decode(stringbytes)
decryptedMetaData = AesGcm256.decrypt(stringDecoded,
bytearray.fromhex(keyString))
metaData = decryptedMetaData.decode("utf-8")
metaData = json.loads(metaData)
folderMetaData = FolderMetaData.ToObject(metaData)
return folderMetaData
def downloadFile(self, fileHandle, savingPath, fileName):
fileId = fileHandle[:64]
fileKey = fileHandle[64:]
key = bytearray.fromhex(fileKey)
payloadJson = json.dumps({"fileID": fileId})
with requests.Session() as s:
response = s.post(self._baseUrl + "download", data=payloadJson)
url = response.content.decode()
# check for good response data
temp_url = json.loads(url)
if isinstance(temp_url, dict) and "fileDownloadUrl" in temp_url:
url = temp_url["fileDownloadUrl"]
else:
raise ValueError(
'Requested file appears to be nonexistent within Opacity storage for your account. Failed to download.'
)
# Get file metadata
with requests.Session() as s:
response = s.get(url + "/metadata")
encryptedMetaData = response.content
# Decrypt file metadata
decryptedMetaData = AesGcm256.decrypt(encryptedMetaData, key)
metaData = json.loads(decryptedMetaData)
# prepare for download
uploadSize = Helper.GetUploadSize(metaData["size"])
partSize = 5245440 # 80 * (Constants.DEFAULT_BLOCK_SIZE + Constants.BLOCK_OVERHEAD)
parts = int(uploadSize / partSize) + 1
# create temp directory where specific file will be saved
temp_download_dir = os.path.join(IN_PROGRESS_DOWNLOADS, fileId)
Path(temp_download_dir).mkdir(parents=True, exist_ok=True)
''' Downloading all parts '''
fileUrl = url + "/file"
print("Downloading file: {}".format(fileName))
# start_time = time.time()
# downloadPart() retrieves all file parts and saves in temp directory
Parallel(n_jobs=5)(
delayed(self.downloadPart)(partNumber, parts, partSize, uploadSize,
fileUrl, temp_download_dir)
for partNumber in range(parts))
# print("--- %s seconds with parallel n = 5---" % (time.time() - start_time))
''' Decrypt the chunks and restore the file '''
chunkSize = metaData["p"]["blockSize"] + Constants.BLOCK_OVERHEAD
chunksAmount = int(uploadSize / chunkSize) + 1
# create complete file name, including path
file_name = os.path.join(savingPath, fileName)
# Remove (Overwrite) local file at same location if it exists
if os.path.isfile(file_name):
os.remove(file_name)
# Assemble and write final file
with open(file_name, 'ab+') as saveFile:
fileIndex = 0
seek = 0
for chunkIndex in range(chunksAmount):
chunkRawBytes = None
with open(
os.path.join(temp_download_dir,
str(fileIndex) + ".part"),
'rb') as partFile:
partFile.seek(seek)
toReadBytes = chunkSize
# if the bytes to read exceed the file in the next iteration of the for loop
# you need to go to the next partFile -> seek from start
if seek + toReadBytes >= os.path.getsize(partFile.name):
toReadBytes = os.path.getsize(partFile.name) - seek
seek = 0
fileIndex = fileIndex + 1
else:
seek = seek + chunkSize
chunkRawBytes = partFile.read(toReadBytes)
decryptedChunk = AesGcm256.decrypt(chunkRawBytes, key)
saveFile.write(decryptedChunk)
# file has been saved to permanent directory
# now cleanup temp download directory and contents
shutil.rmtree(temp_download_dir)
print("Finished download of {}".format(fileName))