def checkAccountStatus(self): ''' fetches the Account data from opacity and returns an status object ''' requestBody = dict() requestBody["timestamp"] = Helper.GetUnixMilliseconds() rawPayload = Helper.GetJson(requestBody) payload = self.signPayloadDict(rawPayload) payloadJson = Helper.GetJson(payload) with requests.Session() as s: response = s.post(self._baseUrl + "account-data", data=payloadJson) if response.status_code == 404: raise AttributeError("The provided account handle is invalid!") else: accountData = response.content.decode("utf-8") return AccountStatus.ToObject(accountData)
def AddFileToFolderMetaData(self, folder, fileOrFolder, isFile=False, isFolder=False): metadata = self.getFolderData(folder=folder) keyString = metadata["keyString"] folderMetaData = metadata["metadata"] if isFile: folderMetaData.files.append(fileOrFolder) elif isFolder: folderMetaData.folders.append(fileOrFolder) else: raise EnvironmentError("neither file nor folder") ## clean out bug deleted files # folderMetaData.files = [temp for temp in folderMetaData.files if len(temp.versions)>0] folderMetaDataString = folderMetaData.toString() encryptedFolderMetaData = AesGcm256.encryptString( folderMetaDataString, bytearray.fromhex(keyString)) encryptedFolderMetaDataBase64 = base64.b64encode( encryptedFolderMetaData).decode("utf-8") AesGcm256.decrypt(encryptedFolderMetaData, bytearray.fromhex(keyString)) metaReqDict = { "timestamp": Helper.GetUnixMilliseconds(), "metadataKey": metadata["metadataKey"], "metadata": encryptedFolderMetaDataBase64 } metaReqDictJson = Helper.GetJson(metaReqDict) payload = self.signPayloadDict(metaReqDictJson) payloadJson = Helper.GetJson(payload) with requests.Session() as s: response = s.post(self._baseUrl + "metadata/set", data=payloadJson) return response
def uploadPart(self, fileInfo, metaData, handle, currentIndex, lastIndex): print("Uploading part {} out of {}".format(currentIndex + 1, lastIndex)) #output.put("Uploading part {} out of {}".format(currentIndex + 1, lastIndex)) # start_time = time.time() try: hashBytes = handle[0:32] keyBytes = handle[32:] fileId = hashBytes.hex() partSize = metaData.p.partSize rawpart = Helper.GetPartial(fileInfo, partSize, currentIndex) numChunks = math.ceil(len(rawpart) / metaData.p.blockSize) encryptedBlob = bytearray(0) for chunkIndex in range(numChunks): remaining = len(rawpart) - (chunkIndex * metaData.p.blockSize) if (remaining <= 0): break chunkSize = min(remaining, metaData.p.blockSize) encryptedChunkSize = chunkSize + Constants.BLOCK_OVERHEAD # chunk = bytearray(chunkSize) chunk = rawpart[chunkIndex * metaData.p.blockSize:chunkIndex * metaData.p.blockSize + chunkSize] encryptedChunk = AesGcm256.encrypt(chunk, keyBytes) if (encryptedChunkSize != len(encryptedChunk)): breakpoint() encryptedBlob += encryptedChunk requestBody = dict() requestBody["fileHandle"] = fileId requestBody["partIndex"] = currentIndex + 1 requestBody["endIndex"] = lastIndex requestBodyJson = Helper.GetJson(requestBody) payload = self.SignPayloadForm(requestBodyJson, {"chunkData": encryptedBlob}) with requests.Session() as s: response = s.post(self._baseUrl + "upload", files=payload) except Exception as e: print( f"Failed upload of part {currentIndex + 1} out of {lastIndex}\nError: {e.args}" )
def delete(self, folderPath, handle, skipGetMetadata=False, metadata=None, deleteFiles=True): if skipGetMetadata is False: metadata = self.getFolderData(folderPath) if len(handle) == 128: # file requestBody = dict() requestBody["fileID"] = handle[:64] rawPayload = Helper.GetJson(requestBody) payload = self.signPayloadDict(rawPayload) payloadJson = Helper.GetJson(payload) with requests.Session() as s: response = s.post(self._baseUrl + "delete", data=payloadJson) response = response.content.decode() # successful delete if response == "{}": folderMetaData = metadata["metadata"] fileToDelete = [ file for file in folderMetaData.files if file.versions[0].handle == handle ][0] folderMetaData.files = [ file for file in folderMetaData.files if file.versions[0].handle != handle ] # metadata changes it files list because folderMetaData is a shallow copy of the metadata self.setMetadata(metadata) #print(Fore.GREEN, "Successfully deleted the file: {}".format(fileToDelete.name)) print("Successfully deleted the file: {}".format( fileToDelete.name)) return folderMetaData else: # file doesn't exist #print(Fore.LIGHTRED_EX, "Error:\n{}".format(response)) print("Error:\n{}".format(response)) elif len(handle) == 64: # folder # delete subdirectories aswell as subfiles first folderMetaData = metadata["metadata"] folderToDelete = [ folder for folder in folderMetaData.folders if folder.handle == handle ][0] folderToDeletePath = posixpath.join(folderPath, folderToDelete.name) print("Starting to delete {}".format(folderToDeletePath)) folderToDeleteMetadata = self.getFolderData(folderToDeletePath) folders = folderToDeleteMetadata["metadata"].folders if len(folders) > 0: for folder in folders: self.delete(folderToDeletePath, folder.handle, True, folderToDeleteMetadata, deleteFiles) if deleteFiles: files = folderToDeleteMetadata["metadata"].files if len(files) > 0: for file in files: self.delete("", file.versions[0].handle, True, folderToDeleteMetadata) # delte the folder itself response = self.deleteMetaData(handle) response = json.loads(response.content.decode()) if response["status"] == "metadata successfully deleted": folderMetaData.folders = [ folder for folder in folderMetaData.folders if folder.handle != handle ] response = self.setMetadata(metadata) #print(Fore.GREEN, "Finished deleting: {}".format(folderToDeletePath)) print("Finished deleting: {}".format(folderToDeletePath)) else: #print(Fore.LIGHTRED_EX, "Error:\n{}".format(response)) print("Error:\n{}".format(response)) else: #print(Fore.LIGHTRED_EX, "Handle hasn't the length of 64 or 128") print("Handle hasn't the length of 64 or 128")
def uploadFile(self, filePath, folder) -> bool: fd = dict() fd["fullName"] = os.path.normpath(filePath) fd["name"] = os.path.basename(filePath) if os.path.getsize(filePath) == 0: print( f"Couldn't upload: {fd['fullName']}\nBecause the filesize is equal to 0." ) return False else: fd["size"] = os.path.getsize(filePath) fd["type"] = mimetypes.guess_type(filePath)[0] # fd["type"] = "application/octet-stream" ''' Check first if the file exists already in the metadata -> If yes skip all of this ''' metadataToCheckIn = self.getFolderData(folder=folder) for file in metadataToCheckIn["metadata"].files: if file.name == fd["name"]: print("File: {} already exists".format(fd["name"])) return else: print("Uploading file: {}".format(fd["name"])) metaData = FileMetaData(fd) uploadSize = Helper.GetUploadSize(fd["size"]) endIndex = Helper.GetEndIndex(uploadSize, metaData.p) handle = Helper.GenerateFileKeys() hashBytes = handle[0:32] keyBytes = handle[32:] metaDataJson = Helper.GetJson(metaData.getDict()) encryptedMetaData = AesGcm256.encryptString(metaDataJson, keyBytes) handleHex = handle.hex() fileId = hashBytes.hex() requestBody = dict() requestBody["fileHandle"] = fileId requestBody["fileSizeInByte"] = uploadSize requestBody["endIndex"] = endIndex requestBodyJson = Helper.GetJson(requestBody) payload = self.SignPayloadForm(requestBodyJson, {"metadata": encryptedMetaData}) with requests.Session() as s: response = s.post(self._baseUrl + "init-upload", files=payload) if response.status_code != 200: raise Exception("Error during init-upload\n{}".format( response.content.decode())) ''' Uploading Parts ''' # start_time = time.time() Parallel(n_jobs=8)( delayed(self.uploadPart)(fd, metaData, handle, index, endIndex) for index in range(endIndex)) # print("--- %s seconds ---" % (time.time() - start_time)) # for index in range(endIndex): # #start_time = time.time() # print("Uploading file %s part %d/%d" % (fd["name"], index, endIndex)) # self.uploadPart(fd, metaData, handle, index, endIndex) # #print("--- %s seconds ---" % (time.time() - start_time)) ''' Verify Upload & Retry missing parts ''' requestBody = dict() requestBody["fileHandle"] = fileId requestBodyJson = Helper.GetJson(requestBody) payload = self.signPayloadDict(requestBodyJson) payloadJson = Helper.GetJson(payload) with requests.Session() as s: response = s.post(self._baseUrl + "upload-status", data=payloadJson) retries = 3 content = json.loads(response.content.decode()) if content["status"] != 'File is uploaded': if content["status"] == 'chunks missing': missing_parts = content["missingIndexes"] while len(missing_parts) > 0 and retries > 0: amount = content["endIndex"] for missingPart in missing_parts: print("Trying to re-upload part {} out of {}".format( missingPart, amount)) self.uploadPart(fd, metaData, handle, missingPart - 1, endIndex) with requests.Session() as s: response = s.post(self._baseUrl + "upload-status", data=payloadJson) retries -= 1 content = json.loads(response.content.decode()) if content["status"] == "File is uploaded": break else: if retries == 0: print( f"Failed to upload the {fd['name']}\nReason: Too many retries" ) return missing_parts = content["missingIndexes"] else: raise AssertionError("Unknown status of upload-status") ''' Add file to the metadata ''' fileInfo = FolderMetaFile() fileInfo.name = fd["name"] fileInfo.created = int(os.path.getctime(fd["fullName"]) * 1000) fileInfo.modified = int(os.path.getmtime(fd["fullName"]) * 1000) # fileInfo.created = Helper.GetUnixMilliseconds() # fileInfo.modified = Helper.GetUnixMilliseconds() # fileInfo.type = "file" fileInfo.versions.append( FolderMetaFileVersion( size=fd["size"], handle=handleHex, modified=fileInfo.modified, created=fileInfo.created, # modified=Helper.GetUnixMilliseconds(), # created=Helper.GetUnixMilliseconds() # modified=int(os.path.getmtime(filePath)), # created=int(os.path.getctime(filePath)) )) try: self.AddFileToFolderMetaData(folder, fileInfo, isFile=True) print("Uploaded file: {}".format(fd["name"])) except Exception as e: print( "Failed to attach the file to the folder\nFilehandle: {}\nFolder: {}\nReason: {}" .format(handleHex, folder, e))