def pushTheDownloadedFileToCache(self, s3Data, selectedFileOrFolder, topicName): logging.info("Inside pushTheDownloadedFileToCache") ThreadServices.ThreadServices().pushToCacheStream( s3Data, selectedFileOrFolder, topicName)
def catch_all(path): transformed_request = request_preparation(request, path) logging.info("Transformed Request From Web %s", transformed_request) service = ServiceInterface.ServiceInterface(transformed_request) logging.info("Response %s", service.result) if type(service.result) == bytes: return service.result return jsonify(service.result)
def __pushFilesToTheCache(self, eachFileToBeUploaded, topicName): logging.info(" Thread: Inside __pushFilesToTheCache") if (eachFileToBeUploaded["path"] == topicName): selectedFileOrFolder = eachFileToBeUploaded["name"] else: selectedFileOrFolder = eachFileToBeUploaded[ "path"] + eachFileToBeUploaded["name"] try: thread = threading.Thread(target=self._pushFileToCache, args=( eachFileToBeUploaded["file"], selectedFileOrFolder, topicName, )) thread.daemon = True thread.start() except: logging.critical( "Error in Pushing to the stream for Cache Insertion %s", selectedFileOrFolder) return False return True
def pushToCacheStream(self, s3Data, selectedFileOrFolder, topicName): """ Function to submit the signup request to the kafka stream """ producer = KafkaProducer( bootstrap_servers=['localhost:9092'], key_serializer=lambda x: x.encode('utf-8'), value_serializer=lambda x: dumps(x).encode('utf-8')) # Record Preparation Begins.. data_to_placed_in_the_stream = {} data_to_placed_in_the_stream[ "content"] = self.convertBtyeToExactString(s3Data) data_to_placed_in_the_stream["key"] = selectedFileOrFolder data_to_placed_in_the_stream["bucket"] = topicName # Record Preparation Ends... result = producer.send('cache', key=self._insertCacheTask, value=data_to_placed_in_the_stream) sleep(10) if result.is_done: logging.info( "The record have been successfully pushed to the stream") else: logging.warning("Pushing to the stream have been failed")
def findAccessDetailsForSpecificFileorFolder(self, neededFileOrFolder): logging.info("Inside findAccessDetailsForSpecificFileorFolder") for eachFileorFolder in self.accessDetailsForFilesAndFolders: # untested if eachFileorFolder["file"] == neededFileOrFolder: return eachFileorFolder
def displayCurrentFilesandFoldersForSelectedTopics(self): logging.info("Inside displayCurrentFilesandFoldersForSelectedTopics") username = self.request["param"].get(self.username) topicName = self.request["param"].get(self.topicName) if topicName == None: topicName = self.defaultTopicName return FileServerReadTaskHandlers.FileServerReadTaskHandlers( ).getLatestContents(username, topicName)
def putTheUploadedFilesToCache(self, filesToBeUploaded, topicName): logging.info("Inside putTheUploadedFilesToCache") cacheInsertionResults = [ self.__pushFilesToTheCache(eachFileToBeUploaded, topicName) for eachFileToBeUploaded in filesToBeUploaded ] return cacheInsertionResults
def uploadFiles(self, filesToBeUploaded, topicName): logging.info("Inside uploadFiles") versionIds = [ self._s3Connection.uploadObject(eachFileToBeUploaded, topicName) for eachFileToBeUploaded in filesToBeUploaded ] return versionIds
def deleteSelectedFileOrFolders(self): logging.info("Inside deleteSelectedFileOrFolders") selctedFilesToBeDeleted = self.request["data"] owner = selctedFilesToBeDeleted["owner"] selectedFiles = selctedFilesToBeDeleted["Objects"] topicName = self.defaultTopicName return FileServerWriteTaskHandlers.FileServerWriteTaskHandlers( ).deleteFiles(owner, selectedFiles, topicName)
def getContentForSelectedFile(self, topicName, fileName): logging.info("Inside getContentForSelectedFile") parameter = {} parameter["topicName"] = topicName parameter["key"] = fileName contentOfSelectedFile = requests.get(url=self._readServerUrl + "downloadSelectedFileOrFolders", params=parameter) return contentOfSelectedFile.text
def removeUserAccessDetailsForDeletedFiles(self, owner, deletedFiles): logging.info("Inside removeUserAccessDetailsForDeletedFiles") DeletedFilesOfTheOwner = {} DeletedFilesOfTheOwner["owner"] = owner DeletedFilesOfTheOwner["files"] = deletedFiles response = requests.post(url=self.__accessDataUrl + "removeUserAccessDetailsForDeletedFiles", json=DeletedFilesOfTheOwner) return response.json()["status"]
def transformationProcessPipeline(self, s3ResultToBeTransformed): logging.info("Inside transformationProcessPipeline") validInput = self._isInputValid(s3ResultToBeTransformed) if validInput == False: return None else: filteredResult = self._filterValidS3Result(s3ResultToBeTransformed) return filteredResult
def addUserAccessDetailsForFileorFolderInUserAccessManagementServer( self, accessRecordsToBeInserted): logging.info( "Inside addUserAccessDetailsForFileorFolderInUserAccessManagementServer" ) response = requests.post(url=self.__accessDataUrl + "addUserAccessDetailForFile", json=accessRecordsToBeInserted) return response.json()["status"]
def downloadSelectedFileOrFolders(self): logging.info("Inside downloadSelectedFileOrFolders") selectedFileOrFolder = self.request["param"].get( self.selectedFileOrFolder) topicName = self.request["param"].get(self.topicName) if topicName == None: topicName = self.defaultTopicName return FileServerReadTaskHandlers.FileServerReadTaskHandlers( ).getFileOrFolder(selectedFileOrFolder, topicName)
def fetchUserAcessDataForSingleFileFromAccessManagementServer( self, selectedFile): logging.info( "Inside fetchUserAcessDataForSingleFileFromAccessManagementServer") parameter = {} parameter["file"] = selectedFile response = requests.get(url=self.__accessDataUrl + "fetchUserAcessDataForSingleFileOrFolder", params=parameter).json() return response
def accessAssignmentToEachFileAndFolder(self, node, metadata): logging.info("Inside accessAssignmentToEachFileAndFolder") node["owner"] = metadata["owner"] for user in metadata["accessingUsers"]: if (user["name"] == self.userName): if "read" in user: node["read"] = user["read"] if "write" in user: node["write"] = user["write"] if "delete" in user: node["delete"] = user["delete"] return node
def rollbackForUploadOperation(self, topicName, filesPresentInTheSavepoint): logging.info("Inside rollbackForUploadOperation") for eachFilePresentInTheSavepoint in filesPresentInTheSavepoint: uploadResult = self._otherApiCallsForDifferentServers.writeOrUpdateSavepointInS3( topicName, eachFilePresentInTheSavepoint["key"], eachFilePresentInTheSavepoint["data"]["content"]) if uploadResult == False: logging.warning('Error in Rollback for Upload Operation %s', eachFilePresentInTheSavepoint)
def deleteFiles(self, owner, selectedFiles, topicName): logging.info("Inside deleteFiles") """ This function handles the task of deleting the files in S3 """ # Savepoint Creation Begins filesToBeDeleted = FileStructureTransformer.FileStructureTransformer( ).extractFileNamesForDeleteOperation(selectedFiles) folderToCreateSavepoint = FileStructureTransformer.FileStructureTransformer( ).extractFolderNameForSavepointCreationInDeleteOperation( filesToBeDeleted) filesToCreateSavepointExtractedFromS3 = self._s3Connection.listObjectsForFolder( bucketName=topicName, selectedFolder=folderToCreateSavepoint) filesToCreateSavepoint = FileStructureTransformer.FileStructureTransformer( ).transformationProcessPipeline(filesToCreateSavepointExtractedFromS3) savepointCreatedAndItsFiles = SavepointHandler.SavepointHandler( ).createSavepointForDeleteOperation(owner, filesToCreateSavepoint) if savepointCreatedAndItsFiles: RecordsToBeDeleted = {} RecordsToBeDeleted["Objects"] = selectedFiles s3DeletionResults = self._s3Connection.deleteObjects( bucketName=topicName, objects=RecordsToBeDeleted) accessDataDeletionResults = FileMetaDataApi.FileMetaDataApi( ).removeUserAccessDetailsForDeletedFiles(owner, selectedFiles) if s3DeletionResults and accessDataDeletionResults is True: try: savepointCreatedAndItsFiles.clear() except: logging.warning("Error unable to delete Savepoint") return ({"status": True}) else: logging.info("Inside Rollback for delete operation") try: rollBackThread = threading.Thread( target=self. __rollBackSavepointForDeleteOperationInBackground, args=( topicName, savepointCreatedAndItsFiles, )) rollBackThread.daemon = True rollBackThread.start() return ({"status": False}) except: logging.warning("Error unable to Rollback") else: logging.warning("Error unable to Create Savepoint") return ({"status": False})
def writeOrUpdateUserAccessData(self, accessRecord): logging.info("Inside writeOrUpdateUserAccessData") accessRecordsToBeInserted = {} accessRecordsToBeInserted["file"] = accessRecord["file"] accessRecordsToBeInserted["owner"] = accessRecord["owner"] accessRecordsToBeInserted["accessing_users"] = accessRecord[ "accessingUsers"] response = requests.post(url=self.__accessDataUrl + "addUserAccessDetailForFile", json=accessRecordsToBeInserted) return response.json()["status"]
def createSavepointForUploadOperation(self, topicName, owner, selectedFiles): logging.info("Inside createSavepointForUploadOperation") filesCreatedAtSavepointDuringUploadOperation = [] for selectedFile in selectedFiles: file = self.__createSavepointDataFromS3ForEachFile( topicName, selectedFile) filesCreatedAtSavepointDuringUploadOperation.append(file) return filesCreatedAtSavepointDuringUploadOperation
def getLatestContents(self, username, bucketName): logging.info("Inside getLatestContents") s3Connection = DataSourceFactory.DataSourceFactory().getS3Access() resultFromS3 = s3Connection.listObjects(bucketName) accesssDetailsForFilesAndFolders = FileAccessMetaDataApi.FileAccessMetaDataApi( ).fetchUserAcessDataForFilesandFolders() fileStructureTransformer = FileStructureTransformer.FileStructureTransformer( username, bucketName, accesssDetailsForFilesAndFolders) hierarchicalStructureForS3result = fileStructureTransformer.transformationProcessPipeline( resultFromS3) return hierarchicalStructureForS3result
def uploadFilesToDesignatedFolder(self, owner, filesToBeUploaded, topicName, selectedFolder): logging.info("Inside uploadFilesToDesignatedFolder") if selectedFolder is None: return self.uploadFilesToRootFolder(owner, filesToBeUploaded, topicName) else: filesToCreateSavepointExtractedFromS3 = self._s3Connection.listObjectsForFolder( bucketName=topicName, selectedFolder=selectedFolder) filesToCreateSavepoint = FileStructureTransformer.FileStructureTransformer( ).transformationProcessPipeline( filesToCreateSavepointExtractedFromS3) savepointCreatedAndItsFiles = SavepointHandler.SavepointHandler( ).createSavepointForUploadOperation(topicName, owner, filesToCreateSavepoint) if savepointCreatedAndItsFiles: versionIds = self.uploadFiles(filesToBeUploaded, topicName) if False not in versionIds: accessRecordsInsertionResults = self.accessRecordCreationForEachUploadedFiles( owner, filesToBeUploaded) if False not in accessRecordsInsertionResults: cacheInsertionResults = self.putTheUploadedFilesToCache( filesToBeUploaded, topicName) if False not in cacheInsertionResults: logging.info("Cache Insertion is successful") else: try: rollBackThread = threading.Thread( target=self. __rollBackSavepointForUploadOperationInBackground, args=( topicName, savepointCreatedAndItsFiles, )) rollBackThread.daemon = True rollBackThread.start() except: logging.warning("Error: unable to Rollback") logging.critical("Uploading to S3 Failed") else: logging.warning("Savepoint Creation Failed")
def dataStructureTransformerPipeline(self, filteredResult): logging.info("Inside dataStructureTransformerPipeline") """ transforms the filtered s3 result""" transformedResult = {} transformedResult["name"] = filteredResult["bucketName"] transformedResult["children"] = [] file_extension_regex = re.compile( "([a-zA-Z0-9\s_\\.\-\(\):])+(\....)$") start_position = transformedResult previous_split = "" file_extension = re.compile("([a-zA-Z0-9\s_\\.\-\(\):])+(\..*)$") for i in filteredResult["bucketData"]: if file_extension_regex.search( i["objectName"]) and "/" not in i["objectName"]: transformedResult["children"].append(self.leafAssignemnt(i)) elif i["objectName"].endswith('/') or file_extension.search( i["objectName"]): if file_extension.search(i["objectName"]): splitted_root = list( filter(None, i["objectName"].split("/"))) del splitted_root[-1] else: splitted_root = list( filter(None, i["objectName"].split("/"))) if previous_split not in splitted_root: start_position = transformedResult previous_split = self.longestSubstringFinder( previous_split, i["objectName"]) for each_split in splitted_root: start_position = self.pathFinder(start_position, each_split) if (start_position["name"]) == self.bucketName: previous_split = "" if file_extension.search(i["objectName"]): start_position["children"].append(self.leafAssignemnt(i)) else: start_position["children"].append( self.branchAssignment( i["objectName"], i["objectName"].replace(previous_split, "", 1).replace("/", "").strip())) previous_split = i["objectName"] pp.pprint(transformedResult) return transformedResult
def uploadFilesToRootFolder(self, owner, filesToBeUploaded, topicName): logging.info("Inside uploadFilesToRootFolder") versionIds = self.uploadFiles(filesToBeUploaded, topicName) if False not in versionIds: accessRecordsInsertionResults = self.accessRecordCreationForEachUploadedFiles( owner, filesToBeUploaded) if False not in accessRecordsInsertionResults: cacheInsertionResults = self.putTheUploadedFilesToCache( filesToBeUploaded, topicName) if False not in cacheInsertionResults: logging.info("Cache Insertion is successful") else: logging.critical("Uploading to S3 Failed")
def createSavepointForDeleteOperation(self, owner, selectedFiles): logging.info("Inside createSavepointForDeleteOperation") filesCreatedAtSavepointDuringDeleteOperation = [] for selectedFile in selectedFiles: file = self.__createSavepointDataFromS3ForEachFile( self.defaultTopicName, selectedFile) file["data"][ "access"] = self._fileMetaDataApi.fetchUserAcessDataForSingleFileFromAccessManagementServer( selectedFile) filesCreatedAtSavepointDuringDeleteOperation.append(file) return filesCreatedAtSavepointDuringDeleteOperation
def transformationProcessPipeline(self, s3ResultToBeTransformed): logging.info("Inside transformationProcessPipeline") validInput = self.checkInput(s3ResultToBeTransformed) if validInput == False: return self.transformInvalidInput(self.bucketName) else: # transformation begins filteredResult = self.filterValidS3Result(s3ResultToBeTransformed) transformedResult = self.dataStructureTransformerPipeline( filteredResult) return transformedResult
def rollBackforDeleteOperation(self, topicName, filesPresentInTheSavepoint): logging.info("Inside rollBackforDeleteOperation") for eachBackupFile in filesPresentInTheSavepoint: uploadResult = self._otherApiCallsForDifferentServers.writeOrUpdateSavepointInS3( topicName, eachBackupFile["key"], eachBackupFile["data"]["content"]) insertOrUpdateResult = self._fileMetaDataApi.writeOrUpdateUserAccessData( eachBackupFile["data"]["access"]) if insertOrUpdateResult and uploadResult == False: logging.warning('Error in Rollback for Delete Operation %s', eachBackupFile)
def _insertIntoCache(recordsToBeInserted): logging.info("Consumer: Inside insertIntoCache") redisConnection = DataSourceFactory.DataSourceFactory().getRedisAccess( role=_redisRole) keyToBeInserted = recordsToBeInserted[ "bucket"] + '/' + recordsToBeInserted["key"] print("keyToBeInserted------->", keyToBeInserted) insertionResult = redisConnection.insertObject( keyToBeInserted, recordsToBeInserted["content"]) if insertionResult == 1: logging.info("Error in Cache Insertion %s", recordsToBeInserted)
def accessRecordCreationForEachUploadedFiles(self, owner, filesToBeUploaded): logging.info("Inside accessRecordCreationForEachUploadedFiles") accessRecordsToBeInserted = [ self.__createAccessRecord(owner, eachFileToBeUploaded) for eachFileToBeUploaded in filesToBeUploaded ] accessRecordsInsertionResults = [ FileMetaDataApi.FileMetaDataApi(). addUserAccessDetailsForFileorFolderInUserAccessManagementServer( eachAccessRecordsToBeInserted) for eachAccessRecordsToBeInserted in accessRecordsToBeInserted ] return accessRecordsInsertionResults
def filterValidS3Result(self, validS3Result): logging.info("Inside filterValidS3Result") filteredResult = {} filteredResult["bucketName"] = validS3Result["Name"] filteredResult["bucketData"] = [] for eachContent in validS3Result["Contents"]: requiredContentInformation = {} requiredContentInformation["objectName"] = eachContent["Key"] requiredContentInformation["lastModified"] = str( eachContent["LastModified"]) requiredContentInformation["size"] = eachContent["Size"] requiredContentInformation["owner"] = eachContent["Owner"][ "DisplayName"] filteredResult["bucketData"].append(requiredContentInformation) return filteredResult