def checkSingleFiles(dbPath): logging = DefaultLogger() if not os.path.exists(dbPath): logging.debug('Acquire File: can\'t find database at path') return datastore = DataStore(dbPath) data = datastore.recordsForVerifying() for record in data: key_id = record.id filePath = record.fileName recordSize = int(record.fileSize) dateModifiedString = record.dateModified pathStructureName = record.pathStructureName operationType = record.operationType isBatch = record.isBatch batchName = record.batchName dateLastModified = datetime.datetime.strptime(dateModifiedString, '%Y-%m-%d %H:%M:%S') timeDifference = datetime.datetime.now() - dateLastModified #This can change with an if/else should I decide I want to put temp files to be decrypted in another place sourcePath = configurationOptions().pathStructureWithName( pathStructureName)['inBox'] workingPath = configurationOptions().pathStructureWithName( pathStructureName)['workingBox'] if timeDifference.seconds < verificationWaitTime: continue lastSize = recordSize currentSize = 0 if not os.path.exists(filePath): logging.debug( 'Acquire File: Will update record status as the file no longer exists' ) datastore.updateRecordAsMissingWithID(key_id) continue currentSize = os.path.getsize(filePath) if lastSize != currentSize: logging.debug(record) logging.debug( 'Acquire File: attempting db modify as file size has changed...' ) datastore.updateRecordWithCurrentSizeAndDateModifiedWithID( currentSize, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id) continue if currentSize == 0: continue # if the current size is zero, then continue until it isn't or never will be # its likely the file has been queued to copy but no data has been moved yet (actual OSX case) logging.debug( 'Acquire File: attempting to lock the file to see if I own the file yet...' ) try: fileToCheck = open(filePath, 'rb') portalocker.lock(fileToCheck, portalocker.LOCK_EX) fileToCheck.close() logging.debug( 'Acquire File: proceeding to update the file status knowing that no one else is using it...' ) except Exception as e: logging.debug( 'Acquire File: unable to lock file as it is likely in use') continue if datastore.doesTheFilePathExistElseWhereInThePathStructure( filePath, operationType, pathStructureName) == True: duplicatePath = configurationOptions().pathStructureWithName( pathStructureName)['duplicateBox'] newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, duplicatePath) datastore.updateRecordAsDuplicateWithNewPath(newPath, key_id) continue newPath = filePath #Only update if isBatch == 1 and operationType == 'Decrypt': amRecord = None uuidString = fileNameForUUIDFileWithPath(os.path.dirname(filePath)) if uuidString == None: #if I can't resolve the UUID, then resovle it though an AM Record #Does file's Archive Manager have data associated with it amRecord = datastore.recordWithNumberFromAMJobsTable(batchName) if amRecord == None: info = "Acquire File: Archive Manager data doesn't exist for " + filePath info = info + " " + "Marking file as having no AM Data. File will not be moved through the processing queue." logging.debug(info) datastore.updateRecordStatusWithID( datastore.noArchiveManagerDataExistsForRecord(), key_id) continue else: logging.debug('Updating record %s with UUID %s' % (filePath, uuidString)) amRecord = datastore.archiveManagerJobsTableRecordWithUUID( uuidString) datastore.updateRecordAWithBatchUUIDReference( uuidString, key_id) else: #at this point, I need to subtract the file's main folder from the pathStructure['inBox'] #this moves the file from the inbox to the working path try: newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, workingPath) except Exception as e: logging.debug( 'This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved' ) logging.debug('Acquire File: Error moving file') info = 'There was a problem moving the file into into the queue for: ' + os.path.basename( filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendFailureEmail(info) continue logging.debug( 'Acquire File: updating record file status and path....') datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
def checkSingleFiles(dbPath): logging = DefaultLogger() if not os.path.exists(dbPath): logging.debug('Acquire File: can\'t find database at path') return datastore = DataStore(dbPath) data = datastore.recordsForVerifying() for record in data: key_id = record.id filePath = record.fileName recordSize = int(record.fileSize) dateModifiedString = record.dateModified pathStructureName = record.pathStructureName operationType = record.operationType isBatch = record.isBatch batchName = record.batchName dateLastModified = datetime.datetime.strptime(dateModifiedString, '%Y-%m-%d %H:%M:%S') timeDifference = datetime.datetime.now() - dateLastModified #This can change with an if/else should I decide I want to put temp files to be decrypted in another place sourcePath = configurationOptions().pathStructureWithName(pathStructureName)['inBox'] workingPath = configurationOptions().pathStructureWithName(pathStructureName)['workingBox'] if timeDifference.seconds < verificationWaitTime: continue lastSize = recordSize currentSize = 0 if not os.path.exists(filePath): logging.debug('Acquire File: Will update record status as the file no longer exists') datastore.updateRecordAsMissingWithID(key_id) continue currentSize = os.path.getsize(filePath) if lastSize != currentSize: logging.debug(record) logging.debug('Acquire File: attempting db modify as file size has changed...') datastore.updateRecordWithCurrentSizeAndDateModifiedWithID(currentSize, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id) continue if currentSize == 0: continue # if the current size is zero, then continue until it isn't or never will be # its likely the file has been queued to copy but no data has been moved yet (actual OSX case) logging.debug('Acquire File: attempting to lock the file to see if I own the file yet...') try: fileToCheck = open(filePath, 'rb') portalocker.lock(fileToCheck, portalocker.LOCK_EX) fileToCheck.close() logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...') except Exception as e: logging.debug('Acquire File: unable to lock file as it is likely in use') continue if datastore.doesTheFilePathExistElseWhereInThePathStructure(filePath, operationType, pathStructureName) == True: duplicatePath = configurationOptions().pathStructureWithName(pathStructureName)['duplicateBox'] newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, duplicatePath) datastore.updateRecordAsDuplicateWithNewPath(newPath, key_id) continue newPath = filePath #Only update if isBatch == 1 and operationType == 'Decrypt': amRecord = None uuidString = fileNameForUUIDFileWithPath(os.path.dirname(filePath)) if uuidString == None: #if I can't resolve the UUID, then resovle it though an AM Record #Does file's Archive Manager have data associated with it amRecord = datastore.recordWithNumberFromAMJobsTable(batchName) if amRecord == None: info = "Acquire File: Archive Manager data doesn't exist for " + filePath info = info + " " + "Marking file as having no AM Data. File will not be moved through the processing queue." logging.debug(info) datastore.updateRecordStatusWithID(datastore.noArchiveManagerDataExistsForRecord(), key_id) continue else: logging.debug('Updating record %s with UUID %s' % (filePath, uuidString)) amRecord = datastore.archiveManagerJobsTableRecordWithUUID(uuidString) datastore.updateRecordAWithBatchUUIDReference(uuidString, key_id) else: #at this point, I need to subtract the file's main folder from the pathStructure['inBox'] #this moves the file from the inbox to the working path try: newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath) except Exception as e: logging.debug('This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved') logging.debug('Acquire File: Error moving file') info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendFailureEmail(info) continue logging.debug('Acquire File: updating record file status and path....') datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
def checkSingleFiles(dbPath): logging = DefaultLogger() if not os.path.exists(dbPath): logging.debug('Acquire File: can\'t find database at path') return datastore = DataStore(dbPath) data = datastore.recordsForVerifying() for record in data: key_id = record.id filePath = record.fileName recordSize = int(record.fileSize) dateModifiedString = record.dateModified dateLastModified = datetime.datetime.strptime(dateModifiedString, '%Y-%m-%d %H:%M:%S') timeDifference = datetime.datetime.now() - dateLastModified #This can change with an if/else should I decide I want to put temp files to be decrypted in another place sourcePath = configurationOptions().defaultPathStructure()['inBox'] workingPath = configurationOptions().defaultPathStructure()['workingBox'] if timeDifference.seconds < verificationWaitTime: continue lastSize = recordSize currentSize = 0 if not os.path.exists(filePath): logging.debug('Acquire File: Will update record status as the file no longer exists') datastore.updateRecordAsMissingWithID(key_id) continue currentSize = os.path.getsize(filePath) if lastSize != currentSize: logging.debug(record) logging.debug('Acquire File: attempting db modify as file size has changed...') datastore.updateRecordWithCurrentSizeAndDateModifiedWithID(currentSize, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id) continue if currentSize == 0: continue # if the current size is zero, then continue until it isn't or never will be # its likely the file has been queued to copy but no data has been moved yet (actual OSX case) logging.debug('Acquire File: attempting to lock the file to see if I own the file yet...') try: fileToCheck = open(filePath, 'rb') portalocker.lock(fileToCheck, portalocker.LOCK_EX) fileToCheck.close() logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...') except Exception as e: logging.debug('Acquire File: unable to lock file as it is likely in use') continue #must test that file doesn't exist elsewhere in the path newPath = filePath try: newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath) except Exception as e: info = '''This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) logging.debug('Acquire File: Error moving file') info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' #SEND FAILURE EMAIL continue logging.debug('Acquire File: updating record file status and path....') datastore.updateRecordAsStaticWithNewPath(newPath, key_id)