def checkArchiveManagerJobs(dbPath): logging = DefaultLogger() datastore = DataStore(dbPath) amRecords = datastore.archiveManagerJobsReadyToStart() for amRecord in amRecords: areAllFilesAvailableAndReady = True recordsInAMRecord = datastore.recordsForUUID(amRecord.uuid) filesInAMRecord = [x.fileName for x in recordsInAMRecord] filesInCurrentFolder = [] try: filesInCurrentFolder = os.listdir(amRecord.amPath) except Exception as e: pass isThereAnUnknownFilePresent = False for currentFile in filesInCurrentFolder: if currentFile not in filesInAMRecord: isThereAnUnknownFilePresent = True if isThereAnUnknownFilePresent == True: logging.debug('Unknown files are present') pass #report error for currentFile in filesInAMRecord: logging.debug('%s' % (currentFile)) lastComponent = os.path.basename(currentFile) if lastComponent not in filesInCurrentFolder: logging.debug('The following file is not yet available: %s' % (lastComponent)) areAllFilesAvailableAndReady = False if areAllFilesAvailableAndReady == False: logging.debug('Not all of the files are staged yet') continue canLockAllRecords = True data = datastore.recordsForUUID(amRecord.uuid) for record in data: filePath = record.fileName try: fileToCheck = open(filePath, 'rb') portalocker.lock(fileToCheck, portalocker.LOCK_EX) fileToCheck.close() logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...') except Exception as e: logging.debug('Acquire File: unable to lock file as it is likely in use') canLockAllRecords = False if canLockAllRecords == False: logging.debug('Can not lock all of the records yet') continue for record in data: key_id = record.id filePath = record.fileName recordSize = int(record.fileSize) dateModifiedString = record.dateModified pathStructureName = record.pathStructureName operationType = record.operationType isBatch = record.isBatch batchName = record.batchName pathStructureName = record.pathStructureName newPath = filePath workingPath = configurationOptions().pathStructureWithName(pathStructureName)['workingBox'] proposedBatchName = batchName + "_" + os.path.basename(filePath) proposedPath = os.path.join(os.path.dirname(filePath), proposedBatchName) #we prepend the job name to the file here as it belongs to a batch try: if os.path.exists(proposedPath): raise Exception('file already exists') os.rename(filePath, proposedPath) filePath = proposedPath except Exception as e: #this is an unlikely occurrence info = 'There is a duplicate file in the queue for: ' + os.path.basename(filePath) + " " + e.message logging.debug(info) sendFailureEmail(info) continue #at this point, I need to subtract the file's main folder from the pathStructure['inBox'] #this moves the file from the inbox to the working path try: newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath) except Exception as e: logging.debug('This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved') logging.debug('Acquire File: Error moving file') info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendFailureEmail(info) continue logging.debug('Acquire File: updating record file status and path....') datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
def checkArchiveManagerJobs(dbPath): logging = DefaultLogger() datastore = DataStore(dbPath) amRecords = datastore.archiveManagerJobsReadyToStart() for amRecord in amRecords: areAllFilesAvailableAndReady = True recordsInAMRecord = datastore.recordsForUUID(amRecord.uuid) filesInAMRecord = [x.fileName for x in recordsInAMRecord] filesInCurrentFolder = [] try: filesInCurrentFolder = os.listdir(amRecord.amPath) except Exception as e: pass isThereAnUnknownFilePresent = False for currentFile in filesInCurrentFolder: if currentFile not in filesInAMRecord: isThereAnUnknownFilePresent = True if isThereAnUnknownFilePresent == True: logging.debug('Unknown files are present') pass #report error for currentFile in filesInAMRecord: logging.debug('%s' % (currentFile)) lastComponent = os.path.basename(currentFile) if lastComponent not in filesInCurrentFolder: logging.debug('The following file is not yet available: %s' % (lastComponent)) areAllFilesAvailableAndReady = False if areAllFilesAvailableAndReady == False: logging.debug('Not all of the files are staged yet') continue canLockAllRecords = True data = datastore.recordsForUUID(amRecord.uuid) for record in data: filePath = record.fileName try: fileToCheck = open(filePath, 'rb') portalocker.lock(fileToCheck, portalocker.LOCK_EX) fileToCheck.close() logging.debug( 'Acquire File: proceeding to update the file status knowing that no one else is using it...' ) except Exception as e: logging.debug( 'Acquire File: unable to lock file as it is likely in use') canLockAllRecords = False if canLockAllRecords == False: logging.debug('Can not lock all of the records yet') continue for record in data: key_id = record.id filePath = record.fileName recordSize = int(record.fileSize) dateModifiedString = record.dateModified pathStructureName = record.pathStructureName operationType = record.operationType isBatch = record.isBatch batchName = record.batchName pathStructureName = record.pathStructureName newPath = filePath workingPath = configurationOptions().pathStructureWithName( pathStructureName)['workingBox'] proposedBatchName = batchName + "_" + os.path.basename(filePath) proposedPath = os.path.join(os.path.dirname(filePath), proposedBatchName) #we prepend the job name to the file here as it belongs to a batch try: if os.path.exists(proposedPath): raise Exception('file already exists') os.rename(filePath, proposedPath) filePath = proposedPath except Exception as e: #this is an unlikely occurrence info = 'There is a duplicate file in the queue for: ' + os.path.basename( filePath) + " " + e.message logging.debug(info) sendFailureEmail(info) continue #at this point, I need to subtract the file's main folder from the pathStructure['inBox'] #this moves the file from the inbox to the working path try: newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, workingPath) except Exception as e: logging.debug( 'This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved' ) logging.debug('Acquire File: Error moving file') info = 'There was a problem moving the file into into the queue for: ' + os.path.basename( filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendFailureEmail(info) continue logging.debug( 'Acquire File: updating record file status and path....') datastore.updateRecordAsStaticWithNewPath(newPath, key_id)