def postprocess(dbPath): ''' This is the post process module ''' if not os.path.exists(dbPath): logging.debug('PreProcess: can\'t find database at path') return datastore = DataStore(dbPath) loopcount = 0 while True: sleep(5) if loopcount % 10 == 0: logging.debug('PostProcess is alive') loopcount += 1 #calculate checksums on decrypted files data = datastore.recordsForReHashing() processRecordsReadyToBeHashed(data, datastore) #delete associated files as the job was successful amRecords = datastore.archiveManagerJobsReadyToComplete() for amRecord in amRecords: dataStoreRecords = datastore.recordsForUUID(amRecord.uuid) for record in dataStoreRecords: recordPath = record.fileName if configurationOptions().shouldDeleteOriginal == True: try: os.remove(recordPath) except OSError as e: info = 'PostProcess: Unable to delete the file %s' % (recordPath,) logging.debug(info) datastore.updateArchiveManagerJobAsComplete(amRecord) #move the associated files to the error box as the job had problems amRecords = datastore.archiveManagerJobsThatErrored() for amRecord in amRecords: logging.debug('performing clean up with ' + amRecord.amNumber) batchName = amRecord.amNumber destinationAMFolder = '' errorPath = '' dataStoreRecords = datastore.recordsForUUID(amRecord.uuid) for record in dataStoreRecords: pathStructureName = record.pathStructureName filePath = record.fileName currentPathStructure = configurationOptions().pathStructureWithName(pathStructureName) errorPath = currentPathStructure['errorBox'] print filePath destinationAMFolder = os.path.join(os.path.dirname(filePath), batchName) print 'This is where the working files will go.', destinationAMFolder if not os.path.exists(destinationAMFolder): try: os.mkdir(destinationAMFolder) except OSError as e: pass originalFileName = os.path.basename(filePath).split((batchName + "_"))[1] proposedAMPath = os.path.join(destinationAMFolder, originalFileName) try: # newPath = pathAfterSafelyMovingFileToDestinationFile(filePath, proposedAMPath) print filePath, proposedAMPath shutil.move(filePath, proposedAMPath) except Exception as e: info = 'There was an error moving a file at %s for Archive Manager job %s. This will need to be manually addressed.' % (filePath, batchName) sendFailureEmail(info) continue currentFiles = os.listdir(destinationAMFolder) filesInJob = amRecord.allFilesInRecord() areAllFilesInPlace = True for nFile in filesInJob: if nFile not in currentFiles: areAllFilesInPlace = False if areAllFilesInPlace == True: print "moving files to the error path" try: pathAfterSafelyMovingFolderToDestinationFolder(destinationAMFolder,errorPath) except Exception as e: info = 'PostProcess: Unable to move the file %s' % (filePath,) logging.debug(info) info = 'There was an error moving the folder %s into the outbox at %s' % (destinationAMFolder, errorPath) info = info + '\n' + 'This will need to be addressed manually' sendFailureEmail(info) continue datastore.updateArchiveManagerJobAsComplete(amRecord)
def postprocess(dbPath): ''' This is the post process module ''' if not os.path.exists(dbPath): logging.debug('PreProcess: can\'t find database at path') return datastore = DataStore(dbPath) loopcount = 0 while True: sleep(5) if loopcount % 10 == 0: logging.debug('PostProcess is alive') loopcount += 1 #calculate checksums on decrypted files data = datastore.recordsForReHashing() processRecordsReadyToBeHashed(data, datastore) #delete associated files as the job was successful amRecords = datastore.archiveManagerJobsReadyToComplete() for amRecord in amRecords: dataStoreRecords = datastore.recordsForUUID(amRecord.uuid) for record in dataStoreRecords: recordPath = record.fileName if configurationOptions().shouldDeleteOriginal == True: try: os.remove(recordPath) except OSError as e: info = 'PostProcess: Unable to delete the file %s' % ( recordPath, ) logging.debug(info) datastore.updateArchiveManagerJobAsComplete(amRecord) #move the associated files to the error box as the job had problems amRecords = datastore.archiveManagerJobsThatErrored() for amRecord in amRecords: logging.debug('performing clean up with ' + amRecord.amNumber) batchName = amRecord.amNumber destinationAMFolder = '' errorPath = '' dataStoreRecords = datastore.recordsForUUID(amRecord.uuid) for record in dataStoreRecords: pathStructureName = record.pathStructureName filePath = record.fileName currentPathStructure = configurationOptions( ).pathStructureWithName(pathStructureName) errorPath = currentPathStructure['errorBox'] print filePath destinationAMFolder = os.path.join(os.path.dirname(filePath), batchName) print 'This is where the working files will go.', destinationAMFolder if not os.path.exists(destinationAMFolder): try: os.mkdir(destinationAMFolder) except OSError as e: pass originalFileName = os.path.basename(filePath).split( (batchName + "_"))[1] proposedAMPath = os.path.join(destinationAMFolder, originalFileName) try: # newPath = pathAfterSafelyMovingFileToDestinationFile(filePath, proposedAMPath) print filePath, proposedAMPath shutil.move(filePath, proposedAMPath) except Exception as e: info = 'There was an error moving a file at %s for Archive Manager job %s. This will need to be manually addressed.' % ( filePath, batchName) sendFailureEmail(info) continue currentFiles = os.listdir(destinationAMFolder) filesInJob = amRecord.allFilesInRecord() areAllFilesInPlace = True for nFile in filesInJob: if nFile not in currentFiles: areAllFilesInPlace = False if areAllFilesInPlace == True: print "moving files to the error path" try: pathAfterSafelyMovingFolderToDestinationFolder( destinationAMFolder, errorPath) except Exception as e: info = 'PostProcess: Unable to move the file %s' % ( filePath, ) logging.debug(info) info = 'There was an error moving the folder %s into the outbox at %s' % ( destinationAMFolder, errorPath) info = info + '\n' + 'This will need to be addressed manually' sendFailureEmail(info) continue datastore.updateArchiveManagerJobAsComplete(amRecord)