Example #1
0
def postprocess(dbPath):
	'''
	This is the post process module
	'''
	
	if not os.path.exists(dbPath):
		logging.debug('PreProcess: can\'t find database at path')
		return

	datastore = DataStore(dbPath)

	loopcount = 0	

	while True:
		sleep(5)

		if loopcount % 10 == 0:
			logging.debug('PostProcess is alive')
		loopcount += 1

		#calculate checksums on decrypted files
		data = datastore.recordsForReHashing()

		processRecordsReadyToBeHashed(data, datastore)
		
		#delete associated files as the job was successful
		amRecords = datastore.archiveManagerJobsReadyToComplete()
		for amRecord in amRecords:
			dataStoreRecords = datastore.recordsForUUID(amRecord.uuid)
			for record in dataStoreRecords:
				recordPath = record.fileName
				if configurationOptions().shouldDeleteOriginal == True:
					try:
						os.remove(recordPath)
					except OSError as e:
						info = 'PostProcess: Unable to delete the file %s' % (recordPath,)
						logging.debug(info)
			datastore.updateArchiveManagerJobAsComplete(amRecord)


		#move the associated files to the error box as the job had problems
		amRecords = datastore.archiveManagerJobsThatErrored()
		for amRecord in amRecords:
			logging.debug('performing clean up with ' + amRecord.amNumber)

			batchName 			= amRecord.amNumber
			destinationAMFolder = ''
			errorPath 			= ''

			dataStoreRecords = datastore.recordsForUUID(amRecord.uuid)
			for record in dataStoreRecords:
				pathStructureName 		= record.pathStructureName
				filePath 				= record.fileName
				currentPathStructure 	= configurationOptions().pathStructureWithName(pathStructureName)
				errorPath 				= currentPathStructure['errorBox']
				print filePath

				destinationAMFolder = os.path.join(os.path.dirname(filePath), batchName)
				print 'This is where the working files will go.', destinationAMFolder

				if not os.path.exists(destinationAMFolder):
					try:
						os.mkdir(destinationAMFolder)
					except OSError as e:
						pass

				originalFileName = os.path.basename(filePath).split((batchName + "_"))[1]
				proposedAMPath = os.path.join(destinationAMFolder, originalFileName)

				try:
					# newPath = pathAfterSafelyMovingFileToDestinationFile(filePath, proposedAMPath)
					print filePath, proposedAMPath
					shutil.move(filePath, proposedAMPath)
				except Exception as e:
					info = 'There was an error moving a file at %s for Archive Manager job %s. This will need to be manually addressed.' % (filePath, batchName)
					sendFailureEmail(info)
					continue

				currentFiles = os.listdir(destinationAMFolder)
				filesInJob = amRecord.allFilesInRecord()

				areAllFilesInPlace = True			
				for nFile in filesInJob:
					if nFile not in currentFiles:
						areAllFilesInPlace = False
				if areAllFilesInPlace == True:
					print "moving files to the error path"
					try:
						pathAfterSafelyMovingFolderToDestinationFolder(destinationAMFolder,errorPath)
					except Exception as e:
						info = 'PostProcess: Unable to move the file %s' % (filePath,)
						logging.debug(info)
						info = 'There was an error moving the folder %s into the outbox at %s' % (destinationAMFolder, errorPath)
						info = info + '\n' + 'This will need to be addressed manually'
						sendFailureEmail(info)
						continue

			datastore.updateArchiveManagerJobAsComplete(amRecord)
Example #2
0
def postprocess(dbPath):
    '''
	This is the post process module
	'''

    if not os.path.exists(dbPath):
        logging.debug('PreProcess: can\'t find database at path')
        return

    datastore = DataStore(dbPath)

    loopcount = 0

    while True:
        sleep(5)

        if loopcount % 10 == 0:
            logging.debug('PostProcess is alive')
        loopcount += 1

        #calculate checksums on decrypted files
        data = datastore.recordsForReHashing()

        processRecordsReadyToBeHashed(data, datastore)

        #delete associated files as the job was successful
        amRecords = datastore.archiveManagerJobsReadyToComplete()
        for amRecord in amRecords:
            dataStoreRecords = datastore.recordsForUUID(amRecord.uuid)
            for record in dataStoreRecords:
                recordPath = record.fileName
                if configurationOptions().shouldDeleteOriginal == True:
                    try:
                        os.remove(recordPath)
                    except OSError as e:
                        info = 'PostProcess: Unable to delete the file %s' % (
                            recordPath, )
                        logging.debug(info)
            datastore.updateArchiveManagerJobAsComplete(amRecord)

        #move the associated files to the error box as the job had problems
        amRecords = datastore.archiveManagerJobsThatErrored()
        for amRecord in amRecords:
            logging.debug('performing clean up with ' + amRecord.amNumber)

            batchName = amRecord.amNumber
            destinationAMFolder = ''
            errorPath = ''

            dataStoreRecords = datastore.recordsForUUID(amRecord.uuid)
            for record in dataStoreRecords:
                pathStructureName = record.pathStructureName
                filePath = record.fileName
                currentPathStructure = configurationOptions(
                ).pathStructureWithName(pathStructureName)
                errorPath = currentPathStructure['errorBox']
                print filePath

                destinationAMFolder = os.path.join(os.path.dirname(filePath),
                                                   batchName)
                print 'This is where the working files will go.', destinationAMFolder

                if not os.path.exists(destinationAMFolder):
                    try:
                        os.mkdir(destinationAMFolder)
                    except OSError as e:
                        pass

                originalFileName = os.path.basename(filePath).split(
                    (batchName + "_"))[1]
                proposedAMPath = os.path.join(destinationAMFolder,
                                              originalFileName)

                try:
                    # newPath = pathAfterSafelyMovingFileToDestinationFile(filePath, proposedAMPath)
                    print filePath, proposedAMPath
                    shutil.move(filePath, proposedAMPath)
                except Exception as e:
                    info = 'There was an error moving a file at %s for Archive Manager job %s. This will need to be manually addressed.' % (
                        filePath, batchName)
                    sendFailureEmail(info)
                    continue

                currentFiles = os.listdir(destinationAMFolder)
                filesInJob = amRecord.allFilesInRecord()

                areAllFilesInPlace = True
                for nFile in filesInJob:
                    if nFile not in currentFiles:
                        areAllFilesInPlace = False
                if areAllFilesInPlace == True:
                    print "moving files to the error path"
                    try:
                        pathAfterSafelyMovingFolderToDestinationFolder(
                            destinationAMFolder, errorPath)
                    except Exception as e:
                        info = 'PostProcess: Unable to move the file %s' % (
                            filePath, )
                        logging.debug(info)
                        info = 'There was an error moving the folder %s into the outbox at %s' % (
                            destinationAMFolder, errorPath)
                        info = info + '\n' + 'This will need to be addressed manually'
                        sendFailureEmail(info)
                        continue

            datastore.updateArchiveManagerJobAsComplete(amRecord)