Example #1
0
def checkArchiveManagerJobs(dbPath):
	logging = DefaultLogger()

	datastore = DataStore(dbPath)
	amRecords = datastore.archiveManagerJobsReadyToStart()

	for amRecord in amRecords:
		areAllFilesAvailableAndReady = True
		
		recordsInAMRecord = datastore.recordsForUUID(amRecord.uuid)
		filesInAMRecord = [x.fileName for x in recordsInAMRecord]
		filesInCurrentFolder = []

		try:
			filesInCurrentFolder = os.listdir(amRecord.amPath)
		except Exception as e:
			pass

		isThereAnUnknownFilePresent = False
		for currentFile in filesInCurrentFolder:
			if currentFile not in filesInAMRecord:
				isThereAnUnknownFilePresent = True

		if isThereAnUnknownFilePresent == True:
			logging.debug('Unknown files are present')
			pass
			#report error

		for currentFile in filesInAMRecord:
			logging.debug('%s' % (currentFile))
			lastComponent = os.path.basename(currentFile)
			if lastComponent not in filesInCurrentFolder:
				logging.debug('The following file is not yet available: %s' % (lastComponent))
				areAllFilesAvailableAndReady = False

		if areAllFilesAvailableAndReady == False:
			logging.debug('Not all of the files are staged yet')
			continue

		canLockAllRecords = True

		data = datastore.recordsForUUID(amRecord.uuid)

		for record in data:
			
			filePath = record.fileName

			try:
				fileToCheck = open(filePath, 'rb')
				portalocker.lock(fileToCheck, portalocker.LOCK_EX)
				fileToCheck.close()
				logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...')
			except Exception as e:
				logging.debug('Acquire File: unable to lock file as it is likely in use')
				canLockAllRecords = False

		if canLockAllRecords == False:
			logging.debug('Can not lock all of the records yet')
			continue

		for record in data:

			key_id 				= record.id
			filePath 			= record.fileName
			recordSize 			= int(record.fileSize)
			dateModifiedString 	= record.dateModified
			pathStructureName 	= record.pathStructureName
			operationType		= record.operationType
			isBatch				= record.isBatch
			batchName			= record.batchName
			pathStructureName 	= record.pathStructureName

			newPath = filePath
			workingPath = configurationOptions().pathStructureWithName(pathStructureName)['workingBox']

			proposedBatchName = batchName + "_" + os.path.basename(filePath)
			proposedPath = os.path.join(os.path.dirname(filePath), proposedBatchName) 

			#we prepend the job name to the file here as it belongs to a batch
			try:
				if os.path.exists(proposedPath):
					raise Exception('file already exists')
				os.rename(filePath, proposedPath)
				filePath = proposedPath
			except Exception as e:
				#this is an unlikely occurrence
				info = 'There is a duplicate file in the queue for: ' + os.path.basename(filePath) + " " + e.message
				logging.debug(info)
				sendFailureEmail(info)
				continue

			#at this point, I need to subtract the file's main folder from the pathStructure['inBox']
			#this moves the file from the inbox to the working path
			try:
				newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath)
			except Exception as e:
				logging.debug('This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved')
				logging.debug('Acquire File: Error moving file')
				info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath)
				info = info + '\n' + 'This will require manual intervention as the occurrence is unique.'
				sendFailureEmail(info)
				continue

			logging.debug('Acquire File: updating record file status and path....')
			datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
Example #2
0
def checkArchiveManagerJobs(dbPath):
    logging = DefaultLogger()

    datastore = DataStore(dbPath)
    amRecords = datastore.archiveManagerJobsReadyToStart()

    for amRecord in amRecords:
        areAllFilesAvailableAndReady = True

        recordsInAMRecord = datastore.recordsForUUID(amRecord.uuid)
        filesInAMRecord = [x.fileName for x in recordsInAMRecord]
        filesInCurrentFolder = []

        try:
            filesInCurrentFolder = os.listdir(amRecord.amPath)
        except Exception as e:
            pass

        isThereAnUnknownFilePresent = False
        for currentFile in filesInCurrentFolder:
            if currentFile not in filesInAMRecord:
                isThereAnUnknownFilePresent = True

        if isThereAnUnknownFilePresent == True:
            logging.debug('Unknown files are present')
            pass
            #report error

        for currentFile in filesInAMRecord:
            logging.debug('%s' % (currentFile))
            lastComponent = os.path.basename(currentFile)
            if lastComponent not in filesInCurrentFolder:
                logging.debug('The following file is not yet available: %s' %
                              (lastComponent))
                areAllFilesAvailableAndReady = False

        if areAllFilesAvailableAndReady == False:
            logging.debug('Not all of the files are staged yet')
            continue

        canLockAllRecords = True

        data = datastore.recordsForUUID(amRecord.uuid)

        for record in data:

            filePath = record.fileName

            try:
                fileToCheck = open(filePath, 'rb')
                portalocker.lock(fileToCheck, portalocker.LOCK_EX)
                fileToCheck.close()
                logging.debug(
                    'Acquire File: proceeding to update the file status knowing that no one else is using it...'
                )
            except Exception as e:
                logging.debug(
                    'Acquire File: unable to lock file as it is likely in use')
                canLockAllRecords = False

        if canLockAllRecords == False:
            logging.debug('Can not lock all of the records yet')
            continue

        for record in data:

            key_id = record.id
            filePath = record.fileName
            recordSize = int(record.fileSize)
            dateModifiedString = record.dateModified
            pathStructureName = record.pathStructureName
            operationType = record.operationType
            isBatch = record.isBatch
            batchName = record.batchName
            pathStructureName = record.pathStructureName

            newPath = filePath
            workingPath = configurationOptions().pathStructureWithName(
                pathStructureName)['workingBox']

            proposedBatchName = batchName + "_" + os.path.basename(filePath)
            proposedPath = os.path.join(os.path.dirname(filePath),
                                        proposedBatchName)

            #we prepend the job name to the file here as it belongs to a batch
            try:
                if os.path.exists(proposedPath):
                    raise Exception('file already exists')
                os.rename(filePath, proposedPath)
                filePath = proposedPath
            except Exception as e:
                #this is an unlikely occurrence
                info = 'There is a duplicate file in the queue for: ' + os.path.basename(
                    filePath) + " " + e.message
                logging.debug(info)
                sendFailureEmail(info)
                continue

            #at this point, I need to subtract the file's main folder from the pathStructure['inBox']
            #this moves the file from the inbox to the working path
            try:
                newPath = pathAfterSafelyMovingFileToDestinationFolder(
                    filePath, workingPath)
            except Exception as e:
                logging.debug(
                    'This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved'
                )
                logging.debug('Acquire File: Error moving file')
                info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(
                    filePath)
                info = info + '\n' + 'This will require manual intervention as the occurrence is unique.'
                sendFailureEmail(info)
                continue

            logging.debug(
                'Acquire File: updating record file status and path....')
            datastore.updateRecordAsStaticWithNewPath(newPath, key_id)