Example #1
0
def checkSingleFiles(dbPath):
    logging = DefaultLogger()

    if not os.path.exists(dbPath):
        logging.debug('Acquire File: can\'t find database at path')
        return

    datastore = DataStore(dbPath)
    data = datastore.recordsForVerifying()

    for record in data:

        key_id = record.id
        filePath = record.fileName
        recordSize = int(record.fileSize)
        dateModifiedString = record.dateModified
        pathStructureName = record.pathStructureName
        operationType = record.operationType
        isBatch = record.isBatch
        batchName = record.batchName

        dateLastModified = datetime.datetime.strptime(dateModifiedString,
                                                      '%Y-%m-%d %H:%M:%S')
        timeDifference = datetime.datetime.now() - dateLastModified

        #This can change with an if/else should I decide I want to put temp files to be decrypted in another place
        sourcePath = configurationOptions().pathStructureWithName(
            pathStructureName)['inBox']
        workingPath = configurationOptions().pathStructureWithName(
            pathStructureName)['workingBox']

        if timeDifference.seconds < verificationWaitTime:
            continue

        lastSize = recordSize
        currentSize = 0

        if not os.path.exists(filePath):
            logging.debug(
                'Acquire File: Will update record status as the file no longer exists'
            )
            datastore.updateRecordAsMissingWithID(key_id)
            continue

        currentSize = os.path.getsize(filePath)

        if lastSize != currentSize:
            logging.debug(record)
            logging.debug(
                'Acquire File: attempting db modify as file size has changed...'
            )
            datastore.updateRecordWithCurrentSizeAndDateModifiedWithID(
                currentSize,
                datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id)
            continue

        if currentSize == 0:
            continue
            # if the current size is zero, then continue until it isn't or never will be
            # its likely the file has been queued to copy but no data has been moved yet (actual OSX case)

        logging.debug(
            'Acquire File: attempting to lock the file to see if I own the file yet...'
        )

        try:
            fileToCheck = open(filePath, 'rb')
            portalocker.lock(fileToCheck, portalocker.LOCK_EX)
            fileToCheck.close()
            logging.debug(
                'Acquire File: proceeding to update the file status knowing that no one else is using it...'
            )
        except Exception as e:
            logging.debug(
                'Acquire File: unable to lock file as it is likely in use')
            continue

        if datastore.doesTheFilePathExistElseWhereInThePathStructure(
                filePath, operationType, pathStructureName) == True:
            duplicatePath = configurationOptions().pathStructureWithName(
                pathStructureName)['duplicateBox']
            newPath = pathAfterSafelyMovingFileToDestinationFolder(
                filePath, duplicatePath)
            datastore.updateRecordAsDuplicateWithNewPath(newPath, key_id)
            continue

        newPath = filePath
        #Only update
        if isBatch == 1 and operationType == 'Decrypt':
            amRecord = None
            uuidString = fileNameForUUIDFileWithPath(os.path.dirname(filePath))

            if uuidString == None:
                #if I can't resolve the UUID, then resovle it though an AM Record
                #Does file's Archive Manager have data associated with it
                amRecord = datastore.recordWithNumberFromAMJobsTable(batchName)
                if amRecord == None:
                    info = "Acquire File: Archive Manager data doesn't exist for " + filePath
                    info = info + " " + "Marking file as having no AM Data. File will not be moved through the processing queue."
                    logging.debug(info)
                    datastore.updateRecordStatusWithID(
                        datastore.noArchiveManagerDataExistsForRecord(),
                        key_id)
                    continue
            else:
                logging.debug('Updating record %s with UUID %s' %
                              (filePath, uuidString))
                amRecord = datastore.archiveManagerJobsTableRecordWithUUID(
                    uuidString)
                datastore.updateRecordAWithBatchUUIDReference(
                    uuidString, key_id)
        else:
            #at this point, I need to subtract the file's main folder from the pathStructure['inBox']
            #this moves the file from the inbox to the working path
            try:
                newPath = pathAfterSafelyMovingFileToDestinationFolder(
                    filePath, workingPath)
            except Exception as e:
                logging.debug(
                    'This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved'
                )
                logging.debug('Acquire File: Error moving file')
                info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(
                    filePath)
                info = info + '\n' + 'This will require manual intervention as the occurrence is unique.'
                sendFailureEmail(info)
                continue

            logging.debug(
                'Acquire File: updating record file status and path....')
            datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
Example #2
0
def checkSingleFiles(dbPath):
	logging = DefaultLogger()

	if not os.path.exists(dbPath):
		logging.debug('Acquire File: can\'t find database at path')
		return
	
	datastore = DataStore(dbPath)
	data = datastore.recordsForVerifying()

	for record in data:

		key_id 				= record.id
		filePath 			= record.fileName
		recordSize 			= int(record.fileSize)
		dateModifiedString 	= record.dateModified
		pathStructureName 	= record.pathStructureName
		operationType		= record.operationType
		isBatch				= record.isBatch
		batchName			= record.batchName

		dateLastModified = datetime.datetime.strptime(dateModifiedString, '%Y-%m-%d %H:%M:%S')
		timeDifference = datetime.datetime.now() - dateLastModified

		#This can change with an if/else should I decide I want to put temp files to be decrypted in another place
		sourcePath = configurationOptions().pathStructureWithName(pathStructureName)['inBox']
		workingPath = configurationOptions().pathStructureWithName(pathStructureName)['workingBox']

		if timeDifference.seconds < verificationWaitTime:
			continue

		lastSize = recordSize
		currentSize = 0

		if not os.path.exists(filePath):
			logging.debug('Acquire File: Will update record status as the file no longer exists')
			datastore.updateRecordAsMissingWithID(key_id)
			continue

		currentSize = os.path.getsize(filePath)

		if lastSize != currentSize:
			logging.debug(record)
			logging.debug('Acquire File: attempting db modify as file size has changed...')
			datastore.updateRecordWithCurrentSizeAndDateModifiedWithID(currentSize, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id)
			continue

		if currentSize == 0:
			continue
			# if the current size is zero, then continue until it isn't or never will be
			# its likely the file has been queued to copy but no data has been moved yet (actual OSX case) 
			
		logging.debug('Acquire File: attempting to lock the file to see if I own the file yet...')

		try:
			fileToCheck = open(filePath, 'rb')
			portalocker.lock(fileToCheck, portalocker.LOCK_EX)
			fileToCheck.close()
			logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...')
		except Exception as e:
			logging.debug('Acquire File: unable to lock file as it is likely in use')
			continue

		if datastore.doesTheFilePathExistElseWhereInThePathStructure(filePath, operationType, pathStructureName) == True:
			duplicatePath = configurationOptions().pathStructureWithName(pathStructureName)['duplicateBox']
			newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, duplicatePath)
			datastore.updateRecordAsDuplicateWithNewPath(newPath, key_id)
			continue

		newPath = filePath
		#Only update 
		if isBatch == 1 and operationType == 'Decrypt':
			amRecord = None
			uuidString = fileNameForUUIDFileWithPath(os.path.dirname(filePath))

			if uuidString == None:
				#if I can't resolve the UUID, then resovle it though an AM Record
				#Does file's Archive Manager have data associated with it
				amRecord = datastore.recordWithNumberFromAMJobsTable(batchName)
				if amRecord == None:
					info = "Acquire File: Archive Manager data doesn't exist for " + filePath
					info = info + " " + "Marking file as having no AM Data. File will not be moved through the processing queue."
					logging.debug(info)
					datastore.updateRecordStatusWithID(datastore.noArchiveManagerDataExistsForRecord(), key_id)
					continue
			else:
				logging.debug('Updating record %s with UUID %s' % (filePath, uuidString))
				amRecord = datastore.archiveManagerJobsTableRecordWithUUID(uuidString)
				datastore.updateRecordAWithBatchUUIDReference(uuidString, key_id)
		else:
			#at this point, I need to subtract the file's main folder from the pathStructure['inBox']
			#this moves the file from the inbox to the working path
			try:
				newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath)
			except Exception as e:
				logging.debug('This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved')
				logging.debug('Acquire File: Error moving file')
				info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath)
				info = info + '\n' + 'This will require manual intervention as the occurrence is unique.'
				sendFailureEmail(info)
				continue

			logging.debug('Acquire File: updating record file status and path....')
			datastore.updateRecordAsStaticWithNewPath(newPath, key_id)
Example #3
0
def checkSingleFiles(dbPath):
	logging = DefaultLogger()

	if not os.path.exists(dbPath):
		logging.debug('Acquire File: can\'t find database at path')
		return
	
	datastore = DataStore(dbPath)
	data = datastore.recordsForVerifying()

	for record in data:

		key_id 				= record.id
		filePath 			= record.fileName
		recordSize 			= int(record.fileSize)
		dateModifiedString 	= record.dateModified

		dateLastModified = datetime.datetime.strptime(dateModifiedString, '%Y-%m-%d %H:%M:%S')
		timeDifference = datetime.datetime.now() - dateLastModified

		#This can change with an if/else should I decide I want to put temp files to be decrypted in another place
		sourcePath = configurationOptions().defaultPathStructure()['inBox']
		workingPath = configurationOptions().defaultPathStructure()['workingBox']

		if timeDifference.seconds < verificationWaitTime:
			continue

		lastSize = recordSize
		currentSize = 0

		if not os.path.exists(filePath):
			logging.debug('Acquire File: Will update record status as the file no longer exists')
			datastore.updateRecordAsMissingWithID(key_id)
			continue

		currentSize = os.path.getsize(filePath)

		if lastSize != currentSize:
			logging.debug(record)
			logging.debug('Acquire File: attempting db modify as file size has changed...')
			datastore.updateRecordWithCurrentSizeAndDateModifiedWithID(currentSize, datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), key_id)
			continue

		if currentSize == 0:
			continue
			# if the current size is zero, then continue until it isn't or never will be
			# its likely the file has been queued to copy but no data has been moved yet (actual OSX case) 
			
		logging.debug('Acquire File: attempting to lock the file to see if I own the file yet...')

		try:
			fileToCheck = open(filePath, 'rb')
			portalocker.lock(fileToCheck, portalocker.LOCK_EX)
			fileToCheck.close()
			logging.debug('Acquire File: proceeding to update the file status knowing that no one else is using it...')
		except Exception as e:
			logging.debug('Acquire File: unable to lock file as it is likely in use')
			continue

		#must test that file doesn't exist elsewhere in the path

		newPath = filePath
		try:
			newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, workingPath)
		except Exception as e:
			info = '''This shouldn\'t happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved'''
			logging.debug(info)
			logging.debug('Acquire File: Error moving file')
			info = 'There was a problem moving the file into into the queue for: ' + os.path.basename(filePath)
			info = info + '\n' + 'This will require manual intervention as the occurrence is unique.'
			#SEND FAILURE EMAIL
			continue

		logging.debug('Acquire File: updating record file status and path....')
		datastore.updateRecordAsStaticWithNewPath(newPath, key_id)