def analyzeBWFFile(dbPath, identifier=1): logging = DefaultLogger() loopcount = 0 datastore = DataStore(dbPath) try: while True: sleep(60 + random.randint(1, 10)) if loopcount % 20 == 0: logging.debug( 'bwf analyzer loop {} is active...'.format(identifier)) loopcount += 1 if not os.path.exists(dbPath): logging.debug('Acquire File: can not find database at path') return record = None #if daisy is not up then just wait until it is if isDaisyUp() == False: logging.debug('Daisy does not appear to be up') continue #get a lock on the file lock = lockWithFile() try: lock.acquire(timeout=-1) if lock.i_am_locking(): record = datastore.oneRecordReadyForProcessing() if record != None: logging.debug( 'process {} is acquiring the lock'.format( identifier)) datastore.updateRecordAsInProcess(record.id) lock.release() except Exception as e: pass if record == None: continue filePath = record.fileName #lets check that is has a genuine Daisy Number if getDaisyNumber(os.path.basename(filePath)) == None: errorBox = configurationOptions().defaultPathStructure( )['errorBox'] errorBox = os.path.expanduser(errorBox) sendProcessFailureMessage({ 'subject': 'BWF Error: file added that has no DANumber', 'message': 'A file, %s, was deposited that does not have a Daisy Number' % (os.path.basename(filePath)) }) #move to errorBox try: print "Moving file %s into %s" % (filePath, errorBox) newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, errorBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into the errorBox for: ' + os.path.basename( filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({ 'subject': 'BWF Error', 'message': info }) logging.debug(info) datastore.updateRecordAsNotHavingADaisyNumber(record.id) continue #lets look up metadata before we even proceed, if can't get the metadata we don't want to analyze files dataTuple = retrieveDataForDANumber(os.path.basename(filePath), identifier) logging.debug('Data for {} Before: {}'.format( os.path.basename(filePath), dataTuple)) if dataTuple == None: #ok, lets send an email that will be sent at a maximum of 1 per 4 hours result = "Process Error: Daisy Information not Available:" + e.message sendPeriodicFailureMessage(result) logging.debug('A Periodic Failure Message attempt was made.') continue result = None resultObject = None vendor = dataTuple[0] comments = dataTuple[1] status = dataTuple[2] #once we have the metadata, lets examine the file try: logging.debug('Will examine %s in loop %s' % (filePath, str(identifier))) resultObject = multiChannelBWFFileAnalysis(filePath) result = json.dumps(resultObject) if resultObject == None: logging.debug( 'The analysis of the file %s is "None". This should not occur.' % (filePath)) raise Exception( 'The analysis of the file %s is "None". This should not occur.' % (filePath)) except Exception as e: logging.debug( 'An exception occurred with %s in identifier %s.' % (filePath, str(identifier))) #mark as error datastore.updateRecordWithAnalysisError(record.id) errorBox = configurationOptions().defaultPathStructure( )['errorBox'] errorBox = os.path.expanduser(errorBox) #send email result = "Process Error: An Exception occurred when processing the file: %s. The file will be moved to %s" % ( e.message, errorBox) logging.debug(result) sendProcessFailureMessage({ 'subject': 'Process Error', 'message': result }) #move to errorBox try: print "Moving file %s into %s" % (filePath, errorBox) logging.debug("Moving file %s into %s" % (filePath, errorBox)) newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, errorBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into the errorBox for: ' + os.path.basename( filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({ 'subject': 'Process Error Moving File', 'message': info }) logging.debug(info) continue info = 'PreExisting Data for the following file %s: %s %s %s' % ( os.path.basename(filePath), comments, vendor, status) logging.debug(info) resultObject['vendor'] = vendor #The result object is not None as we would have bailed otherwise if resultObject['result'] == 'success': if comments == None: comments = '' #update Comments comments = stringMinusBWFAnalyzerInfo(comments) if comments != '': comments += " " comments += BWFAnalyzerInfoForSuccess( os.path.basename(filePath)) success = setComments( getDaisyNumber(os.path.basename(filePath)), comments) #update local datastore datastore.updateRecordWithComments(comments, record.id) #did we successfully update the comments? if success == True: #update comments field in db and set to success logging.debug('updating comments successfully') datastore.successfullyUpdatedDaisyComments(record.id) else: #put infor in db that you couldn't update Daisy logging.debug('not updating comments successfully') datastore.failedToUpdateDaisyComments(record.id) #update the status to pending fix #only if the status is Needs Attention, otherwise we don't have any further knowledge of what is going on nextStatus = 'NO CHANGE' success = True if status == "Needs Attention": #ok to update status success = setStatusAsPendingFix( getDaisyNumber(os.path.basename(filePath))) nextStatus = 'Pending Fix' if status in ['Being Made', 'Ordered']: #ok to update status success = setStatusAsPendingArchive( getDaisyNumber(os.path.basename(filePath))) nextStatus = 'Pending Archive' datastore.updateRecordWithDaisyStatus(nextStatus, record.id) if success == True: #update staus field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) else: sendAnalysisFailure(resultObject) if comments == None: comments = '' #update Comments comments = stringMinusBWFAnalyzerInfo(comments) if comments != '': comments += " " comments += BWFAnalyzerInfoForErrors(resultObject['errors']) success = setComments( getDaisyNumber(os.path.basename(filePath)), comments) #update local datastore datastore.updateRecordWithComments(comments, record.id) if success == True: #update comments field in db and set to success logging.debug('updating comments successfully') datastore.successfullyUpdatedDaisyComments(record.id) else: #put infor in db that you couldn't update Daisy logging.debug('not updating comments successfully') datastore.failedToUpdateDaisyComments(record.id) #update Status if status not in ['Being Made', 'Ordered', 'Pending Archive']: #ok to update status success = setStatusAsNeedsAttention( getDaisyNumber(os.path.basename(filePath))) datastore.updateRecordWithDaisyStatus( 'Needs Attention', record.id) if success == True: #update staus field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) else: success = setStatusAsPendingArchive( getDaisyNumber(os.path.basename(filePath))) datastore.updateRecordWithDaisyStatus( 'Pending Archive', record.id) if success == True: #update status field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) if datastore.updateRecordWithAnalysisData(result, record.id) == False: info = 'Unable to save record %d %s' % (record.id, result) sendProcessFailureMessage({ 'subject': 'Process Error Unable To Save Record', 'message': info }) continue #update vendor info datastore.updateRecordWithVendor(vendor, record.id) dataTuple = retrieveDataForDANumber(os.path.basename(filePath), identifier) logging.debug('Data for {} After: {}'.format( os.path.basename(filePath), dataTuple)) #now that we have saved the data, we are ready to move the file nextBox = configurationOptions().defaultPathStructure()['outBox'] if resultObject['result'] != 'success': nextBox = configurationOptions().defaultPathStructure( )['failBox'] nextBox = os.path.expanduser(nextBox) newPath = filePath try: newPath = pathAfterSafelyMovingFileToDestinationFolder( filePath, nextBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into ' + nextBox + ' for: ' + os.path.basename( filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({ 'subject': 'Process Error Moving File', 'message': info }) continue logging.debug( 'Analyze File: preparing to move file to final path...') datastore.updateRecordAsCompleteWithFinalPath(newPath, record.id) except Exception as e: info = 'Exception in analyzeBWFFile' + e.message logging.debug(info) sendProcessFailureMessage({'subject': 'Exception!', 'message': info})
def analyzeBWFFile(dbPath, identifier = 1): logging = DefaultLogger() loopcount = 0 datastore = DataStore(dbPath) try: while True: sleep(60+random.randint(1,10)) if loopcount % 20 == 0: logging.debug('bwf analyzer loop {} is active...'.format(identifier)) loopcount += 1 if not os.path.exists(dbPath): logging.debug('Acquire File: can not find database at path') return record = None #if daisy is not up then just wait until it is if isDaisyUp() == False: logging.debug('Daisy does not appear to be up') continue #get a lock on the file lock = lockWithFile() try: lock.acquire(timeout=-1) if lock.i_am_locking(): record = datastore.oneRecordReadyForProcessing() if record != None: logging.debug('process {} is acquiring the lock'.format(identifier)) datastore.updateRecordAsInProcess(record.id) lock.release() except Exception as e: pass if record == None: continue filePath = record.fileName #lets check that is has a genuine Daisy Number if getDaisyNumber(os.path.basename(filePath)) == None: errorBox = configurationOptions().defaultPathStructure()['errorBox'] errorBox = os.path.expanduser(errorBox) sendProcessFailureMessage({'subject':'BWF Error: file added that has no DANumber', 'message':'A file, %s, was deposited that does not have a Daisy Number' % (os.path.basename(filePath))}) #move to errorBox try: print "Moving file %s into %s" % (filePath, errorBox) newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, errorBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into the errorBox for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({'subject':'BWF Error', 'message':info}) logging.debug(info) datastore.updateRecordAsNotHavingADaisyNumber(record.id) continue #lets look up metadata before we even proceed, if can't get the metadata we don't want to analyze files dataTuple = retrieveDataForDANumber(os.path.basename(filePath), identifier) logging.debug('Data for {} Before: {}'.format(os.path.basename(filePath), dataTuple)) if dataTuple == None: #ok, lets send an email that will be sent at a maximum of 1 per 4 hours result = "Process Error: Daisy Information not Available:" + e.message sendPeriodicFailureMessage(result) logging.debug('A Periodic Failure Message attempt was made.') continue result = None resultObject = None vendor = dataTuple[0] comments = dataTuple[1] status = dataTuple[2] #once we have the metadata, lets examine the file try: logging.debug('Will examine %s in loop %s' % (filePath, str(identifier))) resultObject = multiChannelBWFFileAnalysis(filePath) result = json.dumps(resultObject) if resultObject == None: logging.debug('The analysis of the file %s is "None". This should not occur.' % (filePath)) raise Exception('The analysis of the file %s is "None". This should not occur.' % (filePath)) except Exception as e: logging.debug('An exception occurred with %s in identifier %s.' % (filePath, str(identifier))) #mark as error datastore.updateRecordWithAnalysisError(record.id) errorBox = configurationOptions().defaultPathStructure()['errorBox'] errorBox = os.path.expanduser(errorBox) #send email result = "Process Error: An Exception occurred when processing the file: %s. The file will be moved to %s" % (e.message, errorBox) logging.debug(result) sendProcessFailureMessage({'subject':'Process Error', 'message':result}) #move to errorBox try: print "Moving file %s into %s" % (filePath, errorBox) logging.debug("Moving file %s into %s" % (filePath, errorBox)) newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, errorBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into the errorBox for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({'subject':'Process Error Moving File', 'message':info}) logging.debug(info) continue info = 'PreExisting Data for the following file %s: %s %s %s' % (os.path.basename(filePath), comments, vendor, status) logging.debug(info) resultObject['vendor'] = vendor #The result object is not None as we would have bailed otherwise if resultObject['result'] == 'success': if comments == None: comments = '' #update Comments comments = stringMinusBWFAnalyzerInfo(comments) if comments != '': comments += " " comments += BWFAnalyzerInfoForSuccess(os.path.basename(filePath)) success = setComments(getDaisyNumber(os.path.basename(filePath)), comments) #update local datastore datastore.updateRecordWithComments(comments, record.id) #did we successfully update the comments? if success == True: #update comments field in db and set to success logging.debug('updating comments successfully') datastore.successfullyUpdatedDaisyComments(record.id) else: #put infor in db that you couldn't update Daisy logging.debug('not updating comments successfully') datastore.failedToUpdateDaisyComments(record.id) #update the status to pending fix #only if the status is Needs Attention, otherwise we don't have any further knowledge of what is going on nextStatus = 'NO CHANGE' success = True if status == "Needs Attention": #ok to update status success = setStatusAsPendingFix(getDaisyNumber(os.path.basename(filePath))) nextStatus = 'Pending Fix' if status in ['Being Made', 'Ordered']: #ok to update status success = setStatusAsPendingArchive(getDaisyNumber(os.path.basename(filePath))) nextStatus = 'Pending Archive' datastore.updateRecordWithDaisyStatus(nextStatus, record.id) if success == True: #update staus field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) else: sendAnalysisFailure(resultObject) if comments == None: comments = '' #update Comments comments = stringMinusBWFAnalyzerInfo(comments) if comments != '': comments += " " comments += BWFAnalyzerInfoForErrors(resultObject['errors']) success = setComments(getDaisyNumber(os.path.basename(filePath)), comments) #update local datastore datastore.updateRecordWithComments(comments, record.id) if success == True: #update comments field in db and set to success logging.debug('updating comments successfully') datastore.successfullyUpdatedDaisyComments(record.id) else: #put infor in db that you couldn't update Daisy logging.debug('not updating comments successfully') datastore.failedToUpdateDaisyComments(record.id) #update Status if status not in ['Being Made', 'Ordered', 'Pending Archive']: #ok to update status success = setStatusAsNeedsAttention(getDaisyNumber(os.path.basename(filePath))) datastore.updateRecordWithDaisyStatus('Needs Attention', record.id) if success == True: #update staus field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) else: success = setStatusAsPendingArchive(getDaisyNumber(os.path.basename(filePath))) datastore.updateRecordWithDaisyStatus('Pending Archive', record.id) if success == True: #update status field in db and set to success logging.debug('updating status successfully') datastore.successfullyUpdatedDaisyStatus(record.id) else: #put infor in db that you couldn't update status in Daisy logging.debug('not updating status successfully') datastore.failedToUpdateDaisyStatus(record.id) if datastore.updateRecordWithAnalysisData(result, record.id) == False: info = 'Unable to save record %d %s' % (record.id, result) sendProcessFailureMessage({'subject':'Process Error Unable To Save Record', 'message':info}) continue #update vendor info datastore.updateRecordWithVendor(vendor, record.id) dataTuple = retrieveDataForDANumber(os.path.basename(filePath), identifier) logging.debug('Data for {} After: {}'.format(os.path.basename(filePath),dataTuple)) #now that we have saved the data, we are ready to move the file nextBox = configurationOptions().defaultPathStructure()['outBox'] if resultObject['result'] != 'success': nextBox = configurationOptions().defaultPathStructure()['failBox'] nextBox = os.path.expanduser(nextBox) newPath = filePath try: newPath = pathAfterSafelyMovingFileToDestinationFolder(filePath, nextBox) except Exception as e: logging.debug('Analyze File: Error moving file') info = '''This should not happen as pathAfterSafelyMovingFileToDestinationFolder should create a unique name that avoids any collisions, otherwise the file has been moved''' logging.debug(info) info = 'There was a problem moving the file into into ' + nextBox + ' for: ' + os.path.basename(filePath) info = info + '\n' + 'This will require manual intervention as the occurrence is unique.' sendProcessFailureMessage({'subject':'Process Error Moving File', 'message':info}) continue logging.debug('Analyze File: preparing to move file to final path...') datastore.updateRecordAsCompleteWithFinalPath(newPath, record.id) except Exception as e: info = 'Exception in analyzeBWFFile' + e.message logging.debug(info) sendProcessFailureMessage({'subject':'Exception!', 'message':info})