def runSequentially(self):
# set up everything needed for logging the errors
root = logging.getLogger()
root.setLevel(logging.INFO)
statusFileName = path_helpers.createPathWExtention(self.dstDir,self.jobname,".run")
fileHandler = logging.FileHandler(statusFileName)
logFormat = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fileHandler.setFormatter(logFormat)
root.addHandler(fileHandler)
numberOfFiles = len(self.files)
numberOfErrors = 0
bTime= time.time()
# The file processing occurs here
logQueue = None
for i, (filename, infodict) in enumerate(zip(self.files, self.infoDicts)):
if numberOfErrors > self.errorNum:
root.info("The job encountered %d errors and the max number of them allowed is %d" %(numberOfErrors,self.errorNum))
break
try:
# returns 1 if file was processed and 0 if file was skipped
exitcode = filerunner.FileRunner(logQueue,filename, self.version,
self.lastVersion, self.modname, self.updatemod,
self.params, self.funcDicts,self.srcDir,
self.dstDir, self.rawDataDir, infodict['reference_Eo'], infodict['technique_name'])
if exitcode.exitSuccess:
root.info('File %s completed %d/%d' %(os.path.basename(filename),i+1,numberOfFiles))
except Exception as someException:
# root.exception will log an ERROR with printed traceback;
# root.error will log an ERROR without traceback
# root.exception(someException)
root.error('Exception raised in file %s:\n' %filename +repr(someException))
numberOfErrors +=1
exitcode = -1
eTime= time.time()
root.info("Processed for %s H:M:S" %(str(datetime.timedelta(seconds=eTime-bTime)),))
timeStamp = time.strftime('%Y%m%d%H%M%S',time.gmtime())
# closing the fileHandler is important or else we cannot rename the file
root.removeHandler(fileHandler)
fileHandler.close()
# the renaming of the run file based on the way the file processing ended
if numberOfErrors > self.errorNum:
try:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname,".error"))
except:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname+timeStamp,".error"))
else:
try:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname,".done"))
except:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname+timeStamp,".done"))
def runParallel(self):
# the path to which to log - will change depending on the way
# processing ends and if a statusFile with the same
# name already exists
statusFileName = path_helpers.createPathWExtention(self.dstDir,self.jobname,".run")
# set up the manager and objects required for logging due to multiprocessing
pmanager = Manager()
# this queue takes messages from individual processes and passes them
# to the QueueListener
loggingQueue = pmanager.Queue()
processPool = Pool()
# handler for the logging file
fileHandler = logging.FileHandler(statusFileName)
logFormat = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
fileHandler.setFormatter(logFormat)
# the QueueListener takes messages from the logging queue and passes
# them through another queue to the fileHandler (logs safely because
# only this main process writes to the fileHandler)
fileLogger = QueueListener(loggingQueue, fileHandler)
fileLogger.start()
# keep track of when processing started
bTime = time.time()
# the jobs to process each of the files
# jobs = [(loggingQueue, filename, self.version, self.lastVersion,
# self.modname, self.updatemod,self.params, self.funcDicts,
# self.srcDir, self.dstDir, self.rawDataDir)
# for filename in self.files]
jobs = [(loggingQueue, filename, self.version, self.lastVersion,
self.modname, self.updatemod, self.params, self.funcDicts,
self.srcDir, self.dstDir, self.rawDataDir, infodict['reference_Eo'], infodict['technique_name']) \
for (filename, infodict)
in zip(self.files, self.infoDicts)]
processPool.map(makeFileRunner, jobs)
# keep track of when processing ended
eTime = time.time()
timeStamp = time.strftime('%Y%m%d%H%M%S',time.gmtime())
# clean up the pool
processPool.close()
processPool.join()
root = logging.getLogger()
if fileLogger.errorCount > self.errorNum:
root.info("The job encountered %d errors and the max number of them allowed is %d" %(fileLogger.errorCount,self.errorNum))
root.info("Processed for %s H:M:S" %(str(datetime.timedelta(seconds=eTime-bTime)),))
fileLogger.stop()
fileHandler.close()
if fileLogger.errorCount > self.errorNum:
try:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname,".error"))
except:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname+timeStamp,".error"))
else:
try:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname,".done"))
except:
os.rename(statusFileName, path_helpers.createPathWExtention(self.dstDir,self.jobname+timeStamp,".done"))
