def postProcessContent(self): os.chdir(self.context.getConfiguration().getReroWorkingDir()) for fname in glob.glob('*.xml'): os.system("mv " + fname + " " + self.context.getConfiguration().getReroSrcDir()) os.chdir(self.context.getConfiguration().getReroSrcDir() ) for fname in glob.glob('*.xml'): os.system("gzip " + fname ) #check the complete number of processed records numberProcessedRecords = self.context.getResultCollector().getIncrementProcessedRecordNoFurtherDetails() maxDocuments = self.context.getConfiguration().getMaxDocuments() blocked = self.context.getConfiguration().getBlocked() #in case the configuration is set to blocked a if (not maxDocuments is None and not numberProcessedRecords is None and int(numberProcessedRecords) > int(maxDocuments)) \ or blocked: archivedFiles = self.context.getResultCollector().getCollectedArchiveFiles() if not archivedFiles is None and len(archivedFiles) > 0: for filename in archivedFiles.keys(): os.system("unlink " + self.context.getConfiguration().getResultDir() + os.sep + filename) os.system("mv " + archivedFiles[filename] + os.sep + filename + " " + self.context.getConfiguration().getArchiveNotSent()) self.context.getConfiguration().setBlocked('true') SwissbibUtilities.sendNotificationMail(receivers=self.context.getConfiguration().getEMailNotifification(), network=self.context.getConfiguration().getNetworkPrefix(), numberDocuments=numberProcessedRecords, mailserver=self.context.getConfiguration().getMailServer())
def moveContentFile(self): if not self.fileClosed is True: self.closeWriteContext() numberProcessedRecords = self.appContext.getResultCollector( ).getIncrementProcessedRecordNoFurtherDetails() maxDocuments = self.appContext.getConfiguration().getMaxDocuments() blocked = self.appContext.getConfiguration().getBlocked() #in case the configuration is set to blocked a if (not maxDocuments is None and not numberProcessedRecords is None and int(numberProcessedRecords) > int(maxDocuments)) \ or blocked: os.chdir(self.appContext.getConfiguration().getDumpDir()) #collected content shouldn't be sent to CBS because number of collected records is too large os.system( "mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveNotSent()) tCommand = "gzip -9 " + self.appContext.getConfiguration( ).getArchiveNotSent() + os.sep + self.appContext.getConfiguration( ).getSummaryContentFile() os.system(tCommand) if not blocked: self.appContext.getConfiguration().setBlocked('true') os.chdir(self.appContext.getConfiguration().getResultDir()) SwissbibUtilities.sendNotificationMail( receivers=self.appContext.getConfiguration( ).getEMailNotifification(), network=self.appContext.getConfiguration().getNetworkPrefix(), numberDocuments=numberProcessedRecords, mailserver=self.appContext.getConfiguration().getMailServer()) else: os.chdir(self.appContext.getConfiguration().getDumpDir()) os.system( "mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveDir()) tCommand = "gzip -9 " + self.appContext.getConfiguration( ).getArchiveDir() + os.sep + self.appContext.getConfiguration( ).getSummaryContentFile() os.system(tCommand) os.chdir(self.appContext.getConfiguration().getResultDir()) tCommand = "ln -s " + self.appContext.getConfiguration().getArchiveDir() + os.sep +\ self.appContext.getConfiguration().getSummaryContentFile() + ".gz " +\ self.appContext.getConfiguration().getSummaryContentFile() + ".gz" os.system(tCommand) if os.path.isdir(self.appContext.getConfiguration().getDumpDir()): os.system("rm -r " + self.appContext.getConfiguration().getDumpDir())
def moveContentFile(self): if not self.fileClosed is True: self.closeWriteContext() numberProcessedRecords = self.appContext.getResultCollector().getIncrementProcessedRecordNoFurtherDetails() maxDocuments = self.appContext.getConfiguration().getMaxDocuments() blocked = self.appContext.getConfiguration().getBlocked() #in case the configuration is set to blocked a if (not maxDocuments is None and not numberProcessedRecords is None and int(numberProcessedRecords) > int(maxDocuments)) \ or blocked: os.chdir(self.appContext.getConfiguration().getDumpDir()) #collected content shouldn't be sent to CBS because number of collected records is too large os.system("mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveNotSent()) tCommand = "gzip -9 " + self.appContext.getConfiguration().getArchiveNotSent() + os.sep + self.appContext.getConfiguration().getSummaryContentFile() os.system(tCommand) if not blocked: self.appContext.getConfiguration().setBlocked('true') os.chdir(self.appContext.getConfiguration().getResultDir()) SwissbibUtilities.sendNotificationMail(receivers=self.appContext.getConfiguration().getEMailNotifification(), network=self.appContext.getConfiguration().getNetworkPrefix(), numberDocuments=numberProcessedRecords, mailserver=self.appContext.getConfiguration().getMailServer()) else: os.chdir(self.appContext.getConfiguration().getDumpDir()) os.system("mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveDir()) tCommand = "gzip -9 " + self.appContext.getConfiguration().getArchiveDir() + os.sep + self.appContext.getConfiguration().getSummaryContentFile() os.system(tCommand) os.chdir(self.appContext.getConfiguration().getResultDir()) tCommand = "ln -s " + self.appContext.getConfiguration().getArchiveDir() + os.sep +\ self.appContext.getConfiguration().getSummaryContentFile() + ".gz " +\ self.appContext.getConfiguration().getSummaryContentFile() + ".gz" os.system(tCommand) if os.path.isdir(self.appContext.getConfiguration().getDumpDir()): os.system("rm -r " + self.appContext.getConfiguration().getDumpDir())
client.initialize() client.lookUpContent() client.preProcessContent() client.process() client.postProcessContent() except Exception as exception: if not appContext is None and not appContext.getWriteContext() is None: procMess=["Exception in FileProcessorImpl.py"] if not appContext.getConfiguration() is None: procMess = SwissbibUtilities.addBlockedMessageToLogSummary(procMess,appContext.getConfiguration()) appContext.getWriteContext().handleOperationAfterError(exType=exception, message="\n".join(procMess) ) elif not appContext is None and not appContext.getConfiguration() is None: logfile = open(appContext.getConfiguration().getErrorLogDir() + os.sep + appContext.getConfiguration().getErrorLogFile(),"a") message = ["no WriteContext after Error: Exception Handler", str(exception)] message = SwissbibUtilities.addBlockedMessageToLogSummary(message,appContext.getConfiguration()) logfile.write("\n".join(message)) logfile.flush() logfile.close() else: print "no WriteContext after Error and Configuration is None: Exception Handler"
print "redirect error message to stdout\n" print str(exceptionType) + "\n" try: oParser = ArgumentParser() oParser.add_argument("-c", "--config", dest="confFile") args = oParser.parse_args() sConfigs = HarvestingConfigs(args.confFile) sConfigs.setApplicationDir(os.getcwd()) startTime = datetime.now() sU = SwissbibUtilities() sU.initializeDirectoriesForHarvesting(sConfigs) #nextTimestampUTC = sU.getNextTimestamp(sConfigs) rCollector = ResultCollector() if not sConfigs.getResumptionToken() is None: resumptionToken = sConfigs.getResumptionToken() else: fromDate = sU.getFromFormat(sConfigs.getTimestampUTC(), sConfigs) if not sConfigs.getManualUntil() is None: untilDate = sU.getUntilDate(sConfigs.getManualUntil(), sConfigs) appContext = ApplicationContext() appContext.setResultCollector(rCollector) appContext.setConfiguration(sConfigs)
try: oParser = ArgumentParser() oParser.add_argument("-c", "--config", dest="confFile") args = oParser.parse_args() sConfigs = HarvestingConfigs(args.confFile) sConfigs.setApplicationDir(os.getcwd()) startTime = datetime.now() sU = SwissbibUtilities() sU.initializeDirectoriesForHarvesting(sConfigs) rCollector = ResultCollector() appContext = ApplicationContext() appContext.setResultCollector(rCollector) appContext.setConfiguration(sConfigs) mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext) appContext.setMongoWrapper(mongoWrapper) writeContext = HarvestingWriteContext(appContext) appContext.setWriteContext(writeContext)
client = globals()[sConfigs.getFileProcessorType()](appContext) client.initialize() client.lookUpContent() client.preProcessContent() client.process() client.postProcessContent() except Exception as exception: if not appContext is None and not appContext.getWriteContext() is None: procMess = ["Exception in FileProcessorImpl.py"] if not appContext.getConfiguration() is None: procMess = SwissbibUtilities.addBlockedMessageToLogSummary( procMess, appContext.getConfiguration()) appContext.getWriteContext().handleOperationAfterError( exType=exception, message="\n".join(procMess)) elif not appContext is None and not appContext.getConfiguration( ) is None: logfile = open( appContext.getConfiguration().getErrorLogDir() + os.sep + appContext.getConfiguration().getErrorLogFile(), "a") message = [ "no WriteContext after Error: Exception Handler", str(exception) ] message = SwissbibUtilities.addBlockedMessageToLogSummary( message, appContext.getConfiguration())