def postProcessContent(self):
os.chdir(self.context.getConfiguration().getReroWorkingDir())
for fname in glob.glob('*.xml'):
os.system("mv " + fname + " " +
self.context.getConfiguration().getReroSrcDir())
os.chdir(self.context.getConfiguration().getReroSrcDir() )
for fname in glob.glob('*.xml'):
os.system("gzip " + fname )
#check the complete number of processed records
numberProcessedRecords = self.context.getResultCollector().getIncrementProcessedRecordNoFurtherDetails()
maxDocuments = self.context.getConfiguration().getMaxDocuments()
blocked = self.context.getConfiguration().getBlocked()
#in case the configuration is set to blocked a
if (not maxDocuments is None and not numberProcessedRecords is None
and int(numberProcessedRecords) > int(maxDocuments)) \
or blocked:
archivedFiles = self.context.getResultCollector().getCollectedArchiveFiles()
if not archivedFiles is None and len(archivedFiles) > 0:
for filename in archivedFiles.keys():
os.system("unlink " + self.context.getConfiguration().getResultDir() + os.sep + filename)
os.system("mv " + archivedFiles[filename] + os.sep + filename + " " + self.context.getConfiguration().getArchiveNotSent())
self.context.getConfiguration().setBlocked('true')
SwissbibUtilities.sendNotificationMail(receivers=self.context.getConfiguration().getEMailNotifification(),
network=self.context.getConfiguration().getNetworkPrefix(),
numberDocuments=numberProcessedRecords,
mailserver=self.context.getConfiguration().getMailServer())
def moveContentFile(self):
if not self.fileClosed is True:
self.closeWriteContext()
numberProcessedRecords = self.appContext.getResultCollector(
).getIncrementProcessedRecordNoFurtherDetails()
maxDocuments = self.appContext.getConfiguration().getMaxDocuments()
blocked = self.appContext.getConfiguration().getBlocked()
#in case the configuration is set to blocked a
if (not maxDocuments is None and not numberProcessedRecords is None
and int(numberProcessedRecords) > int(maxDocuments)) \
or blocked:
os.chdir(self.appContext.getConfiguration().getDumpDir())
#collected content shouldn't be sent to CBS because number of collected records is too large
os.system(
"mv " +
self.appContext.getConfiguration().getSummaryContentFile() +
" " + self.appContext.getConfiguration().getArchiveNotSent())
tCommand = "gzip -9 " + self.appContext.getConfiguration(
).getArchiveNotSent() + os.sep + self.appContext.getConfiguration(
).getSummaryContentFile()
os.system(tCommand)
if not blocked:
self.appContext.getConfiguration().setBlocked('true')
os.chdir(self.appContext.getConfiguration().getResultDir())
SwissbibUtilities.sendNotificationMail(
receivers=self.appContext.getConfiguration(
).getEMailNotifification(),
network=self.appContext.getConfiguration().getNetworkPrefix(),
numberDocuments=numberProcessedRecords,
mailserver=self.appContext.getConfiguration().getMailServer())
else:
os.chdir(self.appContext.getConfiguration().getDumpDir())
os.system(
"mv " +
self.appContext.getConfiguration().getSummaryContentFile() +
" " + self.appContext.getConfiguration().getArchiveDir())
tCommand = "gzip -9 " + self.appContext.getConfiguration(
).getArchiveDir() + os.sep + self.appContext.getConfiguration(
).getSummaryContentFile()
os.system(tCommand)
os.chdir(self.appContext.getConfiguration().getResultDir())
tCommand = "ln -s " + self.appContext.getConfiguration().getArchiveDir() + os.sep +\
self.appContext.getConfiguration().getSummaryContentFile() + ".gz " +\
self.appContext.getConfiguration().getSummaryContentFile() + ".gz"
os.system(tCommand)
if os.path.isdir(self.appContext.getConfiguration().getDumpDir()):
os.system("rm -r " +
self.appContext.getConfiguration().getDumpDir())
def moveContentFile(self):
if not self.fileClosed is True:
self.closeWriteContext()
numberProcessedRecords = self.appContext.getResultCollector().getIncrementProcessedRecordNoFurtherDetails()
maxDocuments = self.appContext.getConfiguration().getMaxDocuments()
blocked = self.appContext.getConfiguration().getBlocked()
#in case the configuration is set to blocked a
if (not maxDocuments is None and not numberProcessedRecords is None
and int(numberProcessedRecords) > int(maxDocuments)) \
or blocked:
os.chdir(self.appContext.getConfiguration().getDumpDir())
#collected content shouldn't be sent to CBS because number of collected records is too large
os.system("mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveNotSent())
tCommand = "gzip -9 " + self.appContext.getConfiguration().getArchiveNotSent() + os.sep + self.appContext.getConfiguration().getSummaryContentFile()
os.system(tCommand)
if not blocked:
self.appContext.getConfiguration().setBlocked('true')
os.chdir(self.appContext.getConfiguration().getResultDir())
SwissbibUtilities.sendNotificationMail(receivers=self.appContext.getConfiguration().getEMailNotifification(),
network=self.appContext.getConfiguration().getNetworkPrefix(),
numberDocuments=numberProcessedRecords,
mailserver=self.appContext.getConfiguration().getMailServer())
else:
os.chdir(self.appContext.getConfiguration().getDumpDir())
os.system("mv " + self.appContext.getConfiguration().getSummaryContentFile() + " " + self.appContext.getConfiguration().getArchiveDir())
tCommand = "gzip -9 " + self.appContext.getConfiguration().getArchiveDir() + os.sep + self.appContext.getConfiguration().getSummaryContentFile()
os.system(tCommand)
os.chdir(self.appContext.getConfiguration().getResultDir())
tCommand = "ln -s " + self.appContext.getConfiguration().getArchiveDir() + os.sep +\
self.appContext.getConfiguration().getSummaryContentFile() + ".gz " +\
self.appContext.getConfiguration().getSummaryContentFile() + ".gz"
os.system(tCommand)
if os.path.isdir(self.appContext.getConfiguration().getDumpDir()):
os.system("rm -r " + self.appContext.getConfiguration().getDumpDir())
client.initialize()
client.lookUpContent()
client.preProcessContent()
client.process()
client.postProcessContent()
except Exception as exception:
if not appContext is None and not appContext.getWriteContext() is None:
procMess=["Exception in FileProcessorImpl.py"]
if not appContext.getConfiguration() is None:
procMess = SwissbibUtilities.addBlockedMessageToLogSummary(procMess,appContext.getConfiguration())
appContext.getWriteContext().handleOperationAfterError(exType=exception,
message="\n".join(procMess) )
elif not appContext is None and not appContext.getConfiguration() is None:
logfile = open(appContext.getConfiguration().getErrorLogDir() + os.sep + appContext.getConfiguration().getErrorLogFile(),"a")
message = ["no WriteContext after Error: Exception Handler",
str(exception)]
message = SwissbibUtilities.addBlockedMessageToLogSummary(message,appContext.getConfiguration())
logfile.write("\n".join(message))
logfile.flush()
logfile.close()
else:
print "no WriteContext after Error and Configuration is None: Exception Handler"
print "redirect error message to stdout\n"
print str(exceptionType) + "\n"
try:
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
#nextTimestampUTC = sU.getNextTimestamp(sConfigs)
rCollector = ResultCollector()
if not sConfigs.getResumptionToken() is None:
resumptionToken = sConfigs.getResumptionToken()
else:
fromDate = sU.getFromFormat(sConfigs.getTimestampUTC(), sConfigs)
if not sConfigs.getManualUntil() is None:
untilDate = sU.getUntilDate(sConfigs.getManualUntil(), sConfigs)
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
try:
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
rCollector = ResultCollector()
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
writeContext = HarvestingWriteContext(appContext)
appContext.setWriteContext(writeContext)
client = globals()[sConfigs.getFileProcessorType()](appContext)
client.initialize()
client.lookUpContent()
client.preProcessContent()
client.process()
client.postProcessContent()
except Exception as exception:
if not appContext is None and not appContext.getWriteContext() is None:
procMess = ["Exception in FileProcessorImpl.py"]
if not appContext.getConfiguration() is None:
procMess = SwissbibUtilities.addBlockedMessageToLogSummary(
procMess, appContext.getConfiguration())
appContext.getWriteContext().handleOperationAfterError(
exType=exception, message="\n".join(procMess))
elif not appContext is None and not appContext.getConfiguration(
) is None:
logfile = open(
appContext.getConfiguration().getErrorLogDir() + os.sep +
appContext.getConfiguration().getErrorLogFile(), "a")
message = [
"no WriteContext after Error: Exception Handler",
str(exception)
]
message = SwissbibUtilities.addBlockedMessageToLogSummary(
message, appContext.getConfiguration())
try:
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
rCollector = ResultCollector()
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
writeContext = HarvestingWriteContext(appContext)
appContext.setWriteContext(writeContext)
