#url reqest Beipiel
#http://services.dnb.de/oai/repository?set=authorities&verb=ListRecords&from=2012-12-01&metadataPrefix=MARC21-xml
#Beispiel fuer einen record mit GNDS (docid: 020425325)
#Beispiel OAI GetRecord
#http://services.dnb.de/oai/repository?verb=GetRecord&metadataPrefix=MARC21-xml&identifier=oai:d-nb.de/authorities/000158976
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
oParser.add_argument("-i", "--input", dest="inputFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
inputFile = args.inputFile
rCollector = ResultCollector()
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
recordLines = []
start = False
for line in open(inputFile,"r"):
if line.find('<record') != -1:
recordLines.append(line)
start = True
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
#nextTimestampUTC = sU.getNextTimestamp(sConfigs)
rCollector = ResultCollector()
if not sConfigs.getResumptionToken() is None:
resumptionToken = sConfigs.getResumptionToken()
else:
fromDate = sU.getFromFormat(sConfigs.getTimestampUTC(), sConfigs)
if not sConfigs.getManualUntil() is None:
untilDate = sU.getUntilDate(sConfigs.getManualUntil(), sConfigs)
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
#registry = SwissBibMetaDataRegistry(sConfigs,mongoWrapper,rCollector)
registry = SwissBibMetaDataRegistry()
writeContext = HarvestingWriteContext(appContext)
#Todo: SwissbibUtilities sollten Teil des Contexts werden??
appContext.setWriteContext(writeContext)
try:
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
oParser.add_argument("-s", "--size", dest="fileSize",default=None)
oParser.add_argument("-o", "--outDir", dest="outDir",default=None)
args = oParser.parse_args()
sConfigs = HarvestingReadConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
appContext = ApplicationContext()
appContext.setConfiguration(sConfigs)
search502Wrapper = MongoDBHarvestingWrapperSearch502(applicationContext=appContext)
search502Wrapper.read502Records(outDir=args.outDir,fileSize=args.fileSize)
except Exception as pythonBaseException:
print str(pythonBaseException)
finally:
#print sys.version_info
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingFilesConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
rCollector = ResultCollector()
startTime = datetime.now()
appContext = ApplicationContext()
appContext.setConfiguration(sConfigs)
appContext.setResultCollector(rCollector)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
client = globals()[sConfigs.getFileProcessorType()](appContext)
client.initialize()
client.lookUpContent()
client.preProcessContent()
client.process()
client.postProcessContent()
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
rCollector = ResultCollector()
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
writeContext = HarvestingWriteContext(appContext)
appContext.setWriteContext(writeContext)
deleteGenerator = globals()[sConfigs.getDeleteMessagesProcessorType()](appContext,writeContext)
oParser = ArgumentParser()
oParser.add_argument("-c", "--config", dest="confFile")
args = oParser.parse_args()
sConfigs = HarvestingConfigs(args.confFile)
sConfigs.setApplicationDir(os.getcwd())
startTime = datetime.now()
sU = SwissbibUtilities()
sU.initializeDirectoriesForHarvesting(sConfigs)
rCollector = ResultCollector()
appContext = ApplicationContext()
appContext.setResultCollector(rCollector)
appContext.setConfiguration(sConfigs)
mongoWrapper = MongoDBHarvestingWrapper(applicationContext=appContext)
appContext.setMongoWrapper(mongoWrapper)
writeContext = HarvestingWriteContext(appContext)
appContext.setWriteContext(writeContext)
deleteGenerator = globals()[sConfigs.getDeleteMessagesProcessorType()](appContext,writeContext)
