def openKeyValDb(dbName, newDb=False, singleProcess=False, prefer=None):
" factory function: returns the right db object given a filename "
#return LevelDb(dbName, newDb=newDb)
# possible other candidates: mdb, cdb, hamsterdb
if prefer == "server":
return pubKeyVal.RedisDb(dbName,
newDb=newDb,
singleProcess=singleProcess)
else:
return pubKeyVal.SqliteKvDb(dbName,
newDb=newDb,
singleProcess=singleProcess,
tmpDir=pubConf.getFastTempDir())
def openKeyValDb(dbName, newDb=False, singleProcess=False, prefer=None):
" factory function: returns the right db object given a filename "
#return LevelDb(dbName, newDb=newDb)
# possible other candidates: mdb, cdb, hamsterdb
if prefer=="server":
return pubKeyVal.RedisDb(dbName, newDb=newDb, singleProcess=singleProcess)
else:
return pubKeyVal.SqliteKvDb(dbName, newDb=newDb, singleProcess=singleProcess, tmpDir=pubConf.getFastTempDir())
def getFastUniqueTempFname():
" create unique tempdir on ramdisk, delete on exit "
tempFname = tempfile.mktemp(dir=pubConf.getFastTempDir())
maxCommon.delOnExit(tempFname)
return tempFname
def getFastUniqueTempFname():
" create unique tempdir on ramdisk, delete on exit "
tempFname = tempfile.mktemp(dir=pubConf.getFastTempDir())
maxCommon.delOnExit(tempFname)
return tempFname
def convertOneChunk(zipDir, inIndexFile, inIdFile, outFile):
"""
get files from inIndexFile, parse Xml,
write everything to outfile in ascii format
"""
diskDir = abspath(join(zipDir, "..", "disk"))
store = pubStore.PubWriterFile(outFile)
# read all already done IDs
doneIds = parseDoneIds(inIdFile)
# open output id files
idFname = join(dirname(outFile), basename(outFile).split(".")[0] + ".ids.tab")
logging.debug("Writing ids to %s" % idFname)
idFh = open(idFname, "w")
idFh.write("#articleId\tdoi\tpmid\n")
pmidFinder = pubCompare.PmidFinder()
unzipTmp = pubGeneric.makeTempDir(prefix="pubConvSpringerUnzip", tmpDir=pubConf.getFastTempDir())
maxCommon.delOnExit(unzipTmp)
i = 0
inRows = list(maxCommon.iterTsvRows(inIndexFile))
logging.info("Converting %d files" % len(inRows))
convCount = 0
pdfNotFound = 0
for row in inRows:
# read line
i += 1
articleId = row.articleId
zipFilename, filename = row.zipFilename, row.filename
if u"\xbf" in filename:
logging.info("Found weird character, skipping file")
continue
articleData = pubStore.createEmptyArticleDict(publisher="springer")
if zipFilename == "":
xmlString, pdfString = getDiskData(diskDir, filename)
articleData["origFile"] = filename
else:
xmlString, pdfString = getUpdateData(unzipTmp, zipDir, zipFilename, filename)
articleData["origFile"] = zipFilename + ":" + filename
if pdfString == None:
pdfNotFound += 1
logging.error("Could not open pdf or xml file")
continue
articleId = int(articleId)
# parse xml
logging.debug("Parsing XML")
try:
xmlTree = pubXml.etreeFromXml(xmlString)
except lxml.etree.XMLSyntaxError:
logging.error("XML parse error, skipping file %s, %s" % (zipFilename, filename))
continue
articleData = parseXml(xmlTree, articleData)
if articleData == None:
logging.warn("Parser got no data for %s" % filename)
continue
if articleData["doi"] in doneIds:
logging.error("article %s has already been converted, skipping" % articleData["doi"])
continue
articleData["pmid"] = pmidFinder.lookupPmid(articleData)
articleData["origFile"] = zipFilename + "/" + filename
articleData["externalId"] = articleData["doi"]
# convert pdf to ascii
fileData = createFileData(articleData, "application/pdf", pdfString)
logging.debug("converting pdf to ascii")
pubGeneric.toAscii(fileData, "application/pdf")
# write to output
store.writeArticle(articleId, articleData)
store.writeFile(articleId, (1000 * (articleId)) + 1, fileData, externalId=articleData["externalId"])
# write IDs to separate file
idRow = [str(articleData["articleId"]), articleData["doi"], str(articleData["pmid"])]
idFh.write("\t".join(idRow))
idFh.write("\n")
doneIds.add(articleData["doi"])
convCount += 1
logging.info("Converted %d files, pdfNotFound=%d" % (convCount, pdfNotFound))
store.close()
idFh.close()
def convertOneChunk(zipDir, inIndexFile, inIdFile, outFile):
"""
get files from inIndexFile, parse Xml,
write everything to outfile in ascii format
"""
diskDir = abspath(join(zipDir, "..", "disk"))
store = pubStore.PubWriterFile(outFile)
# read all already done IDs
doneIds = parseDoneIds(inIdFile)
# open output id files
idFname = join(dirname(outFile), basename(outFile).split(".")[0]+".ids.tab")
logging.debug("Writing ids to %s" % idFname)
idFh = open(idFname, "w")
idFh.write("#articleId\tdoi\tpmid\n")
pmidFinder = pubCompare.PmidFinder()
unzipTmp = pubGeneric.makeTempDir(prefix="pubConvSpringerUnzip", tmpDir=pubConf.getFastTempDir())
maxCommon.delOnExit(unzipTmp)
i = 0
inRows = list(maxCommon.iterTsvRows(inIndexFile))
logging.info("Converting %d files" % len(inRows))
convCount = 0
pdfNotFound = 0
for row in inRows:
# read line
i+=1
articleId = row.articleId
zipFilename, filename = row.zipFilename, row.filename
if u'\xbf' in filename:
logging.info("Found weird character, skipping file")
continue
articleData = pubStore.createEmptyArticleDict(publisher="springer")
if zipFilename=="":
xmlString, pdfString = getDiskData(diskDir, filename)
articleData["origFile"] = filename
else:
xmlString, pdfString = getUpdateData(unzipTmp, zipDir, zipFilename, filename)
articleData["origFile"] = zipFilename+":"+filename
if pdfString==None:
pdfNotFound+=1
logging.error("Could not open pdf or xml file")
continue
articleId=int(articleId)
# parse xml
logging.debug("Parsing XML")
try:
xmlTree = pubXml.etreeFromXml(xmlString)
except lxml.etree.XMLSyntaxError:
logging.error("XML parse error, skipping file %s, %s" % (zipFilename, filename))
continue
articleData = parseXml(xmlTree, articleData)
if articleData==None:
logging.warn("Parser got no data for %s" % filename)
continue
if articleData["doi"] in doneIds:
logging.error("article %s has already been converted, skipping" % articleData["doi"])
continue
articleData["pmid"] = pmidFinder.lookupPmid(articleData)
articleData["origFile"]=zipFilename+"/"+filename
articleData["externalId"]=articleData["doi"]
# convert pdf to ascii
fileData = createFileData(articleData, "application/pdf", pdfString)
logging.debug("converting pdf to ascii")
pubGeneric.toAscii(fileData, "application/pdf")
# write to output
store.writeArticle(articleId, articleData)
store.writeFile(articleId, (1000*(articleId))+1, fileData, externalId=articleData["externalId"])
# write IDs to separate file
idRow = [str(articleData["articleId"]), articleData["doi"], str(articleData["pmid"])]
idFh.write("\t".join(idRow))
idFh.write("\n")
doneIds.add(articleData["doi"])
convCount += 1
logging.info("Converted %d files, pdfNotFound=%d" % (convCount, pdfNotFound))
store.close()
idFh.close()