def convert(bibtex_input, bibtex_output=None):
parser = bibtexparser.bparser.BibTexParser(
ignore_nonstandard_types=True,
homogenize_fields=False,
common_strings=True,
)
bibtex_database = parser.parse_file(io.open(bibtex_input))
journal_dict = load_journal_dict()
for entry in bibtex_database.entries:
if 'journal' in entry:
journal_raw = entry['journal']
journal = reduce_name(journal_raw)
if journal.startswith('arxiv'):
pass
elif journal in journal_dict:
entry['journal'] = journal_dict[journal]
elif journal in journal_dict.values():
pass
else:
print(u"Warning: No abbreviation for '{}'".format(journal_raw),
file=sys.stderr)
if bibtex_output:
bibtexparser.dump(bibtex_database, io.open(bibtex_output, 'w'))
else:
bibtexparser.dump(bibtex_database, sys.stdout)
def render_pdf_from_url(bib_database):
for entry in bib_database.entries:
# Wenn es sich nicht um eine Internetquelle handelt
if entry["ENTRYTYPE"] != "misc":
continue
# oder der Link zur pdf unter 'note' nicht bereits gesetzt wurde
if "note" in entry:
continue
# oder es keine url gibt
if "url" not in entry:
continue
print(entry)
pdf_name = entry["ID"].replace(":", "_") + ".pdf"
try:
pdf = pdfkit.from_url(entry["url"], output_path=os.path.join(current_path, pdf_name))
note = "\href{run:" + rel_pdf_path + pdf_name + "}{export.pdf}"
entry["note"] = note
except Exception as e:
print(e)
entry["urldate"] = datetime.now().strftime(date_format)
print(entry)
print("\n")
# Neue.bib Datei Speichern
with open(os.path.join(current_path, "test.bib"), 'w') as bibtex_file:
bibtexparser.dump(bib_database, bibtex_file)
def main():
parser = bibtexparser.bparser.BibTexParser(interpolate_strings=False,
homogenize_fields=True)
with open('bibtex.bib') as f:
bib = bibtexparser.load(f, parser=parser)
for entry in bib.entries:
if entry['ENTRYTYPE'] == 'article':
publisher = entry['journal']
elif entry['ENTRYTYPE'] == 'inproceedings':
publisher = entry['booktitle']
elif entry['ENTRYTYPE'] == 'misc':
assert 'howpublished' in entry
else:
raise ValueError('unexpected ENTRYTYPE: {}'.format(
entry['ENTRYTYPE']))
if publisher in list(bib.strings):
assert entry['ID'].split(':')[0] == entry['author'].split(',')[0]
assert entry['ID'].split(':')[-1] == \
publisher.upper() + entry['year']
bib.entries = sorted(bib.entries, key=lambda x: x['ID'])
bib.strings = collections.OrderedDict(sorted(bib.strings.items()))
writer = bibtexparser.bwriter.BibTexWriter()
writer.add_trailing_comma = True
with open('bibtex.bib', 'w') as f:
bibtexparser.dump(bib, f, writer=writer)
def set_file_link_name(database):
for entry in database.entries:
# print(entry)
if "note" in entry:
note = entry["note"]
try:
filename = note.split(rel_pdf_path)[1].split("}")[0]
filename = filename.replace("_", " ")
note = note.replace("export.pdf", filename)
entry["note"] = note
except Exception as e:
print(e)
print(entry)
else:
pass
print("**********************************")
print(entry)
print("**********************************")
print(database.entries)
# Neue.bib Datei Speichern
with open(os.path.join(current_path, "test.bib"), 'w', encoding="utf8") as bibtex_file:
bibtexparser.dump(database, bibtex_file)
def main(argv):
parser = argparse.ArgumentParser(description="%prog [options] -f filein -o fileout -m mode ",formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-f", "--filein", action="store", type=str,
dest="fin", help="in file name,.gz would gzip, none|stdin|- means stdin", metavar="<file>")
parser.add_argument("-o", "--fileout", action="store", type=str,
dest="fout", help="out file name,.gz would gzip, none|stdout|- means stdout", metavar="<string>")
parser.add_argument("-m", "--mode", action="store", type=str,
dest="mode", help="abbr(get Journal Abbreviation),fig get 1st figure from PMC", metavar="<string>",default="abbr,fig")
opt = parser.parse_args()
if len(argv) < 2:
parser.print_help()
sys.exit(0)
ffin = checkstdinout(opt.fin,"in","--filein")
if os.path.isfile(opt.fin) and opt.fout == None:
opt.fout = ".".join(opt.fin.split(".")[:-1] + [ "clean", "bib"])
ffout = checkstdinout(opt.fout,"out","--fileout")
modes = opt.mode.split(",")
bibtex_database = bibtexparser.bparser.BibTexParser(common_strings=True).parse_file(ffin)
for entry in bibtex_database.entries:
if "abbr" in modes:
abbr = journal2abbr_jabref(entry['journal'])
entry['abbr'] = abbr
sys.stderr.write (" ".join(["Abbr :", entry['abbr'], "##", entry['journal'], "##", entry['year']]) + "\n")
if "fig" in modes:
if 'pmcid' in entry:
entry = getPMCFig1(entry)
# print (entry.journal, entry.year)
bibtexparser.dump(bibtex_database,ffout)
def replace_pubmed_pmc(filename, new_filename=None, remove_links=False):
if new_filename is None:
new_filename = filename
with open(filename) as f:
bibtex = bibtexparser.load(f)
counter = 0
for entry in bibtex.entries:
logger.info('Processing entry with title: ' +
entry.get('title', '<Unknown>'))
if entry.get('link',
'').startswith('http://www.ncbi.nlm.nih.gov/pubmed/'):
pmid = entry['link'][35:]
logger.debug('Pubmed ID is %s', pmid)
url = find_pmc_pdf(pmid)
if url is not None:
entry['link'] = url
counter += 1
logger.info('Found a PDF on PubMedCentral: %s', url)
elif remove_links and 'doi' in entry:
# Store the pubmed id but remove the link
entry['pmid'] = str(pmid)
del entry['link']
if 'link' in entry:
entry['url'] = entry['link']
del entry['link']
logger.info('Found entries for {counter}/{all} '
'entries'.format(counter=counter, all=len(bibtex.entries)))
with open(new_filename, 'w') as f:
bwriter = bibtexparser.bwriter.BibTexWriter()
bwriter.order_entries_by = ('year', 'month')
bibtexparser.dump(bibtex, f, bwriter)
def main():
re_year = re.compile('\d{4}')
for bib in bibs:
bibfile = bib
print('=== Updating bibs available in %s ===' % (bibfile))
f = open(bibfile)
db = bibparser.load(f)
count = 0
years = []
for entry in db.entries:
try:
year = entry['year']
years.append(int(year))
pass
except:
count += 1
year = re_year.search(title_years[entry['title']])
if year == None:
print('%d: %s -> %s' % (count, entry['title'], 'none'))
else:
print('%d: %s -> %s' %
(count, entry['title'], year.group(0)))
entry['year'] = str(year.group(0))
if count > 0:
new_f = open(bibfile, 'w')
bibparser.dump(db, new_f)
print(sorted(years))
def write_bib_file(bib_database: bp.bibdatabase.BibDatabase,
file_path='new_reference.bib',
verbose=True):
with open(file_path, 'w') as bib_file:
bp.dump(bib_database, bib_file)
if verbose:
print('Write bib database into %s.' % file_path)
def convert(bibtex_input, bibtex_output=None):
parser = bibtexparser.bparser.BibTexParser(
ignore_nonstandard_types=True,
homogenize_fields=False,
common_strings=True,
)
bibtex_database = parser.parse_file(io.open(bibtex_input))
for entry in bibtex_database.entries:
if 'author' in entry:
authors = entry['author'].split(' and ')
for i, _ in enumerate(authors):
fulname = authors[i].strip().split(',')
surname = fulname[0].strip()
if surname.startswith('{'):
pass
else:
surname = "{%s}" % surname
fulname[0] = surname
authors[i] = ",".join(fulname)
entry['author'] = " and ".join(authors)
# entry['language'] = '{en}'
entry.pop('language', None)
if bibtex_output:
bibtexparser.dump(bibtex_database, io.open(bibtex_output, 'w'))
else:
bibtexparser.dump(bibtex_database, sys.stdout)
def write_bibfile(filename, bib_db, inplace):
'''write bib_db to a file. If inplace is True then the file given will be
overwritten.
:filename: string path to new or existing file
:bib_db: BibTexDatabase containing all the bib entries
:inplace: bool, if True then file 'filename' is overwritten. If False then
the original file is not altered and a new file 'filename.fmt.bib' is
created.
:returns: OrderedDict containing sorted bib entries
'''
writer = btp.bwriter.BibTexWriter()
writer.indent = ''
writer.order_entries_by = False
if not inplace:
filename = filename.replace('.bib', '.fmt.bib')
with open(filename, 'w') as outfile:
btp.dump(
bib_database=bib_db,
bibtex_file=outfile,
writer=writer,
)
return
def cli(ctx, db, fuzzy, no_abbr, input, output):
config = Config(db, fuzzy)
with open(input, 'r', encoding='utf-8') as fp:
bib = bibtexparser.load(fp)
failed = False
for entry in bib.entries:
filter_entry(config, entry)
if no_abbr:
continue
try:
journal_name = entry['journal']
except KeyError:
continue
else:
abbr = config.lookup(journal_name)
if abbr:
entry['journal'] = abbr
else:
failed = True
with open(output, 'w', encoding='utf-8') as fp:
bibtexparser.dump(bib, fp)
if failed:
ctx.exit(1)
def main(bibtex_path: str):
"""
Entrypoint for migrate.py
:params bibtex_path: The PATH to the BibTex (.bib) file on the local machine.
"""
# Load BibTex file and parse it
bibtex_db = load_bibtex_file(
file_path=bibtex_path)
# Obtain every entry key name
bibtex_keys = bibtex_db.entries_dict.keys()
# For all entires in the BibTex file. Remove or replace the "{", "}", "\\textbar" and "\\&" characters in the title field
for key in bibtex_keys:
bibtex_entry = bibtex_db.entries_dict[key]
print(f"Entry before update: {bibtex_entry}")
# Update title
print(f"Old title: {bibtex_entry['title']}\nUpdating title.")
new_title = bibtex_entry["title"].replace("{", "").replace(
"}", "").replace("\\textbar", "|").replace("\\&", "&")
bibtex_entry["title"] = new_title
print(f"New title: {bibtex_entry['title']}")
try:
# Make sure booktitle doesn't contain "{", "}" characters
new_book_title = bibtex_entry["booktitle"].replace("{", "").replace(
"}", "")
bibtex_entry["booktitle"] = new_book_title
except KeyError:
print(
f"BibTex entry: {bibtex_entry['title']} isn't a book. Continuing...")
continue
# Update the top level dictionary
bibtex_db.entries_dict[key].update(bibtex_entry)
print(f"After update: {bibtex_db.entries_dict[key]}")
# Write out new BibTex file
with open("bibtex.bib", "w", encoding="utf-8") as bibtex_file:
bibtexparser.dump(bibtex_db, bibtex_file)
def getcitation():
articlesparser = BibTexParser(common_strings=False)
articlesparser.ignore_nonstandard_types = False
with open('/home/limingtao/ircre-bibtex/ircreupdate/articles.bib', encoding='utf8') as articlesfile:
articles_database = bibtexparser.load(articlesfile, articlesparser)
articleentries = articles_database.entries
import random
samplelist = random.sample(range(len(articleentries)), 20)
print(samplelist)
for i in samplelist:
print("---------------------------")
print("Entry number: " + str(i))
title = articleentries[i]['title']
clusterid = articleentries[i]['clusterid']
print("Title: " + title)
print("Cluster ID: " + clusterid)
if not clusterid == "unknown":
print(str(i))
try:
citations = os.popen(
'''/usr/bin/python3 /home/limingtao/ircre-bibtex/ircreupdate/scholarpy/scholar.py -c 1 -C ''' + clusterid + ''' |grep -v list |grep Citations''').read().strip().split()[
-1]
except:
citations = "unknown"
else:
citations = "unknown"
print("new Citations: " + citations)
if 'cited' in articleentries[i]:
oldcitednumber = int(articleentries[i]['cited'])
else:
oldcitednumber = 0
print("Old Cited Number: " + str(oldcitednumber))
if not citations == "unknown":
citednumber = int(citations)
if citednumber > oldcitednumber and ((citednumber - oldcitednumber) < 8):
articleentries[i]['cited'] = str(citednumber)
writer = BibTexWriter()
writer.indent = ' '
writer.order_entries_by = ('order',)
with open('/home/limingtao/ircre-bibtex/ircreupdate/cited-add-articles.bib', 'w', encoding='utf8') as newarticlefile:
bibtexparser.dump(articles_database, newarticlefile, writer=writer)
os.popen("cp /home/limingtao/ircre-bibtex/ircreupdate/cited-add-articles.bib tempcited-add-articles.bib")
os.popen("cp /home/limingtao/ircre-bibtex/ircreupdate/articles.bib /home/limingtao/ircre-bibtex/ircreupdate/oldarticles.bib")
with open('/home/limingtao/ircre-bibtex/ircreupdate/articles.bib', 'w', encoding='utf8') as newarticlefile:
bibtexparser.dump(articles_database, newarticlefile, writer=writer)
return 0
def extract_bibtex(args):
cited = extract_citation_keys(args.auxfile)
bib_db = load_bibtex(args.bibfile)
entries = bib_db.entries
entries_filtered = list(filter(lambda x: x['ID'] in cited, entries))
bib_db.entries = entries_filtered
with open(args.output, 'w') as output_file:
bibtexparser.dump(bib_db, output_file)
def split(bib):
for e in bib.entries:
db = bibtexparser.bibdatabase.BibDatabase()
db.entries = [e]
path = os.path.join(out, e['ID']) + ".bib"
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w') as h:
bibtexparser.dump(db, h)
def prepend_to_bib(new_entry: str, bibfile: str):
with open(bibfile) as bibtex_file:
bib_database = bibtexparser.load(bibtex_file)
new_database = bibtexparser.loads(new_entry)
bib_database.entries.insert(0, new_database.entries[0])
with open(bibfile, 'w') as bibtex_file:
bibtexparser.dump(bib_database, bibtex_file)
def parseBibTex( self, filename ):
self.txt.insert(END, '--------------------------------------------------------\r\n' )
self.txt.insert( END, 'Loading file ' + filename + '... ' )
with open( filename ) as bibtex_file:
self.txt.insert( END, 'File Loaded.\r\n' )
self.txt.insert( END, 'Parsing file ' + filename + '... ' )
bib_database = bibtexparser.load( bibtex_file )
self.txt.insert( END, 'File parsed.\r\n' )
self.txt.insert( END, 'Found ' + str(len(bib_database.entries)) + ' BibTex entries.\r\n' )
self.txt.insert(END, '--------------------------------------------------------\r\n\r\n' )
self.txt.see( END )
if len(bib_database.entries) == 0:
print('No bibtex entries found in file!')
exit()
attrDict = {'doi':1.0, 'ISSN':1.0, 'ISBN':1.0, 'title':0.9}
dup_dict = dict()
for i in range(0,len( attrDict.keys() )):
self.txt.insert( END, 'Searching for duplicate \'' + attrDict.keys()[i] + '\'s, pthr = ' + str(attrDict[attrDict.keys()[i]]) + '\r\n' )
self.progressThread = LoopingThread( self.progressBarAdvance )
self.progressThread.start()
d = self.attributeMatch( bib_database.entries, attrDict.keys()[i], attrDict[attrDict.keys()[i]] ) # exact match for 'doi'
self.progressThread.stop_loop()
dup_dict = dict( dup_dict.items() + d.items() )
# collect duplicates in separate database file, clean original from duplicates
bib_database_dup = copy( bib_database )
bib_database_cln = copy( bib_database )
bib_database_dup.entries = [ bib_database.entries[i] for i in dup_dict.keys() ]
bib_database_cln.entries = [ bib_database.entries[i] for i in range(0,len(bib_database.entries)) if i not in dup_dict.keys() ]
self.txt.insert( END, '\r\nFinished analyzing!\r\n' )
self.txt.insert(END, '--------------------------------------------------------\r\n' )
self.txt.insert(END, 'Found ' + str( len(bib_database_dup.entries) ) + ' DUPLICATE entries. Saving to ' + filename[0:-4] + '_duplicates.bib.' + '\r\n' )
self.txt.insert(END, '--------------------------------------------------------\r\n' )
filename_dup = filename[0:-4] + '_duplicates.bib'
with open( filename_dup, 'w') as bibtex_file:
bibtexparser.dump(bib_database_dup, bibtex_file)
self.txt.insert(END, '--------------------------------------------------------\r\n' )
self.txt.insert(END, 'Found ' + str( len(bib_database_cln.entries) ) + ' UNIQUE entries. Saving to ' + filename[0:-4] + '_cleaned.bib.' + '\r\n' )
self.txt.insert(END, '--------------------------------------------------------\r\n' )
self.txt.see( END )
filename_cln = filename[0:-4] + '_cleaned.bib'
with open( filename_cln, 'w') as bibtex_file:
bibtexparser.dump(bib_database_cln, bibtex_file)
def write_bibtex(db, filename):
'''
Writes the database into the file named filename
'''
with open(filename, 'w', encoding='utf-8') as bibtex_file:
writer = BibTexWriter()
writer.add_trailing_comma = True
writer.indent = ''
bibtexparser.dump(db, bibtex_file, writer)
def test_write_file(self):
with open(self.input_file_path) as bibtex_file:
bibtex_database = bibtexparser.load(bibtex_file)
with TemporaryFile(mode='w+') as bibtex_out_file:
bibtexparser.dump(bibtex_database, bibtex_out_file)
bibtex_out_file.seek(0)
bibtex_out_str = bibtex_out_file.read()
self.assertEqual(bibtex_out_str, self.expected)
def make_bibs(self, prefix, output):
all_bibs = self.get_all_bibs(prefix)
bib_db = BibDatabase()
bib_db.entries = all_bibs
writer = BibTexWriter()
writer.indent = '\t'
with open(output, 'w') as f:
bibtexparser.dump(bib_db, f, writer)
logging.info('processed %d bib entries', len(all_bibs))
def identify_langs(infile, outfile):
bib_db = bibtexparser.load(infile)
for entry in bib_db.entries:
try:
lang, conf = langid.classify(entry['title'])
except KeyError:
continue
if lang in langmap:
entry['langid'] = langmap[lang]
bibtexparser.dump(bib_db, outfile)
def exportDocs(self, filename="./mybib.bib.bk"):
print("export documents to %s" % filename)
SQL = "SELECT description, bib FROM Document WHERE type != 'topic';"
docsInfo = self._search(SQL)
bibDicList = []
for description, bib in docsInfo:
bibDicList += bibtexparser.loads(bib).entries
bibDicList[-1]['description'] = description
self.db.entries = bibDicList
with open(filename, 'w') as bibtexFp:
bibtexparser.dump(self.db, bibtexFp)
def __main__():
testfile = "5G_sec.json"
with open(testfile, "rt") as f:
# parse json into python objects
data = json.load(f)
bibtex_db = bibtexize(data)
with open("foo.bib", "w") as bf:
bibtexparser.dump(bibtex_db, bf)
def save_to_file(self, bib_file, **parse_settings):
"""
Save this database to a file.
:param bib_file: the path to the file as a string. Note! Will be overwritten.
:return: none
"""
with open(bib_file, 'w') as bib_obj:
for key, value in parse_settings.items():
bib_obj.write('%{}={}\n'.format(key, value))
bib_obj.write('\n')
bibtexparser.dump(self, bib_obj)
def save_bib(commit_message = None, token = None):
with open(repo_path + "/" + repo_name, "w") as bibtex_file:
bibtexparser.dump(bib_database, bibtex_file)
if repo and not no_commit:
msg = commit_message if commit_message else "update"
if tokens:
msg += " (Token %s)" % (token if token else "none")
msg = "[BibTool] %s" % msg
repo.index.add(repo_path + "/" + repo_name)
repo.index.commit(msg)
repo.remotes.origin.push()
def format_bib(path):
'''
Format the given bibliography file.
'''
# read bibliography
with open(path, "r") as f:
db = _fixdb(bp.load(f, _parser()))
# write the bibliography
with open(path, "w") as f:
bp.dump(db, f, _writer())
def strip_and_write(inputfile, outputfile):
bibdata = None
with open(inputfile, 'r') as f:
parser = BibTexParser()
parser.customization = customization
bibdata = bibtexparser.load(f, parser=parser)
if bibdata is None:
sys.exit("Could not load input file {}".format(inputfile))
with open(outputfile, 'w') as f:
bibtexparser.dump(bibdata, f)
def write_bib(self, wpath, pids=None, mode="w"):
entries = []
for num, entry in enumerate(self.entries):
if pids is None:
entries.append(entry)
elif num in pids:
entries.append(entry)
else:
raise AssertionError("")
bibdb = BibDatabase()
bibdb.entries = entries
with open(wpath, mode) as write:
bibtexparser.dump(bibdb, write)
def clear_bib(bibFile, intermDir, poplist, verbose, strsubList):
if verbose:
log.info('Removing following fields from {}:\n{}'
.format(bibFile, ', '.join(poplist)))
if not intermDir.is_dir():
intermDir.mkdir()
try:
sh.copyfile(bibFile, Path(intermDir, bibFile.name + '.bak'))
except Exception as e:
log.warning('Could not create a backup of {}:\n{}'
.format(bibFile, e))
try:
with open(bibFile, 'r') as bf:
bibDB = bibtexparser.load(bf)
except Exception as e:
log.critical('Could not read {}:\n{}'.format(bibFile, e))
return -1
if hasattr(bibDB, 'comments'):
if 'Cleared by NeatLatex' in bibDB.comments:
log.info('The bibtex file appears to be cleaned before. Skipping.')
return
else:
bibDB.comments = ['Cleared by NeatLatex']
else:
log.info('Comments section not available in bibtex. Adding.')
bibDB.comments = ['Cleared by NeatLatex']
for e in bibDB.entries:
for f in poplist:
try:
if e['ENTRYTYPE'] == 'misc' and f == 'url':
continue
e.pop(f)
except:
continue
if len(strsubList) > 0:
for e in bibDB.entries:
if 'url' in e.keys():
for s in strsubList:
e['url'] = e['url'].replace(s[0],s[1])
try:
with open(bibFile,'w') as bf:
bibtexparser.dump(bibDB, bf)
except Exception as e:
log.info('Error occurred while writing {}!\n'.format(bibFile))
return -1
log.info('Bibliography file {} cleaned up.'.format(bibFile))
def write_selected_to_file(selected):
db = BibDatabase()
result = []
for item in selected:
path = str(bib_dir) + str(files[item])
with open(path, 'r') as f:
db = bibtexparser.load(f)
result.append(db.entries[0])
db.entries = result
print db.entries
with open(website_dir, 'w') as f:
bibtexparser.dump(db, f)
subprocess.call(['bib2html', '-f', website_dir])
def writeBibtex(results: list, filename: str):
"""
Exports the list of results to a BibTeX file.
:param results: a list of either SearchResult or Paper objects, with a .bib dict property
:param filename: file to export the bibtex to
"""
db = bibtexparser.bibdatabase.BibDatabase()
for index, result in enumerate(results):
db.entries.append(fixBibData(result.bib, index))
with open(filename, 'w') as bibtex_file:
bibtexparser.dump(db, bibtex_file)
def save(self, localdir=None):
"""
Saves content locally
Parameters
----------
localdir : Path, optional
The local directory for the .bib files. If not given, will use
the default path in potentials/data/bibtex directory.
"""
localfile = self.localfilepath(localdir=localdir)
with open(localfile, 'w', encoding='UTF-8') as f:
bibtexparser.dump(self.__bibdatabase, f)
def ircrebibmerge():
articlesparser = BibTexParser(common_strings=False)
articlesparser.ignore_nonstandard_types = False
with open('/home/limingtao/ircre-bibtex/ircreupdate/sorted-articles.bib', encoding='utf8') as sortedarticle_file:
sortedarticle_database = bibtexparser.load(sortedarticle_file, articlesparser)
sortedarticles = sortedarticle_database.entries.copy()
top15parser = BibTexParser(common_strings=False)
top15parser.ignore_nonstandard_types = False
with open('/home/limingtao/ircre-bibtex/ircreupdate/top15.bib', encoding='utf8') as top15_file:
top15_database = bibtexparser.load(top15_file, top15parser)
top15articles = top15_database.entries.copy()
othersparser = BibTexParser(common_strings = False)
othersparser.ignore_nonstandard_types = False
with open('/home/limingtao/ircre-bibtex/ircreupdate/others.bib', encoding='utf8') as others_file:
others_database = bibtexparser.load(others_file, othersparser)
others = others_database.entries.copy()
alldb = BibDatabase()
entries = []
for i in range(len(top15articles)):
entries.append(top15articles[i].copy())
for i in range(len(sortedarticles)):
entries.append(sortedarticles[i].copy())
for i in range(len(others)):
entries.append(others[i].copy())
alldb.entries = entries
writer = BibTexWriter()
writer.indent = ' '
writer.order_entries_by = None
with open('/home/limingtao/ircre-bibtex/ircreupdate/newircre.bib', 'w', encoding='utf8') as newircrebibfile:
bibtexparser.dump(alldb, newircrebibfile, writer=writer)
return 0
def main():
parser = argparse.ArgumentParser(description="""Reads an input file that is a bibtex flat file containing referencing information
and removes useless fields like. Emits strings to the standard output""")
parser.add_argument('-i', '--inputData', required=True, help='''The file containing elements you want to change.
The input file should just contain data and no column/ row headers etc''')
args = parser.parse_args()
fileString = args.inputData
references = parseBibtexFile(fileString)
fixedRefs = deleteAnote(references)
references.entries = fixedRefs
# The problem is here
f = open('output.bib', 'w')
bibtexparser.dump(bib_database = references, bibtex_file=f)
def write_selected_to_file(self, selected, website):
db = BibDatabase()
result = []
for item in selected:
path = str(self.bib_dir) + str(item)
with open(path, 'r') as f:
db = bibtexparser.load(f)
result.append(db.entries[0])
db.entries = result
if website == 'personal':
with open(self.personal_website_bib, 'w') as f:
bibtexparser.dump(db, f)
elif website == 'group':
with open(self.group_website_bib, 'w') as f:
bibtexparser.dump(db, f)
#Make sure the file is uploaded to Dropbox before it is send to BibBase
time.sleep(1)
#Query to BibBase with the right URL
if website == 'personal':
html = urllib2.urlopen("http://bibbase.org/show?bib=" + str(self.personal_link)).read()
elif website == 'group':
html = urllib2.urlopen("http://bibbase.org/show?bib=" + str(self.group_link)).read()
#The html does not contain styling and jquery or javascript
html = '<head>' + \
'<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.3/jquery.min.js"></script>' + \
'<script src="http://bibbase.org/js/bibbase.min.js" type="text/javascript"></script>' + \
'<link rel="stylesheet" href="http://maxcdn.bootstrapcdn.com/font-awesome/4.3.0/css/font-awesome.min.css">' + \
'<link rel="stylesheet" href="http://bibbase.org/css/bootstrap.min.css" type="text/css" media="screen">' + \
'<link rel="stylesheet" href="http://bibbase.org/css/styles/default.css" type="text/css" media="screen">' + \
'<link rel="stylesheet" href="http://bibbase.org/css/styles/common.css" type="text/css" media="screen">' + \
'<link rel="stylesheet" href="http://bibbase.org/css/styles/hide_text.css" type="text/css" media="screen">' + str(html)
if website == 'personal':
with open(self.personal_website_html, 'w') as website:
website.write(html)
elif website == 'group':
with open(self.group_website_html, 'w') as website:
website.write(html)
def test_unicode_problems(self):
# See #51
bibtex = """
@article{Mesa-Gresa2013,
abstract = {During a 4-week period half the mice (n = 16) were exposed to EE and the other half (n = 16) remained in a standard environment (SE). Aggr. Behav. 9999:XX-XX, 2013. © 2013 Wiley Periodicals, Inc.},
author = {Mesa-Gresa, Patricia and P\'{e}rez-Martinez, Asunci\'{o}n and Redolat, Rosa},
doi = {10.1002/ab.21481},
file = {:Users/jscholz/Documents/mendeley/Mesa-Gresa, P\'{e}rez-Martinez, Redolat - 2013 - Environmental Enrichment Improves Novel Object Recognition and Enhances Agonistic Behavior.pdf:pdf},
issn = {1098-2337},
journal = {Aggressive behavior},
month = "apr",
number = {April},
pages = {269--279},
pmid = {23588702},
title = {{Environmental Enrichment Improves Novel Object Recognition and Enhances Agonistic Behavior in Male Mice.}},
url = {http://www.ncbi.nlm.nih.gov/pubmed/23588702},
volume = {39},
year = {2013}
}
"""
bibdb = bibtexparser.loads(bibtex)
with tempfile.TemporaryFile(mode='w+') as bibtex_file:
bibtexparser.dump(bibdb, bibtex_file)
def save(self, f):
entries = []
for key, citation in self.items():
entry = citation.fields.copy()
entry['ID'] = key
entry['ENTRYTYPE'] = citation.type
entries.append(entry)
db = bp.bibdatabase.BibDatabase()
db.entries = entries
writer = bp.bwriter.BibTexWriter()
writer.order_entries_by = tuple(self.keys())
owned = False
if type(f) is str:
f = open(f, 'w')
owned = True
try:
bp.dump(db, f, writer=writer)
finally:
if owned:
f.close()
def cli(library,outfile,keys=None,aux=None, journal_abbreviations=None,
clean=False, protect_titles=False, backend='natbib'):
parser = BibTexParser(common_strings=True)
db = bibtexparser.load(library, parser=parser)
_keys = []
if keys is not None:
_keys += list(parse_list(keys))
if aux is not None:
_keys += list(parse_aux(aux, backend=backend))
_keys = sorted(set(_keys))
click.echo(" ".join(_keys))
parser = BibTexParser(common_strings=True)
db = bibtexparser.load(library, parser=parser)
# If we don't have any keys, we just keep going with
# all keys
if len(_keys) > 0:
db.entries = [e for e in db.entries
if e['ID'] in _keys]
if journal_abbreviations is not None:
abbreviate_matching = create_abbreviator(journal_abbreviations)
db.entries = [abbreviate_matching(e) for e in db.entries]
if protect_titles:
db.entries = [__protect_titles(e) for e in db.entries]
if clean:
for entry in db.entries:
entry.pop('file', None)
bibtexparser.dump(db, outfile)
'Nat': 'Nature',
'Comms': 'Communications',
'chemical': 'Chemical',
'physics': 'Physics'
}
special = {
'United States of America': 'U. S. A.',
'A European Journal': 'European Journal'
}
def shorten(word):
word, suff = re.findall(r'([^:]+)(.*)', word)[0]
return abbreviate(word) + suff
def process(title):
for full, abbr in special.items():
title = title.replace(full, abbr)
words = [corrections.get(w, w) for w in title.split() if w.lower() not in ignored]
if len(words) > 1:
words = [word if word.endswith('.') else shorten(word) for word in words]
return ' '.join(words)
bib = bibtex.load(sys.stdin)
for item in bib.entries:
if 'journal' in item:
item['journal'] = process(item['journal'])
bibtex.dump(bib, sys.stdout)
changed_list.append("CHANGE STRING {}: {} -> {}".format(key, personal_str, master_str))
personal_db.strings[key] = master_str
else:
new_list.append("ADD STRING {}: {}".format(key, master_str))
personal_db.strings[key] = master_str
#
# user confirmation and write
#
if not new_list and not changed_list:
print("No modifications to be made.")
exit(0)
if new_list:
print("New entries:", *new_list, sep='\n')
if changed_list:
print("Changed entries:", *changed_list, sep='\n')
choice = raw_input("Write changes? [y/n]: ")
if choice == 'y':
# write backup, followed by changes
shutil.copyfile(PERSONAL_FILENAME, PERSONAL_FILENAME_BACKUP)
with open(PERSONAL_FILENAME, 'w') as outfile:
btp.dump(personal_db, outfile)
print("Changes written to: ", PERSONAL_FILENAME)
print("Backup found at: ", PERSONAL_FILENAME_BACKUP)
else:
print("Write aborted.")
def main():
parser = ArgumentParser()
parser.add_argument('files', metavar='TEX', nargs='+', help='tex files to search citation keys')
parser.add_argument('-o', '--output', metavar='BIB', required=True, help='output bibtex file')
parser.add_argument('--no-update', dest='update', action='store_false')
args = parser.parse_args()
keys = search_keys(args.files)
if os.path.isfile(args.output):
with open(args.output) as fp:
bib = bibtexparser.load(fp)
else:
bib = bibtexparser.loads('')
not_found = set()
to_retrieve = set()
all_entries = defaultdict(list)
try:
for key in keys:
if key in bib.entries_dict:
if args.update:
bibcode = extract_bibcode(bib.entries_dict[key])
bibcode_new = entry2bibcode(bib.entries_dict[key])
if bibcode_new:
all_entries[bibcode_new].append(key)
if bibcode_new != bibcode:
to_retrieve.add(bibcode_new)
print '{}: UPDATE => {}'.format(key, bibcode_new)
continue
print '{}: EXISTING'.format(key)
continue
bibcode = find_bibcode(key)
if bibcode:
to_retrieve.add(bibcode)
all_entries[bibcode].append(key)
print '{}: NEW ENTRY => {}'.format(key, bibcode)
else:
not_found.add(key)
print '{}: NOT FOUND'.format(key)
except KeyboardInterrupt:
print
if not_found:
print _headerize('Please check the following keys')
for key in not_found:
print key
repeated_keys = [t for t in all_entries.iteritems() if len(t[1]) > 1]
if repeated_keys:
print _headerize('The following keys refer to the same entry')
for b, k in repeated_keys:
print '{1} has been referred as the following keys; please keep only one:\n{0}\n'.format(' '.join(k), b)
if to_retrieve:
print _headerize('Building new bibtex file, please wait...')
bib_new = bibtexparser.loads(ads.ExportQuery(list(to_retrieve), 'bibtex').execute())
for entry in bib_new.entries:
entry['ID'] = all_entries[entry['ID']][0]
bib = update_bib(bib, bib_new)
with open(args.output, 'w') as fp:
bibtexparser.dump(bib, fp)
print _headerize('Done!')
if(e['type'] == 'misc'):
#If there is a link and no howpublished component then we can edit
if('link' in e and 'howpublished' not in e):
#Remove any back slashes, there is no need to escape characters, they already should be escaped
url = e['link'].replace('\\', '')
url = url.replace('{', '')
url = url.replace('}', '')
howpublished = '\\url{' + url + '}'
e['howpublished'] = howpublished
count += 1
#If there is an author (there always should be) then make sure the capitalisation is preserved for misc entries that are organisations
if('author' in e and ',' not in e['author']):
e['author'] = '{' + e['author'] + '}'
#If there is a URL date then add this as a note so it'll appear in the references
if('urldate' in e):
#We assume the date is in the format yyyy-mm-dd (as used by Mendeley
dateSplit = e['urldate'].split('-')
#Get the components of the date and format them nicely for the output
if(len(dateSplit) == 3):
date = datetime.date(int(dateSplit[0]), int(dateSplit[1]), int(dateSplit[2]))
note = date.strftime('Accessed: %d %b %Y')
e['note'] = note
#Open the output database and write the modified database
with open(output, 'w', encoding="utf-8") as bibtex_output:
bibtexparser.dump(bib_database, bibtex_output)
#Print some info for the user
print(input + ' --> ' + output)
print('Converted ' + str(count) + ' entries')
parser = BibTexParser(common_strings=True)
parser.customization = convert_to_unicode
bib_database = bibtexparser.load(bibtex_file, parser=parser)
#print('<div class="campusviews-timetable">')
zfio = io.BytesIO()
with ZipFile(zfio, mode='w') as zf:
for entry in bib_database.entries:
if 'year' not in entry:
raise Warning("year missing in {}".format(entry['title']))
db = BibDatabase()
db.entries.append(entry)
bibtexio = io.StringIO()
bibtexparser.dump(db, bibtexio)
digest = sha256(bibtexio.getvalue().encode('utf-8')).hexdigest()
filename = digest + '.txt'
zf.writestr(filename, bibtexio.getvalue())
bibtexio.close()
publisher = next(entry[k] for k in ['publisher', 'organization'] if k in entry)
booktitle = next(entry[k] for k in ['booktitle', 'journal'] if k in entry)
url = (f"""<div class="col-md-2">
<a class="link external" href="{entry['url']}">Link</a>
</div>""" if 'url' in entry else '')
pdf = (f"""<div class="col-md-2">
<a class="link download" href="assets/public/files/research/publications/pdf/{entry['pdf']}">PDF</a>
</div>""" if 'pdf' in entry else '')
def main():
parser = argparse.ArgumentParser(
description='Fixes unescaped acronyms in titles in bibtex files '
'by automatically detecting them and optionally parsing '
'them from a specification file.')
parser.add_argument(
'-e', '--encoding',
default='utf-8',
help='Encoding to use for parsing and writing the bib files')
parser.add_argument(
'-i', '--include',
help='A file with strings to additionally escape. '
'Each line marks a single string to protect with curly braces')
parser.add_argument(
'-a', '--abbrev',
action='store_true',
default=False,
help='Abbreviate common conference and journal title parts.')
parser.add_argument(
'-x', '--execute',
type=argparse.FileType(),
help='Specifies a python file with custom transformation to apply to '
'each entry. For this purpose, a single function '
'transform(entry) needs to be defined. This is performed after '
'all other transformation and therefore allows to clean up '
'errors.')
parser.add_argument(
'infile',
metavar='INFILE',
help='The bibtex file to process')
parser.add_argument(
'outfile',
metavar='OUTFILE',
help='The bibtex file to write to or - for stdout.')
args = parser.parse_args()
# prepare -x argument
if args.execute:
exec_globals = {}
eval(compile(args.execute.read(), args.execute.name, 'exec'),
exec_globals)
globals()['transform'] = exec_globals['transform']
parser = bibtexparser.bparser.BibTexParser()
parser.ignore_nonstandard_types = False
parser.homogenise_fields = False
with open(args.infile, encoding=args.encoding) as infile:
database = bibtexparser.load(infile, parser=parser)
if args.include:
with open(args.include, encoding=args.encoding) as include_file:
includes = include_file.readlines()
includes = [s.strip() for s in includes if s.strip()]
for entry in database.entries:
for pattern in includes:
entry['title'] = entry['title'].replace(
pattern, '{' + pattern + '}')
acro_re = re.compile(r'(\w*[A-Z]\w*[A-Z]\w*)')
for entry in database.entries:
entry['title'] = acro_re.sub(r'{\1}', entry['title'])
conf_replacements = [
('Proceedings of the', 'Proc.'),
('Proceedings', 'Proc.'),
('International Conference on', 'Int. Conf.'),
('International', 'Int.'),
('Conference', 'Conf.'),
('Symposium', 'Symp.'),
]
journal_replacements = [
('International', 'Int.'),
]
# abbreviation
if args.abbrev:
for entry in database.entries:
if entry['ENTRYTYPE'] == 'inproceedings':
for source, target in conf_replacements:
entry['booktitle'] = entry['booktitle'].replace(
source, target)
if entry['ENTRYTYPE'] == 'article':
for source, target in journal_replacements:
entry['journal'] = entry['journal'].replace(
source, target)
if args.execute:
for entry in database.entries:
transform(entry)
if args.outfile == '-':
bibtexparser.dump(database, sys.stdout)
else:
with open(args.outfile, 'w', encoding=args.encoding) as outfile:
bibtexparser.dump(database, outfile)
