def _normalize(self, s):
if self.strict:
return s
else:
s = unicodedata.normalize('NFC', latex_to_unicode(s))
# Note: in theory latex_to_unicode also normalizes
return s if self.case else s.lower()
def _normalize(self, s):
if self.strict:
return s
else:
s = unicodedata.normalize('NFC', latex_to_unicode(s))
# Note: in theory latex_to_unicode also normalizes
return s if self.case else s.lower()
def bibent_to_fmt(bibent, fmt):
citation_key = bibent['ID']
title = bibent['title'].strip("{}")
authors = bibent['author']
# Convert letters accented via LaTeX (e.g., {\'{e}}) to Unicode, so they display in Markdown
authors = latex_to_unicode(authors)
year = None
if 'year' in bibent:
year = bibent['year']
authors = authors.replace("{", "")
authors = authors.replace("}", "")
authros = authors.replace('\n', ' ').replace(
'\r', ''
) # sometimes author names are given on separate lines, which breaks the Markdown formatting
citation_key = citation_key.replace(
"+", "plus"
) # beautiful-jekyll is not that beautiful and doesn't like '+' in footnote names
if fmt == "markdown":
to_fmt = "[^" + citation_key + "]: **" + title + "**, by " + authors
elif fmt == "text":
to_fmt = "[" + citation_key + "] " + title + "; by " + authors
else:
print_error("Unknown format: " + fmt)
raise "Internal error"
venue = bibent_get_venue(bibent)
if venue != None:
if fmt == "markdown":
to_fmt = to_fmt + ", *in " + venue + "*"
elif fmt == "text":
to_fmt = to_fmt + "; in " + venue
else:
print_error("Unknown format: " + fmt)
raise "Internal error"
if year != None:
if fmt == "markdown":
to_fmt = to_fmt + ", " + year
elif fmt == "text":
to_fmt = to_fmt + "; " + year
else:
print_error("Unknown format: " + fmt)
raise "Internal error"
url = bibent_get_url(bibent)
if url is not None:
if fmt == "markdown":
mdurl = "[[URL]](" + url + ")"
to_fmt = to_fmt + ", " + mdurl
elif fmt == "text":
to_fmt = to_fmt + "; " + url
else:
print_error("Unknown format: " + fmt)
raise "Internal error"
return to_fmt
def convert_to_unicode(record):
"""
Convert accent from latex to unicode style.
:param record: the record.
:type record: dict
:returns: dict -- the modified record.
"""
for val in record:
if isinstance(record[val], list):
record[val] = [latex_to_unicode(x) for x in record[val]]
elif isinstance(record[val], dict):
record[val] = {
k: latex_to_unicode(v)
for k, v in record[val].items()
}
else:
record[val] = latex_to_unicode(record[val])
return record
def abbreviate_firstname(name: str, sep: str = " ") -> str:
"""
Abbreviate first name(s) to initials.
For example::
de Geus, Thomas Willem Jan ->
de Geus, T. W. J.
:param name: The name formatted as "Lastname, firstname secondname ...".
:param sep: Separator to place between initials.
:return: Formatted name.
"""
if len(name.split(",")) == 1:
return name
if len(name.split(",")) > 2:
raise OSError(f'Unable to interpret name "{name}"')
match = [
(re.compile(r"(.*)(\(.*\))", re.UNICODE), r"\1"),
(
re.compile(r"([\w][\}]*)([\w0-9\{\}\`\'\"\\\.\^\{]*)", re.UNICODE),
r"\1.",
),
(re.compile(r"([\w\.][\-]?)([\ ]*)", re.UNICODE), r"\1"),
]
last, first = name.split(",")
first = latex_to_unicode(first)
first = first.replace(".", ". ").replace("-", "- ").replace(r"\. ",
r"\.") + " "
names = [
latex_to_unicode(i[0]) for i in re.findall(r"([^\s]*)(\s+)", first)
][1:]
for i in range(len(names)):
for regex, sub in match:
names[i] = re.sub(regex, sub, names[i])
return last + ", " + sep.join([rm_unicode(i) for i in names]).upper()
def convert_to_unicode(record):
"""
Convert accent from latex to unicode style.
:param record: the record.
:type record: dict
:returns: dict -- the modified record.
"""
for val in record:
if isinstance(record[val], list):
record[val] = [
latex_to_unicode(x) for x in record[val]
]
elif isinstance(record[val], dict):
record[val] = {
k: latex_to_unicode(v) for k, v in record[val].items()
}
else:
record[val] = latex_to_unicode(record[val])
return record
def decode(entry):
if isinstance(entry, list):
return [decode(e) for e in entry]
elif isinstance(entry, dict):
return dict((k, decode(v)) for k, v in entry.items())
else:
# TODO: latex_to_unicode sometimes fails with exception. I do
# not understand why, but let's just sweep it under the rug
# for now ok? Great.
try:
return latex_to_unicode(entry)
except Exception:
return entry
def prepare_string(x, max_length=None):
""" Converts a string from LaTeX escapes to UTF8 and truncates it to max_length """
# data = latex2text(x, tolerant_parsing=True)
try:
data = latex_to_unicode(filter_using_re(x))
if max_length is not None:
data = (data[:max_length - 5] +
'[...]') if len(data) > max_length else data
return smart_text(data)
except TypeError:
logger.warning(
"Encountered a TypeError which may be linked to unicode handling "
"in bibtexparser when processing the following string: %s." % x)
return ""
def bibfile_latex_to_unicode(bibtex_fname):
parser = BibTexParser(common_strings=True)
with open(bibtex_fname) as bibtex_file:
bibdb = bibtexparser.load(bibtex_file, parser=parser)
for i, entry in enumerate(bibdb.entries):
delete_field(bibdb, i, 'file')
for field in entry.keys():
bibdb.entries[i][field] = latex_to_unicode(entry[field])
bibdb.comments = []
writer = BibTexWriter()
writer.display_order = ['title', 'year', 'author', 'journal', 'booktitle']
clean_file = writer.write(bibdb)
# Use for debug purposes:
# with open('tmp.bib','w') as f:
# f.write(clean_file)
return clean_file
def latex_to_ascii(tex):
r""" Transforms LaTeX strings to ascii text ignoring accents
Args:
tex (str): LaTeX string
Returns:
str: unicode string containing only ascii characters
Examples:
>>> latex_to_ascii(r"\^ile")
'ile'
>>> latex_to_ascii(r"\^ile") == latex_to_ascii('île')
True
>>> latex_to_ascii(r"Bartoszy\'nski Ros\l anowski")
'Bartoszynski Rosl anowski'
"""
for pat, sub in latex_to_ascii.dict.items():
tex = tex.replace(pat, sub)
uni = latex_to_unicode(tex)
asc = unicodedata.normalize('NFD', uni)
asc = asc.encode('ascii', 'ignore').decode('utf-8')
return asc
def test_ignores_trailing_modifier(self):
string = "a\\\'"
result = latex_to_unicode(string)
expected = 'a'
self.assertEqual(result, expected)
def test_accents(self):
string = "{\`a} {\\\'e} {\`e} {\\\"o}"
result = latex_to_unicode(string)
expected = 'à é è ö'
self.assertEqual(result, expected)
def test_ignores_trailing_modifier(self):
string = "a\\\'"
result = latex_to_unicode(string)
expected = 'a'
self.assertEqual(result, expected)
def test_special_caracter(self):
string = '{\c c}'
result = latex_to_unicode(string)
expected = 'ç'
self.assertEqual(result, expected)
def test_does_not_modify_two_existing_combining(self):
string = b'pho\xcc\x9b\xcc\x89'.decode('utf8')
result = latex_to_unicode(string)
expected = 'phở' # normalized
self.assertEqual(result, expected)
def test_accents(self):
string = "{\`a} {\\\'e} {\`e} {\\\"o}"
result = latex_to_unicode(string)
expected = 'à é è ö'
self.assertEqual(result, expected)
def test_does_not_modify_two_existing_combining(self):
string = b'pho\xcc\x9b\xcc\x89'.decode('utf8')
result = latex_to_unicode(string)
expected = 'phở' # normalized
self.assertEqual(result, expected)
def test_special_caracter(self):
string = '{\c c}'
result = latex_to_unicode(string)
expected = 'ç'
self.assertEqual(result, expected)
def p2b(pmidlist):
''' by Nick Loman '''
if type(pmidlist) != list:
pmidlist = [str(pmidlist)]
## Fetch XML data from Entrez.
efetch = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi'
url = '{}?db=pubmed&id={}&rettype=abstract'.format(efetch,
','.join(pmidlist))
try:
r = requests.get(url)
except:
return []
##print(r.text) # to examine the returned xml
## Loop over the PubMed IDs and parse the XML using https://docs.python.org/2/library/xml.etree.elementtree.html
bibout = []
root = ET.fromstring(r.text)
for PubmedArticle in root.iter('PubmedArticle'):
PMID = PubmedArticle.find('./MedlineCitation/PMID')
ISSN = PubmedArticle.find('./MedlineCitation/Article/Journal/ISSN')
Volume = PubmedArticle.find(
'./MedlineCitation/Article/Journal/JournalIssue/Volume')
Issue = PubmedArticle.find(
'./MedlineCitation/Article/Journal/JournalIssue/Issue')
Year = PubmedArticle.find(
'./MedlineCitation/Article/Journal/JournalIssue/PubDate/Year')
Month = PubmedArticle.find(
'./MedlineCitation/Article/Journal/JournalIssue/PubDate/Month')
Title = PubmedArticle.find('./MedlineCitation/Article/Journal/Title')
ArticleTitle = PubmedArticle.find(
'./MedlineCitation/Article/ArticleTitle')
MedlinePgn = PubmedArticle.find(
'./MedlineCitation/Article/Pagination/MedlinePgn')
Abstract = PubmedArticle.find(
'./MedlineCitation/Article/Abstract/AbstractText')
# jkb additions
PMCID = None
DOI = None
theseids = PubmedArticle.findall(
'./PubmedData/ArticleIdList/ArticleId')
for thisid in theseids:
if thisid.attrib['IdType'] == 'pmc':
PMCID = thisid
elif thisid.attrib['IdType'] == 'doi':
DOI = thisid
# format author list
authors = []
for Author in PubmedArticle.iter('Author'):
try:
LastName = Author.find('LastName').text
ForeName = Author.find('ForeName').text
except AttributeError: # e.g. CollectiveName
continue
authors.append('{}, {}'.format(LastName, ForeName))
## Use InvestigatorList instead of AuthorList
if len(authors) == 0:
## './MedlineCitation/Article/Journal/InvestigatorList'
for Investigator in PubmedArticle.iter('Investigator'):
try:
LastName = Investigator.find('LastName').text
ForeName = Investigator.find('ForeName').text
except AttributeError: # e.g. CollectiveName
continue
authors.append('{}, {}'.format(LastName, ForeName))
if Year is None:
_ = PubmedArticle.find(
'./MedlineCitation/Article/Journal/JournalIssue/PubDate/MedlineDate'
)
Year = _.text[:4]
Month = '{:02d}'.format(
list(calendar.month_abbr).index(_.text[5:8]))
else:
Year = Year.text
if Month is not None:
Month = Month.text
'''
try:
for _ in (PMID.text, Volume.text, Title.text, ArticleTitle.text, MedlinePgn.text, Abstract.text, ''.join(authors)):
if _ is None:
continue
assert '{' not in _, _
assert '}' not in _, _
except AttributeError:
pass
'''
# make the bibtex formatted output.
bib = {}
if len(authors) > 0:
authorname = authors[0].split(',')[0]
else:
authorname = ''
titlewords = [x for x in ArticleTitle.text.split(' ') if len(x) > 3]
if len(titlewords) > 2:
titlestring = ''.join(titlewords[:3])
elif len(titlewords) > 0:
titlestring = ''.join(titlewords)
else:
titlestring = ''
if len(authorname + titlestring) == 0:
titlestring = "PMID{}_".format(PMID.text)
new_id = '{}{}{}'.format(authorname, titlestring, Year)
new_id = re.sub(r'\W+', '', new_id)
bib["ID"] = latexchars.replace_accents(new_id)
bib["Author"] = ' and '.join(authors)
bib["Title"] = ArticleTitle.text
bib["Journal"] = Title.text
bib["Year"] = Year
if Volume is not None:
bib["Volume"] = Volume.text
if Issue is not None:
bib["Number"] = Issue.text
if MedlinePgn is not None:
bib["Pages"] = MedlinePgn.text
if Month is not None:
bib["Month"] = Month
# bib[""] = (' Abstract={{{}}},'.format(Abstract.text))
if PMCID is not None:
bib["pmcid"] = PMCID.text
if DOI is not None:
bib["doi"] = DOI.text
if ISSN is not None:
bib["ISSN"] = ISSN.text
bib["pmid"] = PMID.text
# always return clean latex
bib = {d: latex_to_unicode(bib[d]) for d in bib.keys()}
bibout.append(bib)
return bibout