def on_ready_event(self, document):
document_id = utopia.tools.utils.metadata(document,
'identifiers[utopia]')
if document_id is not None:
kwargs = {'document': document_id, 'context': self._context_}
doi = utopia.tools.utils.metadata(document, 'identifiers[doi]')
if doi is not None:
kwargs['doi'] = doi
annotations = kend.client.Client().annotations(**kwargs)
link = None
if annotations is not None:
for group in annotations:
for ann in group.annotations:
a = kend.converter.Annotation.kend2spineapi(
ann, document)
if a.get(
'author'
) == 'http://utopia.cs.manchester.ac.uk/users/11679':
if a.get('concept') in ("DocumentMetadata",
"AuthorAffiliation",
"DocumentReference"):
if link is None:
link = document.newAccList('metadata', 100)
link[
'property:sourceDatabase'] = 'biochemj'
link[
'property:sourceTitle'] = 'The Semantic Biochemical Journal'
link[
'property:sourceDescription'] = '<p>Made available by <a href="http://www.portlandpress.com/">Portland Press Limited</a> as part of the <a href="http://www.biochemj.org/bj/semantic_faq.htm">Semantic Biochemical Journal</a>.'
# Modify Bibliography Entries
if a.get('concept') == 'Citation':
for keyTo, keyFrom in {
'property:title':
'property:articleTitle',
'property:authors':
'property:articleAuthors',
'property:year':
'property:articleYear',
'property:volume':
'property:articleVolume',
'property:source':
'property:journalTitle',
}.iteritems():
if keyFrom in a:
a[keyTo] = a[keyFrom]
a['property:sourceDatabase'] = 'biochemj'
a['property:sourceDescription'] = 'Thingy'
document.addAnnotation(a, link['scratch'])
else:
document.addAnnotation(a)
else:
document.addAnnotation(a)
def on_ready_event(self, document):
document_id, doi = self._resolve(document)
if document_id is not None:
kwargs = { 'document': document_id, 'context': self._context_ }
if doi is not None:
kwargs['doi'] = doi
annotations = kend.client.Client().annotations(**kwargs)
link = None
if annotations is not None:
for group in annotations:
for ann in group.annotations:
a = kend.converter.Annotation.kend2spineapi(ann, document)
if a.get('author') == 'http://utopia.cs.manchester.ac.uk/users/11679':
if a.get('concept') in ("DocumentMetadata", "AuthorAffiliation", "DocumentReference"):
if link is None:
link = document.newAccList('metadata', 100)
link['property:sourceDatabase'] = 'biochemj'
link['property:sourceTitle'] = 'The Semantic Biochemical Journal'
link['property:sourceDescription'] = '<p>Made available by <a href="http://www.portlandpress.com/">Portland Press Limited</a> as part of the <a href="http://www.biochemj.org/bj/semantic_faq.htm">Semantic Biochemical Journal</a>.'
# Modify Bibliography Entries
if a.get('concept') == 'DocumentReference':
for keyTo, keyFrom in {
'property:title': 'property:articleTitle',
'property:authors': 'property:articleAuthors',
'property:year': 'property:articleYear',
'property:volume': 'property:articleVolume',
'property:source': 'property:journalTitle',
}.iteritems():
if keyFrom in a:
a[keyTo] = a[keyFrom]
a['property:sourceDatabase'] = 'biochemj'
a['property:sourceDescription'] = 'Thingy'
document.addAnnotation(a, link['scratch'])
else:
document.addAnnotation(a)
else:
document.addAnnotation(a)
def on_ready_event(self, document):
doi = utopialib.utils.metadata(document, 'identifiers[doi]')
if doi is not None:
info = {}
# Resolve the DOI to find the publisher's website
response = urllib2.urlopen('http://dx.doi.org/{0}'.format(doi),
timeout=8)
# Parse page to find (if there) the full text URL
parser = etree.HTMLParser()
html = etree.parse(response, parser)
# Only continue if this is a highwire HTML page
if len(html.xpath("/html/head/meta[@name='HW.identifier']")) == 0:
return
# Now make sure we have the full text XHTML
citation_fulltext_html_url = html.xpath(
"/html/head/meta[@name='citation_fulltext_html_url']/@content")
if len(citation_fulltext_html_url) > 0:
citation_fulltext_html_url = citation_fulltext_html_url[0]
# Fetch that full text page (if different to the current one)
if citation_fulltext_html_url != response.geturl():
response = urllib2.urlopen(citation_fulltext_html_url,
timeout=8)
html = etree.parse(response, parser)
#print etree.tostring(html, pretty_print=True, encoding='utf8')
# Now parse out the bibliography
info['citations'] = []
info['citations_by_id'] = {}
for bibitem in html.xpath(
"//div[contains(concat(' ', normalize-space(@class), ' '), ' ref-list ')]//ol[contains(concat(' ', normalize-space(@class), ' '), ' cit-list ')]/li"
):
citation = query(
bibitem, {
'id':
'a/@id',
'label':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' ref-label ')]/text()",
'title':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-article-title ')]/text()",
'year':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-date ')]/text()",
'publication-title':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-jnl-abbrev ')]/text()",
'volume':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-vol ')]/text()",
'issue':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-issue ')]/text()",
'pagefrom':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-fpage ')]/text()",
'pageto':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-lpage ')]/text()",
'pmid':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-id-pmid ')]/text()",
'doi':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-id-doi ')]/text()",
'etree':
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-metadata ')]",
})
authors = []
for a in bibitem.xpath(
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-auth ')]"
):
surname = a.xpath(
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-name-surname ')]/text()"
)
given_names = a.xpath(
".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-name-given-names ')]/text()"
)
if len(surname) > 0 and len(given_names) > 0:
authors.append(u'{0}, {1}'.format(
surname[0], given_names[0]).strip(', '))
if len(authors) > 0:
citation['authors'] = authors
citation['contexts'] = []
citation['displayText'] = utopia.citation.format(citation)
info['citations'].append(citation)
info['citations_by_id'][citation['id']] = citation
#print citation
#######################################################################################
# Parse in-text citations if present
min_length = 10
max_length = 20
for paragraph in html.xpath(
"//div[contains(concat(' ', normalize-space(@class), ' '), ' article ')]//p"
):
text_stack = [paragraph.text or '']
xref_stack = [None]
for elem in paragraph:
if len(elem.xpath('self::a[@class="xref-bibr"]')) > 0:
text_stack.append(
etree.tostring(elem,
method='text',
encoding=unicode,
with_tail=False))
text_stack.append(elem.tail or '')
xref = info['citations_by_id'].get(
elem.get('href', '')[1:])
if xref is not None:
xref_stack += [[xref], None]
else:
xref_stack += [[], None]
elif isinstance(elem, etree._Entity):
points = entities.get(elem.text[1:-1])
if points is not None:
text_stack[-1] += ''.join(
(unichr(p) for p in points))
else:
text_stack[-1] += etree.tostring(
elem, encoding=unicode)
else:
if elem.get('position') == 'float':
text_stack[-1] += elem.tail or ''
else:
text_stack[-1] += etree.tostring(
elem, method='text', encoding=unicode)
# Find and collapse ranges in the text
for i in xrange(len(xref_stack) - 3, 1, -2):
text = text_stack[i].strip()
#print len(text), (text in u'-\u2010\u2011\u2012\u2013\u2014\u2015'), u''.join(text_stack[i-1:i+2]).encode('utf8')
# if this text is a dash, we need to coalesce the text fragments
if len(
text
) == 1 and text in u'-\u2010\u2011\u2012\u2013\u2014\u2015':
text_stack[i - 1:i + 2] = [
u''.join(text_stack[i - 1:i + 2])
]
xref_stack[i - 1:i + 2] = [
xref_stack[i - 1] + xref_stack[i + 1]
]
#for text in text_stack:
# print text.encode('utf8')
# Then make sure we resolve the implied citations
for i in xrange(1, len(xref_stack), 2):
# Get actual cross references
xrefs = xref_stack[i]
# Expand cross references
try:
if len(xrefs) == 2:
labelfrom = int(xrefs[0].get('label'))
labelto = int(xrefs[1].get('label'))
candidates = {}
midlabels = [
unicode(midlabel) for midlabel in xrange(
labelfrom + 1, labelto)
]
for candidate in info['citations']:
if candidate.get('label') in midlabels:
candidates[int(candidate.get(
'label'))] = candidate
xrefs[1:-1] = candidates.values()
except:
raise
# Find and collapse lists in the text
for i in xrange(len(xref_stack) - 3, 1, -2):
text = text_stack[i].strip()
# if this text is a comma, we need to coalesce the text fragments
if len(text) == 1 and text == ',':
text_stack[i - 1:i + 2] = [
u''.join(text_stack[i - 1:i + 2])
]
xref_stack[i - 1:i + 2] = [
xref_stack[i - 1] + xref_stack[i + 1]
]
# Expand citations to include brackets (on both sides)
for i in xrange(len(xref_stack) - 2, 0, -2):
before = text_stack[i - 1].strip()[-1:]
text = text_stack[i].strip()
after = text_stack[i + 1].strip()[:1]
# if this text is a comma, we need to coalesce the text fragments
#print before.encode('utf'), after.encode('utf')
if len(before) > 0 and before in '({[' and len(
after) > 0 and after in ')}]':
text_stack[i - 1] = re.sub(r'[({[](\s*)$', r'\1',
text_stack[i - 1])
text_stack[i + 1] = re.sub(r'^(\s*)[)}\]]', r'\1',
text_stack[i + 1])
text_stack[i] = before + text_stack[i] + after
#print repr(text_stack)
for i in xrange(1, len(xref_stack), 2):
# Get context
before = u' '.join(text_stack[:i]).strip()
label = text_stack[i].strip()
after = u' '.join(text_stack[i + 1:]).strip()
# Strip out extraneous brackets
if len(
xref_stack[i]
) > 1: # Hack to differentiate single / multiple citations
# as multiple numbers tend not to have spaces between them
label = re.sub(
ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,])\s?[({[]?',
r'\1', label)
else:
label = re.sub(
ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,]\s?)[({[]?',
r'\1', label)
# Normalise context
before = re.sub(r'\s+', ' ',
before)[-max_length:].strip()
label = re.sub(r'\s+', ' ', label)
after = re.sub(r'\s+', ' ', after)[:max_length].strip()
#print (before.encode('utf8'), label.encode('utf8'), after.encode('utf8'))
if len(before + after) > min_length:
for xref in xref_stack[i]:
xref['contexts'].append((before, label, after))
#print xref_stack[i]
#######################################################################################
# Parse tables if present
info['tables'] = {}
for table_url in html.xpath(
"//div[contains(concat(' ', normalize-space(@class), ' '), ' table-inline ')]/div[contains(concat(' ', normalize-space(@class), ' '), ' callout ')]//li[1]/a/@href"
):
table_url = urlparse.urljoin(citation_fulltext_html_url,
table_url)
#print table_url
response = urllib2.urlopen(table_url, timeout=8)
table_html = etree.parse(response, parser)
for table_expansion in table_html.xpath(
"//div[contains(concat(' ', normalize-space(@class), ' '), ' table-expansion ')]"
):
id = table_expansion.get('id')
table = {}
table['xml'] = table_expansion.xpath('.//table[1]')[0]
table['caption_raw'] = table_expansion.xpath(
".//span[contains(concat(' ', normalize-space(@class), ' '), ' caption-title ')][1]"
)[0]
if 'caption' not in table and 'caption_raw' in table:
table['caption'] = table['caption_raw']
if 'caption' in table:
table['caption'] = re.sub(
r'\s+', ' ',
etree.tostring(table['caption'],
method='text',
encoding=unicode).strip())
if 'xml' in table:
table['xml'] = etree.tostring(table['xml'],
encoding='utf8')
info['tables'][id] = table
#print table
#print info
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopialib.eutils.efetch(id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid',
'pmc'),
('pii', 'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Generate sensible titles / descriptions / icons?
journalTitle = info.get('publication-title', '')
journalTitleSuffix = ''
publisher = info.get('publisher', 'the publisher')
if len(journalTitle) > 0:
journalTitleSuffix = ' ({0})'.format(journalTitle)
# Create Metadata link annotation
link = document.newAccList('metadata', 90)
link['property:sourceIcon'] = ''
link['property:sourceTitle'] = publisher
link['property:sourceDescription'] = '''
<p>This information was provided by {0}{1}.</p>
'''.format(publisher, journalTitleSuffix)
# Create Metadata annotation
annotation = utopialib.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopialib.utils.citation_to_annotation(
citation)
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
#print (pre, label, post)
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation = utopialib.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(
citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation,
link['scratch'])
except:
raise
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
#print regex
# convert oasis tables
ns = {
'oasis':
'http://docs.oasis-open.org/ns/oasis-exchange/table'
}
xml = etree.fromstring(table['xml'])
if xml.tag == '{{{0}}}table'.format(ns['oasis']):
for tgroup in xml.xpath('//oasis:tgroup',
namespaces=ns):
columns = {}
for colspec in tgroup.xpath('./oasis:colspec',
namespaces=ns):
columns[colspec.get('colname')] = int(
colspec.get('colnum'))
for section in tgroup.xpath(
'./oasis:thead|./oasis:tbody',
namespaces=ns):
isHead = (
section.tag == '{{{0}}}thead'.format(
ns['oasis']))
for row in section.xpath('./oasis:row',
namespaces=ns):
for entry in row.xpath('./oasis:entry',
namespaces=ns):
colname = entry.get('colname')
colst = entry.get('namest')
colend = entry.get('nameend')
if colst is not None and colend is not None:
colspan = columns[
colend] - columns[colst] + 1
else:
colspan = 1
if colspan > 1:
entry.set(
'colspan',
unicode(colspan))
morerows = entry.get('morerows')
if morerows is not None:
rowspan = int(morerows) + 1
else:
rowspan = 1
if rowspan > 1:
entry.set(
'rowspan',
unicode(rowspan))
entry.tag = 'td'
row.tag = 'tr'
if isHead:
section.tag = 'thead'
else:
section.tag = 'tbody'
xml.append(section)
xml.tag = 'table'
#print etree.tostring(xml, pretty_print=True, encoding='utf8')
table['xml'] = etree.tostring(xml,
encoding='utf8')
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
# See if we have any publishers' NLM hosted for this DOI
doi = common.utils.metadata(document, 'doi')
#print '----- DOI', doi
if doi is not None:
info = None
try:
url = 'https://utopia.cs.manchester.ac.uk/ext/hosted/nlm?'
url += urllib.urlencode({'doi': doi.encode('utf8')})
nlm = urllib2.urlopen(url, timeout=8).read()
info = common.nlm.parse(nlm)
except (urllib2.URLError, socket.timeout):
# info will remain None
pass
#print info
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding='utf8')
pmids = dict(((citation['pmid'], citation['id']) for citation in info['citations'] if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(common.eutils.efetch(id=','.join(pmids.keys()), retmode='xml', rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath('PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid', 'pmc'), ('pii', 'pii')):
id = idList.findtext('ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Generate sensible titles / descriptions / icons?
journalTitle = info.get('publication-title', '')
journalTitleSuffix = ''
publisher = info.get('publisher', 'the publisher')
if len(journalTitle) > 0:
journalTitleSuffix = ' ({0})'.format(journalTitle)
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link['property:sourceIcon'] = ''
link['property:sourceTitle'] = publisher
link['property:sourceDescription'] = '''
<p>This information was provided by {0}{1}.</p>
'''.format(publisher, journalTitleSuffix)
# Publisher identity
if doi[:8] in ('10.1104/', '10.1105/'):
annotation = spineapi.Annotation()
annotation['concept'] = 'PublisherIdentity'
logo = utopia.get_plugin_data_as_url('images/aspb_logo.png', 'image/png')
webpageUrl = 'http://www.aspb.org/'
title = publisher
#print '====', publisher, '---', journalTitle, '---', webpageUrl
if doi.startswith('10.1104/'):
logo = utopia.get_plugin_data_as_url('images/pp_logo.png', 'image/png')
title = journalTitle
webpageUrl = 'http://www.plantphysiol.org/'
elif doi.startswith('10.1105/'):
logo = utopia.get_plugin_data_as_url('images/tpc_logo.png', 'image/png')
title = journalTitle
webpageUrl = 'http://www.plantcell.org/'
annotation['property:logo'] = logo
annotation['property:title'] = title
annotation['property:webpageUrl'] = webpageUrl
document.addAnnotation(annotation, 'PublisherMetadata')
link['property:sourceIcon'] = logo
link['property:sourceTitle'] = title
# Create Metadata annotation
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentMetadata'
for k in self.keys:
v = info.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentReference'
for k in self.keys:
v = citation.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation['concept'] = 'ForwardCitation'
annotation['property:state'] = 'found'
if 'title' in citation:
annotation['property:title'] = citation['title']
if 'id' in citation:
annotation['property:bibid'] = citation['id']
if 'doi' in citation and citation['doi'].startswith('10.1371/'):
citation['pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format('info:doi/{0}'.format(citation['doi']))
if 'pmcid' in citation:
citation['pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(citation['pmcid'])
for k in ('displayText', 'label', 'pdf', 'pmid', 'pmc', 'pii', 'doi', 'first_author_surname', 'year', 'journal', 'volume', 'page_from'):
if k in citation:
annotation['property:{0}'.format(k)] = citation[k]
#print annotation.get('property:label'), annotation.get('property:pdf')
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link['scratch'])
#print citation
except:
raise
pass # FIXME
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict = True)
#print regex
# convert oasis tables
ns = {'oasis': 'http://docs.oasis-open.org/ns/oasis-exchange/table'}
xml = etree.fromstring(table['xml'])
if xml.tag == '{{{0}}}table'.format(ns['oasis']):
for tgroup in xml.xpath('//oasis:tgroup', namespaces=ns):
columns = {}
for colspec in tgroup.xpath('./oasis:colspec', namespaces=ns):
columns[colspec.get('colname')] = int(colspec.get('colnum'))
for section in tgroup.xpath('./oasis:thead|./oasis:tbody', namespaces=ns):
isHead = (section.tag == '{{{0}}}thead'.format(ns['oasis']))
for row in section.xpath('./oasis:row', namespaces=ns):
for entry in row.xpath('./oasis:entry', namespaces=ns):
colname = entry.get('colname')
colst = entry.get('namest')
colend = entry.get('nameend')
if colst is not None and colend is not None:
colspan = columns[colend] - columns[colst] + 1
else:
colspan = 1
if colspan > 1:
entry.set('colspan', unicode(colspan))
morerows = entry.get('morerows')
if morerows is not None:
rowspan = int(morerows) + 1
else:
rowspan = 1
if rowspan > 1:
entry.set('rowspan', unicode(rowspan))
entry.tag = 'td'
row.tag = 'tr'
if isHead:
section.tag = 'thead'
else:
section.tag = 'tbody'
xml.append(section)
xml.tag = 'table'
#print etree.tostring(xml, pretty_print=True, encoding='utf8')
table['xml'] = etree.tostring(xml, encoding='utf8')
matches = document.search(regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation['session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def after_ready_event(self, document):
print 'Formatting metadata'
# Find highest matching metadata accumulation list for references
source = None
for accListLink in document.getAccLists('metadata'):
matches = document.annotationsIf({'concept': 'Citation'},
accListLink['scratch'])
if len(matches) > 0:
print 'Selected for [Citation] list %s with rank %s' % (
accListLink['scratch'], repr(accListLink.get('rank', 0)))
source = accListLink
bibliography = list(matches)
bibliography.sort(key=sortfn)
rt = ''
for annotation in bibliography:
citation = utopia.tools.utils.citation_from_annotation(
annotation)
rt += utopia.citation.render(citation, links=True)
if len(bibliography) > 0:
# Create Metadata link annotation
link = document.newAccList('citation_list')
link['property:list_name'] = 'Bibliography'
document.addAnnotations(bibliography, link['scratch'])
if len(rt) > 0:
references = spineapi.Annotation()
references['displayBibliography'] = rt
references['concept'] = 'BibliographyMetadata'
references['property:identifier'] = '#bibliography'
references['property:name'] = 'Bibliography'
references['displayName'] = 'Bibliography'
references['displayRelevance'] = '800'
if accListLink is not None:
for i in ('sourceIcon', 'sourceTitle',
'sourceDescription', 'sourceDatabase'):
k = 'property:{0}'.format(i)
if k in accListLink:
references[k] = accListLink[k]
references[
'property:description'] = 'From ' + accListLink[
'property:sourceTitle']
document.addAnnotation(references)
break
if source is None:
print 'No metadata found'
# Find highest matching metadata accumulation list for in-text citations
for accListLink in document.getAccLists('metadata'):
matches = document.annotationsIf({'concept': 'ForwardCitation'},
accListLink['scratch'])
if len(matches) > 0:
print 'Selected for [ForwardCitation] list %s with rank %s' % (
accListLink['scratch'], repr(accListLink.get('rank', 0)))
document.addAnnotations(matches)
break
# Find highest matching metadata accumulation list for in-text citations
for accListLink in document.getAccLists('metadata'):
matches = document.annotationsIf({'concept': 'Table'},
accListLink['scratch'])
if len(matches) > 0:
print 'Selected for [Table] list %s with rank %s' % (
accListLink['scratch'], repr(accListLink.get('rank', 0)))
document.addAnnotations(matches)
break
metadata = None
if source is not None:
for annotation in document.annotations(source['scratch']):
if annotation.get('concept') == 'DocumentMetadata':
metadata = annotation
if metadata:
metadata['displayName'] = 'Document Information'
metadata['displayRelevance'] = '1000'
document.addAnnotation(metadata, 'Document Metadata')
def on_ready_event(self, document):
info = utopialib.nlm.parse(
utopialib.utils.metadata(document, 'raw_pmc_nlm'))
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopialib.eutils.efetch(id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'),
('pmcid', 'pmc'), ('pii',
'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
# Create Metadata link annotation
link = document.newAccList('metadata', 50)
link['property:sourceDatabase'] = 'pmc'
link['property:sourceTitle'] = 'PubMed Central'
link[
'property:sourceDescription'] = '<p><a href="http://www.ncbi.nlm.nih.gov/pmc/">PubMed Central</a> is the U.S. National Institutes of Health (NIH) digital archive of biomedical and life sciences journal literature.</p>'
# Create Metadata annotation
annotation = utopialib.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopialib.utils.citation_to_annotation(citation)
document.addAnnotation(annotation, link['scratch'])
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = utopialib.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link['scratch'])
except:
raise
# Tables
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
# Only send if the DOI has a Springer prefix
doi = utopialib.utils.metadata(document, 'identifiers[doi]')
if doi is not None and doi[:7] in registrants:
annotation = spineapi.Annotation()
annotation['concept'] = 'PublisherIdentity'
if False and doi.startswith(
'10.1186/'): # This turns out not to be reliable
annotation['property:logo'] = utopia.get_plugin_data_as_url(
'images/gigascience_logo.png', 'image/png')
annotation['property:title'] = 'Giga Science'
annotation[
'property:webpageUrl'] = 'http://www.gigasciencejournal.com/'
else:
annotation['property:logo'] = utopia.get_plugin_data_as_url(
'images/logo.png', 'image/png')
annotation['property:title'] = 'Springer'
annotation['property:webpageUrl'] = 'http://www.springer.com/'
document.addAnnotation(annotation, 'PublisherMetadata')
# Make a request to the utopia ext web service
url = 'https://utopia.cs.manchester.ac.uk/ext/springer/nlm?{0}'
url = url.format(urllib.urlencode({'doi': doi}))
try:
nlm = urllib2.urlopen(url, timeout=8).read()
except (urllib2.URLError, socket.timeout):
return
info = utopialib.nlm.parse(nlm)
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopialib.eutils.efetch(id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid',
'pmc'),
('pii', 'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link['property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/annotation_icon.png', 'image/png')
link['property:sourceTitle'] = 'Springer'
link['property:sourceDescription'] = '''
<p><a href="http://www.springer.com/">Springer</a> publishing company.</p>
'''
# Create Metadata annotation
annotation = utopialib.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopialib.utils.citation_to_annotation(
citation)
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = utopialib.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(
citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation,
link['scratch'])
except:
raise
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
print regex
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def _populate(self, document):
# Start by seeing what is already known about this document
nlm = common.utils.metadata(document, 'raw_pmc_nlm')
doi = common.utils.metadata(document, 'doi')
if nlm is not None:
info = self.JournalXMLParser(nlm)
try:
nlmdoi = info.articleDOI().lower()
except: # FIXME which exception(s)?
nlmdoi = None
print "PMC returned nothing"
if doi != nlmdoi:
print "PMC returned wrong article:", info.articleDOI()
else:
print "PMC returned information about article:", info.articleTitle(
)
link = document.newAccList('metadata')
link['property:sourceDatabase'] = 'pubmed'
link['property:sourceTitle'] = 'PubMed'
link[
'property:sourceDescription'] = '<p><a href="http://www.ncbi.nlm.nih.gov/pubmed/">PubMed</a> comprises more than 21 million citations for biomedical literature from MEDLINE, life science journals, and online books.</p>'
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentMetadata'
# print nlm.articlePublicationDate('epub')
# print nlm.articlePublicationDate('epreprint')
# print nlm.journalISSN('epub')
annotation[
"property:identifier"] = 'info:doi%s' % info.articleDOI()
annotation["property:source"] = 'Publisher/NLM'
annotation["property:curatedBy"] = "PMC"
annotation["property:journalTitle"] = info.journalTitle()
annotation[
"property:journalPublisher"] = info.journalPublisher()
annotation["property:journalISSN"] = info.journalISSN()
annotation["property:articleAuthors"] = info.articleAuthors()
annotation["property:articleTitle"] = info.articleTitle()
annotation["property:articleDOI"] = info.articleDOI()
annotation["property:articlePMID"] = info.articlePMID()
annotation[
"property:articlePublisherID"] = info.articlePublisherID()
annotation[
"property:articlePublicationDate"] = info.articlePublicationDate(
)
annotation["property:articleVolume"] = info.articleVolume()
annotation["property:articleIssue"] = info.articleIssue()
if info.articlePages() is not None:
annotation[
"property:articlePages"] = "%s-%s" % info.articlePages(
)
annotation["property:articleAbstract"] = info.articleAbstract()
annotation["property:articleKeywords"] = info.articleKeywords()
annotation[
"property:articleAbbreviations"] = info.articleAbbreviations(
)
document.addAnnotation(annotation, link['scratch'])
# FIXME: Annotation properties need to be lists
for surname, forename, aff in info.articleAuthorAffiliationList(
):
annotation = spineapi.Annotation()
annotation['concept'] = "AuthorAffiliation"
annotation["property:curatedBy"] = "PMC"
annotation["property:authorSurname"] = surname
annotation["property:authorForename"] = forename
annotation["property:articleAuthor"] = "%s, %s" % (
surname, forename)
annotation["property:affiliation"] = aff
document.addAnnotation(annotation, link['scratch'])
for ref in info.articleReferenceList():
annotation = spineapi.Annotation()
annotation['concept'] = "DocumentReference"
if 'doi' in ref:
annotation["property:doi"] = ref['doi']
if 'pmid' in ref:
annotation["property:pmid"] = ref['pmid']
if 'title' in ref:
annotation["property:title"] = ref['title']
if 'label' in ref:
annotation["property:label"] = ref['label']
if 'authors' in ref:
annotation["property:authors"] = ref['authors']
if 'editors' in ref:
annotation["property:articleEditors"] = ref['editors']
if 'publication-title' in ref:
annotation["property:publication-title"] = ref[
'publication-title']
if 'type' in ref:
annotation["property:publicationType"] = ref['type']
if 'volume' in ref:
annotation["property:volume"] = ref['volume']
if 'issue' in ref:
annotation["property:issue"] = ref['issue']
if 'publisher' in ref:
annotation["property:publisher"] = ref['publisher']
if 'fpage' in ref and 'lpage' in ref:
annotation["property:pages"] = "%s-%s" % (ref['fpage'],
ref['lpage'])
if 'year' in ref:
annotation["property:year"] = ref['year']
document.addAnnotation(annotation, link['scratch'])
def _populate(self, document):
# Start by seeing what is already known about this document
nlm = common.utils.metadata(document, 'raw_pmc_nlm')
doi = common.utils.metadata(document, 'doi')
if nlm is not None:
info = self.JournalXMLParser(nlm)
try:
nlmdoi = info.articleDOI().lower()
except: # FIXME which exception(s)?
nlmdoi = None
print "PMC returned nothing"
if doi != nlmdoi:
print "PMC returned wrong article:", info.articleDOI()
else:
print "PMC returned information about article:", info.articleTitle()
link = document.newAccList('metadata')
link['property:sourceDatabase'] = 'pubmed'
link['property:sourceTitle'] = 'PubMed'
link['property:sourceDescription'] = '<p><a href="http://www.ncbi.nlm.nih.gov/pubmed/">PubMed</a> comprises more than 21 million citations for biomedical literature from MEDLINE, life science journals, and online books.</p>'
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentMetadata'
# print nlm.articlePublicationDate('epub')
# print nlm.articlePublicationDate('epreprint')
# print nlm.journalISSN('epub')
annotation["property:identifier"] = 'info:doi%s' % info.articleDOI()
annotation["property:source"] = 'Publisher/NLM'
annotation["property:curatedBy"] = "PMC"
annotation["property:journalTitle"] = info.journalTitle()
annotation["property:journalPublisher"] = info.journalPublisher()
annotation["property:journalISSN"] = info.journalISSN()
annotation["property:articleAuthors"] = info.articleAuthors()
annotation["property:articleTitle"] = info.articleTitle()
annotation["property:articleDOI"] = info.articleDOI()
annotation["property:articlePMID"] = info.articlePMID()
annotation["property:articlePublisherID"] = info.articlePublisherID()
annotation["property:articlePublicationDate"] = info.articlePublicationDate()
annotation["property:articleVolume"] = info.articleVolume()
annotation["property:articleIssue"] = info.articleIssue()
if info.articlePages() is not None:
annotation["property:articlePages"] = "%s-%s" % info.articlePages()
annotation["property:articleAbstract"] = info.articleAbstract()
annotation["property:articleKeywords"] = info.articleKeywords()
annotation["property:articleAbbreviations"] = info.articleAbbreviations()
document.addAnnotation(annotation, link['scratch'])
# FIXME: Annotation properties need to be lists
for surname, forename, aff in info.articleAuthorAffiliationList():
annotation = spineapi.Annotation()
annotation['concept'] = "AuthorAffiliation"
annotation["property:curatedBy"] = "PMC"
annotation["property:authorSurname"] = surname
annotation["property:authorForename"] = forename
annotation["property:articleAuthor"] = "%s, %s" %(surname, forename)
annotation["property:affiliation"] = aff
document.addAnnotation(annotation, link['scratch'])
for ref in info.articleReferenceList():
annotation = spineapi.Annotation()
annotation['concept'] = "DocumentReference"
if 'doi' in ref:
annotation["property:doi"] = ref['doi']
if 'pmid' in ref:
annotation["property:pmid"] = ref['pmid']
if 'title' in ref:
annotation["property:title"] = ref['title']
if 'label' in ref:
annotation["property:label"] = ref['label']
if 'authors' in ref:
annotation["property:authors"] = ref['authors']
if 'editors' in ref:
annotation["property:articleEditors"] = ref['editors']
if 'publication-title' in ref:
annotation["property:publication-title"] = ref['publication-title']
if 'type' in ref:
annotation["property:publicationType"] = ref['type']
if 'volume' in ref:
annotation["property:volume"] = ref['volume']
if 'issue' in ref:
annotation["property:issue"] = ref['issue']
if 'publisher' in ref:
annotation["property:publisher"] = ref['publisher']
if 'fpage' in ref and 'lpage' in ref:
annotation["property:pages"] = "%s-%s" % (ref['fpage'],ref['lpage'])
if 'year' in ref:
annotation["property:year"] = ref['year']
document.addAnnotation(annotation, link['scratch'])
def on_ready_event(self, document):
volume, page = None, None
# Only send if the DOI has a Portland prefix
doi = common.utils.metadata(document, 'doi')
if doi is not None and doi[:7] in registrants:
crossref_unixref = common.utils.metadata(document, 'raw_crossref_unixref')
if crossref_unixref is not None:
# Parse CrossRef redirect URL
dom = etree.fromstring(crossref_unixref.encode('utf8'))
resource = dom.findtext('doi_record/crossref/journal/journal_article/doi_data/resource')
if resource is not None:
match = self.resourceRegExp.match(resource)
if match is not None:
volume, page = match.groups()
### FIXME What information should be shown? Portland? BJ?
#annotation = spineapi.Annotation()
#annotation['concept'] = 'PublisherIdentity'
#annotation['property:logo'] = utopia.get_plugin_data_as_url('images/logo.png', 'image/png')
#annotation['property:title'] = 'Portland Press Limited'
#annotation['property:webpageUrl'] = 'http://www.portlandpress.com/'
#document.addAnnotation(annotation, 'PublisherMetadata')
# If this document was resolved, off we go to fetch the NLM
if None not in (volume, page):
# Make a request to the utopia ext web service
url = 'https://utopia.cs.manchester.ac.uk/ext/portland/nlm?{0}'
url = url.format(urllib.urlencode({'volume': volume, 'page': page}))
try:
nlm = urllib2.urlopen(url, timeout=8).read()
except:
raise
return
info = common.nlm.parse(nlm)
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding='utf8')
pmids = dict(((citation['pmid'], citation['id']) for citation in info['citations'] if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(common.eutils.efetch(id=','.join(pmids.keys()), retmode='xml', rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath('PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid', 'pmc'), ('pii', 'pii')):
id = idList.findtext('ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link['property:sourceIcon'] = utopia.get_plugin_data_as_url('images/biochemj.png', 'image/png')
link['property:sourceTitle'] = 'Portland'
link['property:sourceDescription'] = '''
<p><a href="http://www.portlandpress.com/">Portland Press Limited</a>.</p>
'''
# Create Metadata annotation
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentMetadata'
for k in self.keys:
v = info.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentReference'
for k in self.keys:
v = citation.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation['concept'] = 'ForwardCitation'
annotation['property:state'] = 'found'
if 'title' in citation:
annotation['property:title'] = citation['title']
if 'id' in citation:
annotation['property:bibid'] = citation['id']
if 'doi' in citation and citation['doi'].startswith('10.1371/'):
citation['pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format('info:doi/{0}'.format(citation['doi']))
if 'pmcid' in citation:
citation['pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(citation['pmcid'])
for k in ('displayText', 'label', 'pdf', 'pmid', 'pmc', 'pii', 'doi', 'first_author_surname', 'year', 'journal', 'volume', 'page_from'):
if k in citation:
annotation['property:{0}'.format(k)] = citation[k]
#print annotation.get('property:label'), annotation.get('property:pdf')
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link['scratch'])
#print citation
except:
raise
pass # FIXME
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict = True)
#print regex
matches = document.search(regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation['session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
# See if we have any publishers' NLM hosted for this DOI
doi = utopialib.utils.metadata(document, 'identifiers[doi]')
#print '----- DOI', doi
if doi is not None:
info = None
try:
url = 'https://utopia.cs.manchester.ac.uk/ext/hosted/nlm?'
url += urllib.urlencode({'doi': doi.encode('utf8')})
nlm = urllib2.urlopen(url, timeout=8).read()
info = utopialib.nlm.parse(nlm)
except (urllib2.URLError, socket.timeout):
# info will remain None
pass
#print info
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopialib.eutils.efetch(id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid',
'pmc'),
('pii', 'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Generate sensible titles / descriptions / icons?
journalTitle = info.get('publication-title', '')
journalTitleSuffix = ''
publisher = info.get('publisher', 'the publisher')
if len(journalTitle) > 0:
journalTitleSuffix = ' ({0})'.format(journalTitle)
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link['property:sourceIcon'] = ''
link['property:sourceTitle'] = publisher
link['property:sourceDescription'] = '''
<p>This information was provided by {0}{1}.</p>
'''.format(publisher, journalTitleSuffix)
# Publisher identity
if doi[:8] in ('10.1104/', '10.1105/'):
annotation = spineapi.Annotation()
annotation['concept'] = 'PublisherIdentity'
logo = utopia.get_plugin_data_as_url(
'images/aspb_logo.png', 'image/png')
webpageUrl = 'http://www.aspb.org/'
title = publisher
#print '====', publisher, '---', journalTitle, '---', webpageUrl
if doi.startswith('10.1104/'):
logo = utopia.get_plugin_data_as_url(
'images/pp_logo.png', 'image/png')
title = journalTitle
webpageUrl = 'http://www.plantphysiol.org/'
elif doi.startswith('10.1105/'):
logo = utopia.get_plugin_data_as_url(
'images/tpc_logo.png', 'image/png')
title = journalTitle
webpageUrl = 'http://www.plantcell.org/'
annotation['property:logo'] = logo
annotation['property:title'] = title
annotation['property:webpageUrl'] = webpageUrl
document.addAnnotation(annotation, 'PublisherMetadata')
link['property:sourceIcon'] = logo
link['property:sourceTitle'] = title
# Create Metadata annotation
annotation = utopialib.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopialib.utils.citation_to_annotation(
citation)
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = utopialib.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(
citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation,
link['scratch'])
except:
raise
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
#print regex
# convert oasis tables
ns = {
'oasis':
'http://docs.oasis-open.org/ns/oasis-exchange/table'
}
xml = etree.fromstring(table['xml'])
if xml.tag == '{{{0}}}table'.format(ns['oasis']):
for tgroup in xml.xpath('//oasis:tgroup',
namespaces=ns):
columns = {}
for colspec in tgroup.xpath('./oasis:colspec',
namespaces=ns):
columns[colspec.get('colname')] = int(
colspec.get('colnum'))
for section in tgroup.xpath(
'./oasis:thead|./oasis:tbody',
namespaces=ns):
isHead = (
section.tag == '{{{0}}}thead'.format(
ns['oasis']))
for row in section.xpath('./oasis:row',
namespaces=ns):
for entry in row.xpath('./oasis:entry',
namespaces=ns):
colname = entry.get('colname')
colst = entry.get('namest')
colend = entry.get('nameend')
if colst is not None and colend is not None:
colspan = columns[
colend] - columns[colst] + 1
else:
colspan = 1
if colspan > 1:
entry.set(
'colspan',
unicode(colspan))
morerows = entry.get('morerows')
if morerows is not None:
rowspan = int(morerows) + 1
else:
rowspan = 1
if rowspan > 1:
entry.set(
'rowspan',
unicode(rowspan))
entry.tag = 'td'
row.tag = 'tr'
if isHead:
section.tag = 'thead'
else:
section.tag = 'tbody'
xml.append(section)
xml.tag = 'table'
#print etree.tostring(xml, pretty_print=True, encoding='utf8')
table['xml'] = etree.tostring(xml,
encoding='utf8')
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
doi = common.utils.metadata(document, 'doi')
if doi is not None:
info = {}
# Resolve the DOI to find the publisher's website
response = urllib2.urlopen('http://dx.doi.org/{0}'.format(doi), timeout=8)
# Parse page to find (if there) the full text URL
parser = etree.HTMLParser()
html = etree.parse(response, parser)
# Only continue if this is a highwire HTML page
if len(html.xpath("/html/head/meta[@name='HW.identifier']")) == 0:
return
# Now make sure we have the full text XHTML
citation_fulltext_html_url = html.xpath("/html/head/meta[@name='citation_fulltext_html_url']/@content")
if len(citation_fulltext_html_url) > 0:
citation_fulltext_html_url = citation_fulltext_html_url[0]
# Fetch that full text page (if different to the current one)
if citation_fulltext_html_url != response.geturl():
response = urllib2.urlopen(citation_fulltext_html_url, timeout=8)
html = etree.parse(response, parser)
#print etree.tostring(html, pretty_print=True, encoding='utf8')
# Now parse out the bibliography
info['citations'] = []
info['citations_by_id'] = {}
for bibitem in html.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' ref-list ')]//ol[contains(concat(' ', normalize-space(@class), ' '), ' cit-list ')]/li"):
citation = query(bibitem, {
'id': 'a/@id',
'label': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' ref-label ')]/text()",
'title': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-article-title ')]/text()",
'year': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-date ')]/text()",
'publication-title': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-jnl-abbrev ')]/text()",
'volume': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-vol ')]/text()",
'issue': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-issue ')]/text()",
'pagefrom': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-fpage ')]/text()",
'pageto': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-lpage ')]/text()",
'pmid': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-id-pmid ')]/text()",
'doi': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-pub-id-doi ')]/text()",
'etree': ".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-metadata ')]",
})
authors = []
for a in bibitem.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-auth ')]"):
surname = a.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-name-surname ')]/text()")
given_names = a.xpath(".//*[contains(concat(' ', normalize-space(@class), ' '), ' cit-name-given-names ')]/text()")
if len(surname) > 0 and len(given_names) > 0:
authors.append(u'{0}, {1}'.format(surname[0], given_names[0]).strip(', '))
if len(authors) > 0:
citation['authors'] = authors
citation['contexts'] = []
citation['displayText'] = common.utils.format_citation(citation)
info['citations'].append(citation)
info['citations_by_id'][citation['id']] = citation
#print citation
#######################################################################################
# Parse in-text citations if present
min_length = 10
max_length = 20
for paragraph in html.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' article ')]//p"):
text_stack = [paragraph.text or '']
xref_stack = [None]
for elem in paragraph:
if len(elem.xpath('self::a[@class="xref-bibr"]')) > 0:
text_stack.append(etree.tostring(elem, method='text', encoding=unicode, with_tail=False))
text_stack.append(elem.tail or '')
xref = info['citations_by_id'].get(elem.get('href', '')[1:])
if xref is not None:
xref_stack += [[xref], None]
else:
xref_stack += [[], None]
elif isinstance(elem, etree._Entity):
points = entities.get(elem.text[1:-1])
if points is not None:
text_stack[-1] += ''.join((unichr(p) for p in points))
else:
text_stack[-1] += etree.tostring(elem, encoding=unicode)
else:
if elem.get('position') == 'float':
text_stack[-1] += elem.tail or ''
else:
text_stack[-1] += etree.tostring(elem, method='text', encoding=unicode)
# Find and collapse ranges in the text
for i in xrange(len(xref_stack) - 3, 1, -2):
text = text_stack[i].strip()
#print len(text), (text in u'-\u2010\u2011\u2012\u2013\u2014\u2015'), u''.join(text_stack[i-1:i+2]).encode('utf8')
# if this text is a dash, we need to coalesce the text fragments
if len(text) == 1 and text in u'-\u2010\u2011\u2012\u2013\u2014\u2015':
text_stack[i-1:i+2] = [u''.join(text_stack[i-1:i+2])]
xref_stack[i-1:i+2] = [xref_stack[i-1] + xref_stack[i+1]]
#for text in text_stack:
# print text.encode('utf8')
# Then make sure we resolve the implied citations
for i in xrange(1, len(xref_stack), 2):
# Get actual cross references
xrefs = xref_stack[i]
# Expand cross references
try:
if len(xrefs) == 2:
labelfrom = int(xrefs[0].get('label'))
labelto = int(xrefs[1].get('label'))
candidates = {}
midlabels = [unicode(midlabel) for midlabel in xrange(labelfrom+1, labelto)]
for candidate in info['citations']:
if candidate.get('label') in midlabels:
candidates[int(candidate.get('label'))] = candidate
xrefs[1:-1] = candidates.values()
except:
raise
# Find and collapse lists in the text
for i in xrange(len(xref_stack) - 3, 1, -2):
text = text_stack[i].strip()
# if this text is a comma, we need to coalesce the text fragments
if len(text) == 1 and text == ',':
text_stack[i-1:i+2] = [u''.join(text_stack[i-1:i+2])]
xref_stack[i-1:i+2] = [xref_stack[i-1] + xref_stack[i+1]]
# Expand citations to include brackets (on both sides)
for i in xrange(len(xref_stack) - 2, 0, -2):
before = text_stack[i-1].strip()[-1:]
text = text_stack[i].strip()
after = text_stack[i+1].strip()[:1]
# if this text is a comma, we need to coalesce the text fragments
#print before.encode('utf'), after.encode('utf')
if len(before) > 0 and before in '({[' and len(after) > 0 and after in ')}]':
text_stack[i-1] = re.sub(r'[({[](\s*)$', r'\1', text_stack[i-1])
text_stack[i+1] = re.sub(r'^(\s*)[)}\]]', r'\1', text_stack[i+1])
text_stack[i] = before + text_stack[i] + after
#print repr(text_stack)
for i in xrange(1, len(xref_stack), 2):
# Get context
before = u' '.join(text_stack[:i]).strip()
label = text_stack[i].strip()
after = u' '.join(text_stack[i+1:]).strip()
# Strip out extraneous brackets
if len(xref_stack[i]) > 1: # Hack to differentiate single / multiple citations
# as multiple numbers tend not to have spaces between them
label = re.sub(ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,])\s?[({[]?', r'\1', label)
else:
label = re.sub(ur'[)}\]]?([-\u2010\u2011\u2012\u2013\u2014\u2015,]\s?)[({[]?', r'\1', label)
# Normalise context
before = re.sub(r'\s+', ' ', before)[-max_length:].strip()
label = re.sub(r'\s+', ' ', label)
after = re.sub(r'\s+', ' ', after)[:max_length].strip()
#print (before.encode('utf8'), label.encode('utf8'), after.encode('utf8'))
if len(before + after) > min_length:
for xref in xref_stack[i]:
xref['contexts'].append((before, label, after))
#print xref_stack[i]
#######################################################################################
# Parse tables if present
info['tables'] = {}
for table_url in html.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' table-inline ')]/div[contains(concat(' ', normalize-space(@class), ' '), ' callout ')]//li[1]/a/@href"):
table_url = urlparse.urljoin(citation_fulltext_html_url, table_url)
#print table_url
response = urllib2.urlopen(table_url, timeout=8)
table_html = etree.parse(response, parser)
for table_expansion in table_html.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' table-expansion ')]"):
id = table_expansion.get('id')
table = {}
table['xml'] = table_expansion.xpath('.//table[1]')[0]
table['caption_raw'] = table_expansion.xpath(".//span[contains(concat(' ', normalize-space(@class), ' '), ' caption-title ')][1]")[0]
if 'caption' not in table and 'caption_raw' in table:
table['caption'] = table['caption_raw']
if 'caption' in table:
table['caption'] = re.sub(r'\s+', ' ', etree.tostring(table['caption'], method='text', encoding=unicode).strip())
if 'xml' in table: table['xml'] = etree.tostring(table['xml'], encoding='utf8')
info['tables'][id] = table
#print table
#print info
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding='utf8')
pmids = dict(((citation['pmid'], citation['id']) for citation in info['citations'] if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(common.eutils.efetch(id=','.join(pmids.keys()), retmode='xml', rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath('PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid', 'pmc'), ('pii', 'pii')):
id = idList.findtext('ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Generate sensible titles / descriptions / icons?
journalTitle = info.get('publication-title', '')
journalTitleSuffix = ''
publisher = info.get('publisher', 'the publisher')
if len(journalTitle) > 0:
journalTitleSuffix = ' ({0})'.format(journalTitle)
# Create Metadata link annotation
link = document.newAccList('metadata', 90)
link['property:sourceIcon'] = ''
link['property:sourceTitle'] = publisher
link['property:sourceDescription'] = '''
<p>This information was provided by {0}{1}.</p>
'''.format(publisher, journalTitleSuffix)
# Create Metadata annotation
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentMetadata'
for k in self.keys:
v = info.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = spineapi.Annotation()
annotation['concept'] = 'DocumentReference'
for k in self.keys:
v = citation.get(k)
if v is not None:
annotation['property:{0}'.format(k)] = v
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
#print (pre, label, post)
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation['concept'] = 'ForwardCitation'
annotation['property:state'] = 'found'
if 'title' in citation:
annotation['property:title'] = citation['title']
if 'id' in citation:
annotation['property:bibid'] = citation['id']
if 'doi' in citation and citation['doi'].startswith('10.1371/'):
citation['pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format('info:doi/{0}'.format(citation['doi']))
if 'pmcid' in citation:
citation['pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(citation['pmcid'])
for k in ('displayText', 'label', 'pdf', 'pmid', 'pmc', 'pii', 'doi', 'first_author_surname', 'year', 'journal', 'volume', 'page_from'):
if k in citation:
annotation['property:{0}'.format(k)] = citation[k]
#print annotation.get('property:label'), annotation.get('property:pdf')
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link['scratch'])
#print citation
except:
raise
pass # FIXME
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict = True)
#print regex
# convert oasis tables
ns = {'oasis': 'http://docs.oasis-open.org/ns/oasis-exchange/table'}
xml = etree.fromstring(table['xml'])
if xml.tag == '{{{0}}}table'.format(ns['oasis']):
for tgroup in xml.xpath('//oasis:tgroup', namespaces=ns):
columns = {}
for colspec in tgroup.xpath('./oasis:colspec', namespaces=ns):
columns[colspec.get('colname')] = int(colspec.get('colnum'))
for section in tgroup.xpath('./oasis:thead|./oasis:tbody', namespaces=ns):
isHead = (section.tag == '{{{0}}}thead'.format(ns['oasis']))
for row in section.xpath('./oasis:row', namespaces=ns):
for entry in row.xpath('./oasis:entry', namespaces=ns):
colname = entry.get('colname')
colst = entry.get('namest')
colend = entry.get('nameend')
if colst is not None and colend is not None:
colspan = columns[colend] - columns[colst] + 1
else:
colspan = 1
if colspan > 1:
entry.set('colspan', unicode(colspan))
morerows = entry.get('morerows')
if morerows is not None:
rowspan = int(morerows) + 1
else:
rowspan = 1
if rowspan > 1:
entry.set('rowspan', unicode(rowspan))
entry.tag = 'td'
row.tag = 'tr'
if isHead:
section.tag = 'thead'
else:
section.tag = 'tbody'
xml.append(section)
xml.tag = 'table'
#print etree.tostring(xml, pretty_print=True, encoding='utf8')
table['xml'] = etree.tostring(xml, encoding='utf8')
matches = document.search(regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation['session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
info = common.nlm.parse(common.utils.metadata(document, "raw_pmc_nlm"))
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding="utf8")
pmids = dict(
(
(citation["pmid"], citation["id"])
for citation in info["citations"]
if "pmid" in citation and "id" in citation
)
)
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
common.eutils.efetch(id=",".join(pmids.keys()), retmode="xml", rettype="abstract"), parser
)
for idList in pubmed_abstracts.xpath("PubmedArticle/PubmedData/ArticleIdList"):
# print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info["citations_by_id"][pmids[pmid]]
for key_name, id_name in (("doi", "doi"), ("pmcid", "pmc"), ("pii", "pii")):
id = idList.findtext('ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
# Create Metadata link annotation
link = document.newAccList("metadata", 50)
link["property:sourceDatabase"] = "pmc"
link["property:sourceTitle"] = "PubMed Central"
link[
"property:sourceDescription"
] = '<p><a href="http://www.ncbi.nlm.nih.gov/pmc/">PubMed Central</a> is the U.S. National Institutes of Health (NIH) digital archive of biomedical and life sciences journal literature.</p>'
# Create Metadata annotation
annotation = spineapi.Annotation()
annotation["concept"] = "DocumentMetadata"
for k in self.keys:
v = info.get(k)
if v is not None:
annotation["property:{0}".format(k)] = v
document.addAnnotation(annotation, link["scratch"])
# Create Bibliography annotations
for citation in info.get("citations", []):
annotation = spineapi.Annotation()
annotation["concept"] = "DocumentReference"
for k in self.keys:
v = citation.get(k)
if v is not None:
annotation["property:{0}".format(k)] = v
document.addAnnotation(annotation, link["scratch"])
# Citations
for citation in info["citations"]:
# Find cross refs
for pre, label, post in citation.get("contexts", []):
matches = document.findInContext(pre, label, post)
# print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation["concept"] = "ForwardCitation"
annotation["property:state"] = "found"
if "title" in citation:
annotation["property:title"] = citation["title"]
if "id" in citation:
annotation["property:bibid"] = citation["id"]
if "doi" in citation and citation["doi"].startswith("10.1371/"):
citation[
"pdf"
] = "http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF".format(
"info:doi/{0}".format(citation["doi"])
)
if "pmcid" in citation:
citation["pdf"] = "http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/".format(
citation["pmcid"]
)
# print citation
for k in self.keys + ("authors", "pdf", "first_author_surname"):
if k in citation:
annotation["property:{0}".format(k)] = citation[k]
# print annotation.get('property:label'), annotation.get('property:pdf')
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link["scratch"])
# print citation
except:
raise
pass # FIXME
# Tables
for id, table in info.get("tables", {}).iteritems():
if "caption" in table and "xml" in table:
regex = fuzz(table["caption"], strict=True)
matches = document.search(regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation["concept"] = "Table"
annotation[
"session:upload_files"
] = "data:application/xml;name=data.xml;base64,%s" % base64.standard_b64encode(table["xml"])
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link["scratch"])
else:
print "*********** failed to match table:", id
def on_ready_event(self, document):
volume, page = None, None
# Only send if the DOI has a Portland prefix
doi = utopialib.utils.metadata(document, 'identifiers[doi]')
if doi is not None and doi[:7] in registrants:
crossref_unixref = utopialib.utils.metadata(
document, 'raw_crossref_unixref')
if crossref_unixref is not None:
# Parse CrossRef redirect URL
dom = etree.fromstring(crossref_unixref.encode('utf8'))
resource = dom.findtext(
'doi_record/crossref/journal/journal_article/doi_data/resource'
)
if resource is not None:
match = self.resourceRegExp.match(resource)
if match is not None:
volume, page = match.groups()
### FIXME What information should be shown? Portland? BJ?
#annotation = spineapi.Annotation()
#annotation['concept'] = 'PublisherIdentity'
#annotation['property:logo'] = utopia.get_plugin_data_as_url('images/logo.png', 'image/png')
#annotation['property:title'] = 'Portland Press Limited'
#annotation['property:webpageUrl'] = 'http://www.portlandpress.com/'
#document.addAnnotation(annotation, 'PublisherMetadata')
# If this document was resolved, off we go to fetch the NLM
if None not in (volume, page):
# Make a request to the utopia ext web service
url = 'https://utopia.cs.manchester.ac.uk/ext/portland/nlm?{0}'
url = url.format(urllib.urlencode({
'volume': volume,
'page': page
}))
try:
nlm = urllib2.urlopen(url, timeout=8).read()
except:
raise
return
info = utopialib.nlm.parse(nlm)
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopialib.eutils.efetch(id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'), ('pmcid',
'pmc'),
('pii', 'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link['property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/biochemj.png', 'image/png')
link['property:sourceTitle'] = 'Portland'
link['property:sourceDescription'] = '''
<p><a href="http://www.portlandpress.com/">Portland Press Limited</a>.</p>
'''
# Create Metadata annotation
annotation = utopialib.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopialib.utils.citation_to_annotation(
citation)
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = utopialib.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(
citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation,
link['scratch'])
except:
raise
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
#print regex
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
doi = utopia.tools.utils.metadata(document, 'identifiers[doi]')
if doi is not None:
match = self.splitRegEx.match(doi)
if match is not None:
articleNumber = match.group('number')
annotation = spineapi.Annotation()
annotation['concept'] = 'PublisherIdentity'
annotation['property:logo'] = utopia.get_plugin_data_as_url(
'images/logo.png', 'image/png')
annotation['property:title'] = 'eLife'
annotation[
'property:webpageUrl'] = 'http://www.elifesciences.org/'
document.addAnnotation(annotation, 'PublisherMetadata')
# Turn all the DOIs that are sub-DOIs of this document into links
regex = r'{0}\.\d+'.format(re.escape(doi))
for match in document.search(regex, spineapi.RegExp):
url = 'http://dx.doi.org/{0}'.format(match.text())
annotation = spineapi.Annotation()
annotation['concept'] = 'Hyperlink'
annotation['property:webpageUrl'] = url
annotation['session:volatile'] = '1'
annotation.addExtent(match)
document.addAnnotation(annotation)
# Try to get the NLM directly from eLife
url = 'http://elife.elifesciences.org/elife-source-xml/10.7554/eLife.{0}'
url = url.format(articleNumber)
try:
nlm = urllib2.urlopen(url, timeout=8).read()
except (urllib2.URLError, socket.timeout):
return
info = utopia.tools.nlm.parse(nlm)
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True,
recover=True,
remove_blank_text=True,
encoding='utf8')
pmids = dict(((citation['pmid'], citation['id'])
for citation in info['citations']
if 'pmid' in citation and 'id' in citation))
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
utopia.tools.eutils.efetch(
id=','.join(pmids.keys()),
retmode='xml',
rettype='abstract'), parser)
for idList in pubmed_abstracts.xpath(
'PubmedArticle/PubmedData/ArticleIdList'):
#print etree.tostring(idList)
pmid = idList.findtext(
'ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info['citations_by_id'][pmids[pmid]]
for key_name, id_name in (('doi', 'doi'),
('pmcid', 'pmc'),
('pii', 'pii')):
id = idList.findtext(
'ArticleId[@IdType="{0}"]'.format(
id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
#print 'KEY', key_name, id
# Create Metadata link annotation
link = document.newAccList('metadata', 100)
link[
'property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/annotation_icon.png', 'image/png')
link['property:sourceTitle'] = 'eLife'
link['property:sourceDescription'] = '''
<p>The <a href="http://www.elifesciences.org/">eLife</a> open access publishing platform.</p>
'''
# Create Metadata annotation
annotation = utopia.tools.utils.citation_to_annotation(
info.get('self', {}), 'DocumentMetadata')
document.addAnnotation(annotation, link['scratch'])
# Create Bibliography annotations
for citation in info.get('citations', []):
annotation = utopia.tools.utils.citation_to_annotation(
citation)
document.addAnnotation(annotation, link['scratch'])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info['citations']:
# Find cross refs
for pre, label, post in citation.get('contexts', []):
matches = document.findInContext(pre, label, post)
#print matches
if len(matches) > 0:
try:
annotation = utopia.tools.utils.citation_to_annotation(
citation, concept='ForwardCitation')
if 'doi' in citation and citation[
'doi'].startswith('10.1371/'):
citation[
'pdf'] = 'http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF'.format(
'info:doi/{0}'.format(
citation['doi']))
if 'pmcid' in citation:
citation[
'pdf'] = 'http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/'.format(
citation['pmcid'])
for match in matches:
annotation.addExtent(match)
document.addAnnotation(
annotation, link['scratch'])
except:
raise
for id, table in info.get('tables', {}).iteritems():
if 'caption' in table and 'xml' in table:
regex = fuzz(table['caption'], strict=True)
print regex
matches = document.search(
regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation['concept'] = 'Table'
annotation[
'session:upload_files'] = 'data:application/xml;name=data.xml;base64,%s' % base64.standard_b64encode(
table['xml'])
annotation['session:volatile'] = '1'
annotation.addExtent(matches[0])
document.addAnnotation(annotation,
link['scratch'])
else:
print '*********** failed to match table:', id
def on_ready_event(self, document):
'''Fetch information from the Lazarus service'''
permission = self.get_config('permission', False)
if permission:
# If an outline already exists, don't make a new one
needs_outline = True
for annotation in document.annotations():
if annotation.get('concept') == 'OutlineItem':
needs_outline = False
break
# The Lazarus server needs to know what this document is
document_id = utopia.tools.utils.metadata(document,
'identifiers[utopia]')
this_doi = utopia.tools.utils.metadata(document,
'identifiers[doi]')
if this_doi is not None:
this_doi = u'doi:' + this_doi
# Speak to server
params = {'fingerprint': document.fingerprints()}
url = '{0}?{1}'.format(laz_docUrl,
urllib.urlencode(params, doseq=True))
response = urllib2.urlopen(url, timeout=60)
if response.getcode() == 204:
request = urllib2.Request(
url,
data=document.data(),
headers={'Content-Type': 'application/pdf'})
response = urllib2.urlopen(request, timeout=60)
#response = open('/Users/dave/Desktop/ananiadou_tibtech06.pdf-response.xml', 'r')
# Create Metadata link annotation
link = document.newAccList('metadata', 50)
link['property:sourceDatabase'] = 'lazarus'
link['property:sourceTitle'] = 'Lazarus'
link['property:sourceDescription'] = self.sourceDescription
link['property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/lazarus-prefs-logo.png', 'image/png')
headers = []
pos = []
refs = []
annotations = []
concepts = {}
hits = []
expression_annotations = []
for kAnnotation in kend.converter.XML.parse(
response, kend.model.Document):
#print kend.converter.XML.serialise(kAnnotation)[0]
try:
annotations.append(
utopia.tools.converters.Annotation.kend2spineapi(
kAnnotation, document))
except:
pass
annotations.sort(key=lambda a: int(a.get('structure:order', 0)))
for sAnnotation in annotations:
if sAnnotation['concept'] == 'structure_element':
role, level = self.getHeaderRole(sAnnotation)
if role is not None and needs_outline:
while len(pos) < level:
pos.append(0)
while len(pos) > level:
pos.pop()
pos[-1] += 1
outline = u'.'.join([unicode(i) for i in pos])
anchor_name = '#lazarus.outline.{0}'.format(outline)
anchor = spineapi.Annotation()
anchor['concept'] = 'Anchor'
anchor['property:anchor'] = anchor_name
anchor.addExtents(sAnnotation.extents())
anchor.addAreas(sAnnotation.areas())
document.addAnnotation(anchor)
header = spineapi.Annotation()
header['concept'] = 'OutlineItem'
header['property:outlinePosition'] = outline
header['property:outlineTitle'] = u' '.join(
[e.text() for e in sAnnotation.extents()])
header['property:destinationAnchorName'] = anchor_name
document.addAnnotation(header)
print((u' ' * level +
u'.'.join([unicode(i)
for i in pos]) + u' ' + u' '.join([
e.text()
for e in sAnnotation.extents()
])).encode('utf8'))
elif 'bibitem' in sAnnotation.getAllProperties(
'structure:role'):
#refs.append(sAnnotation)
pass
elif sAnnotation['concept'] == 'Citation':
# Hack to fix a mistake in authors property name
if 'property:author' in sAnnotation and not 'property:authors' in sAnnotation:
sAnnotation[
'property:authors'] = sAnnotation.getAllProperties(
'property:author')
refs.append(sAnnotation)
elif sAnnotation['concept'] == 'LazarusConcept':
concept_id = sAnnotation.get('property:identifier')
if concept_id is not None:
sAnnotation['id'] = str(uuid.uuid4())
concepts[concept_id] = sAnnotation
document.addAnnotation(sAnnotation, 'Lazarus Concept')
elif sAnnotation['concept'] == 'LazarusConceptHit':
hits.append(sAnnotation)
elif sAnnotation['concept'] == 'LazarusSentenceExpression':
expression_annotations.append(sAnnotation)
else:
document.addAnnotation(sAnnotation)
for ref in refs:
#print(ref.get('structure:order', '0'))
pass
refs = sorted(refs,
key=lambda ref: int(ref.get('property:order', '0')))
for ref in refs:
#print(ref.get('structure:order', '0'))
pass
for ref in refs:
# Create Bibliography annotations
#citation = {'unstructured': u' '.join([e.text() for e in ref.extents()])}
#annotation = utopia.tools.utils.citation_to_annotation(citation)
#annotation['property:order'] = ref.get('structure:order')
#annotation.addExtents(ref.extents())
#annotation.addAreas(ref.areas())
#document.addAnnotation(annotation, link['scratch'])
document.addAnnotation(ref, link['scratch'])
# Now link hits to concepts
for i, hit in enumerate(hits):
concept_id = hit.get('property:identifier')
concept = concepts.get(concept_id)
if concept is not None:
concept_uuid = concept.get('id')
hit['property:concept_id'] = concept_uuid
identifier = concept.get('property:identifier')
name = concept.get('property:name', '???')
sources = concept.get('property:externalSources',
'json:[]')
if sources.startswith('json:'):
sources = json.loads(sources[5:])
if 'property:stdInchiKey' in concept:
sources.append({
'database':
' InchiKey',
'identifier':
concept['property:stdInchiKey']
})
if 'property:canonicalSmiles' in concept:
sources.append({
'database':
' SMILES',
'identifier':
concept['property:canonicalSmiles']
})
kind = concept.get('property:kind')
kind = self.dbs.get(kind, {}).get('title', kind)
links = {}
for source in sources:
uri = source.get('uri')
if 'primary' in source.get('relationship', []):
links.setdefault('definition', [])
links['definition'].append(u'''
<a href="{uri}" title="{uri}">{database}</a>
'''.format(**source))
elif uri is None:
if source.get('database') in (' InchiKey',
' SMILES'):
links.setdefault('main', [])
links['main'].append(u'''
<tr><td>{database}:</td><td>{identifier}</td></tr>
'''.format(**source))
else:
identifier = source.get('identifier')
links_category = 'xref'
if 'seeAlso' in source.get('relationship',
[]) or uri is None:
links_category = 'seeAlso'
links.setdefault(links_category, [])
if identifier is not None:
links[links_category].append(u'''
<a href="{uri}" title="{uri}">{name}...</a> ({identifier})
'''.format(**source))
else:
links[links_category].append(u'''
<a href="{uri}" title="{uri}">{name}...</a>
'''.format(**source))
style = u'''
<style>
.lazarus-table tbody {
border: none;
}
.lazarus-table td:first-of-type {
text-align: right;
font-weight: bold;
}
.lazarus-table td {
vertical-align: top;
}
.lazarus-table td:first-of-type {
white-space: nowrap;
}
.lazarus-table td:not(:first-of-type) {
word-break: break-all;
}
.lazarus-table tr td {
padding-top: 0ex;
padding-bottom: 0ex;
}
.lazarus-table tbody:not(:first-of-type) tr:first-of-type td {
padding-top: 1ex;
}
</style>
'''
html = u'''
<table class="lazarus-table">
<tr><td>Name:</td><td>{name}</td></tr>
'''.format(**{'name': name})
categories = {
'xref': 'Related:',
'seeAlso': 'See also:',
'definition': 'Defined in:'
}
for links_category in ('main', 'xref', 'seeAlso',
'definition'):
links_title = categories.get(links_category)
these_links = sorted(
list(set(links.get(links_category, []))))
if len(these_links) > 0:
html += '<tbody>'
if links_category != 'main':
html += u'<tr><td>{0}</td><td>'.format(
links_title)
html += u'<br>'.join(these_links)
html += '</td></tr>'
else:
html += ''.join(these_links)
html += '</tbody>'
#pprint('------------------------')
html += u'''
</table>
'''
#print(html)
hasLinks = len(
links.get('xref', []) + links.get('seeAlso', [])) > 0
ann = spineapi.Annotation()
ann['concept'] = 'Collated'
ann['property:name'] = u'{0}'.format(name)
ann['property:description'] = 'Lazarus Concept'
ann['session:semanticTerm'] = name
ann['property:html'] = [style, html]
ann['property:sourceDescription'] = self.sourceDescription
ann['property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/lazarus-prefs-logo.png', 'image/png')
ann['session:overlay'] = 'hyperlink'
ann['session:color'] = '#880000'
count = 0
print('====', 7)
if 'property:hitFragments' in hit:
hitFragments = hit.getAllProperties(
'property:hitFragments') or []
#pprint(hitFragments)
for hitFragment in hitFragments:
pre, _, rest = hitFragment.partition('{!')
match, _, post = rest.partition('!}')
#pprint((pre, match, post))
matches = document.findInContext(pre,
match,
post,
fuzzy=True)
count += len(matches)
ann.addExtents(matches)
if hasLinks and count > 0:
document.addAnnotation(ann)
style = u'''
<style>
.lazarus-expression .box {
background-color: #FFF0E8;
border-color: #EEE0D8;
}
.lazarus-related {
padding-left: 42px;
background-image: url(%s);
background-repeat: no-repeat;
background-position: top left;
background-size: 37px 48px;
min-height: 53px;
}
.lazarus-related + .lazarus-related {
margin-top: 5px;
border-top: 1px dotted #aaa;
padding-top: 5px;
background-position-y: 5px;
min-height: 58px;
}
.lazarus-sentence {
padding-left: 0.5em;
color: black;
}
.lazarus-sentence.negative {
border-left: solid 5px #bb0000;
}
.lazarus-sentence.positive {
border-left: solid 5px #008800;
}
.lazarus-sentence.negative a {
color: #bb0000;
}
.lazarus-sentence.positive a {
color: #008800;
}
</style>
''' % utopia.get_plugin_data_as_url('images/pdf-page-icon.png',
'image/png')
expressions = []
for sAnnotation in expression_annotations:
exp = sAnnotation.get('property:expressions', 'json:{}')
if exp.startswith('json:'):
exp = json.loads(exp[5:])
context = sAnnotation.get('property:context')
if context is not None:
if exp.get('negative', False):
exp['posneg'] = 'negative'
else:
exp['posneg'] = 'positive'
pprint(context)
pprint(exp)
matched_context = exp.get('context')
matches = []
if matched_context is not None:
matches = document.search(
re.sub(r'\s+', ' ', matched_context))
if len(matches) > 0:
anchor_id = str(uuid.uuid4())[1:-1]
anchor = spineapi.Annotation()
anchor['concept'] = 'Anchor'
anchor['property:anchor'] = anchor_id
anchor.addExtents(matches)
document.addAnnotation(anchor)
exp.update({
'anchor_id': anchor_id,
'sentence': context
})
expressions.append(exp)
js = u'''
<script>
$(document).on('DOMNodeInserted', function(e) {
var element = e.target;
$(element).filter('a[target="tab"]').add('a[target="tab"]', element).each(function () {
var fragment = $(this).closest('.-papyro-internal-citation').data('citation')['userdef']['first_fragment'];
$(this).attr('target', 'pdf; show=highlight; text=[' + encodeURIComponent(fragment) + ']');
});
});
$(function () {
var lazarus = {
expressions: %s,
fingerprints: %s,
relUrl: %s
};
var more_expressions_link = $('#lazarus-expression > p.more').hide();
var more_expressions_spinner = $('#lazarus-expression > div.spinner');
Spinners.create(more_expressions_spinner);
Spinners.play(more_expressions_spinner);
var exp_divs = [];
var identifiers = [];
for (var e = 0; e < lazarus.expressions.length; e++) {
var expression = lazarus.expressions[e];
var exp_div = $('<div class="box"></div>');
exp_div.data('expression', expression);
exp_div.hide();
exp_divs.push(exp_div);
identifiers.push(expression.identifiers);
}
var params = {
fingerprint: lazarus.fingerprints
};
var url = lazarus.relUrl + '?' + $.param(params, traditional=true);
$.ajax({
url: url,
type: 'POST',
dataType: 'json',
data: JSON.stringify(identifiers),
contentType: "application/json",
error: function (xhr, ajaxOptions, thrownError) {
console.log(xhr.statusText);
console.log(xhr.responseText);
console.log(xhr.status);
console.log(thrownError);
// FIXME do something here
Spinners.remove(more_expressions_spinner);
},
success: function (related) {
// Sort related according to the number of articles found
related.results.sort(function (l, r) {
var lv = Object.keys(l.related).length;
var rv = Object.keys(r.related).length;
return (lv > rv) ? -1 : (lv < rv) ? 1 : 0;
});
$.each(related.results, function (idx, result) {
var exp_div = exp_divs[idx];
var expression = exp_div.data('expression');
expression.related = result.related;
delete expression.related[%s];
split = expression.sentence.split(expression.context);
pre = split[0];
pre = pre.replace(/(\w)$/, '$1 ');
pre = pre.replace(/^\s*/, '');
match = expression.context;
post = split[1];
post = post.replace(/^(\w)/, ' $1');
post = post.replace(/\s*$/, '');
expression.pre = pre;
expression.match = match;
expression.post = post;
// Create expression element
exp_div.append('<p class="lazarus-sentence ' + expression.posneg + '">“' + expression.pre + '<a target="pdf; show=select; anchor=' + expression.anchor_id + '"><strong>' + expression.match + '</strong></a>' + expression.post + '”</p>');
exp_div.data('expression', expression);
$('#lazarus-expression > .content').append(exp_div);
if (Object.keys(expression.related).length > 0) {
var related_div = $('<div class="expandable" title="Related expressions elsewhere"></div>');
var related_div_content = $('<div></div>').appendTo(related_div);
function on_expand() {
related_div.off('papyro:expandable:expand', on_expand);
$.each(expression.related, function (idx, obj) {
fragments = [];
$.each(obj, function (id, obj) {
fragments.push(obj.context);
});
fragments.join('\\n');
related_div_content.append($('<div class="lazarus-related unprocessed"></div>').append('<p><strong>“…'+fragments+'…”</strong></p>').hide().data('citation', {identifiers:{doi:idx},userdef:{first_fragment:fragments[0]}}));
// .append(utopia.citation.render({identifiers:{doi:idx},first_fragment:fragments[0]}, true, true))
});
expression.related.length = 0; // empty for future
if ($('.lazarus-related.unprocessed', exp_div).length > 0) {
var more = $('<p class="more right"><a class="more">More related articles...</a></p>');
related_div_content.append(more);
function show_five_related(e) {
e.preventDefault();
$('.lazarus-related.unprocessed', exp_div).slice(0, 5).each(function (idx, obj) {
var citation = $(obj).data('citation');
$(obj).append(utopia.citation.render(citation, true, true));
$(obj).show().removeClass('unprocessed');
});
if ($('.lazarus-related.unprocessed', exp_div).length == 0) {
more.remove();
}
}
more.on('click', show_five_related).click();
}
}
related_div.on('papyro:expandable:expand', on_expand);
exp_div.append(related_div);
utopia.processNewContent(related_div);
}
});
Spinners.remove(more_expressions_spinner);
more_expressions_link.show();
$('a.more', more_expressions_link).click();
}
});
function append_five(e) {
e.preventDefault();
// Show the next five
$('#lazarus-expression > .content').children().filter(':hidden').slice(0,5).show();
// Hide the 'more' link if everything is now visible
if ($('#lazarus-expression > .content').children().filter(':hidden').length == 0) {
more_expressions_link.hide();
}
}
// Hook up 'more' link
$('#lazarus-expression > p.more > a.more').on('click', append_five).click();
});
</script>
''' % (json.dumps(expressions), json.dumps(
document.fingerprints()), json.dumps(laz_docRelUrl),
json.dumps(this_doi))
#print(js.encode('utf8'))
html = u'''
<div id="lazarus-expression"><div class="content"></div><div class="spinner"></div><p class="more"><a class="more">More expressions...</a></p></div>
'''
if len(expressions) > 0:
ann = spineapi.Annotation()
ann['concept'] = 'Collated'
ann['property:name'] = 'Lazarus Expressions'
ann['property:description'] = u'Summarizing expression(s)'
ann['property:html'] = [js, style, html]
ann['property:sourceDescription'] = self.sourceDescription
ann['property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/lazarus-prefs-logo.png', 'image/png')
document.addAnnotation(ann)
else: # no permission
noprompt = self.get_config('noprompt', False)
if not noprompt:
annotation = spineapi.Annotation()
annotation['concept'] = 'Collated'
params = {
'uuid': self.uuid(),
}
annotation['property:html'] = utopia.get_plugin_data(
'tpl/denied.html').format(**params)
annotation['property:name'] = 'Lazarus'
annotation[
'property:description'] = 'Lazarus functionality is turned off'
annotation[
'property:sourceDescription'] = self.sourceDescription
annotation[
'property:sourceIcon'] = utopia.get_plugin_data_as_url(
'images/lazarus-prefs-logo.png', 'image/png')
annotation['session:default'] = '1'
document.addAnnotation(annotation)
def on_ready_event(self, document):
doi = common.utils.metadata(document, "doi", "")
match = self.splitRegEx.match(doi)
if match is not None:
articleNumber = match.group("number")
annotation = spineapi.Annotation()
annotation["concept"] = "PublisherIdentity"
annotation["property:logo"] = utopia.get_plugin_data_as_url("images/logo.png", "image/png")
annotation["property:title"] = "eLife"
annotation["property:webpageUrl"] = "http://www.elifesciences.org/"
document.addAnnotation(annotation, "PublisherMetadata")
# Turn all the DOIs that are sub-DOIs of this document into links
regex = r"{0}\.\d+".format(re.escape(doi))
for match in document.search(regex, spineapi.RegExp):
url = "http://dx.doi.org/{0}".format(match.text())
annotation = spineapi.Annotation()
annotation["concept"] = "Hyperlink"
annotation["property:webpageUrl"] = url
annotation["session:volatile"] = "1"
annotation.addExtent(match)
document.addAnnotation(annotation)
# Try to get the NLM directly from eLife
url = "http://elife.elifesciences.org/elife-source-xml/10.7554/eLife.{0}"
url = url.format(articleNumber)
try:
nlm = urllib2.urlopen(url, timeout=8).read()
except (urllib2.URLError, socket.timeout):
return
info = common.nlm.parse(nlm)
if info is not None and len(info) > 0:
# Enrich citation information with identifiers from PMC
parser = etree.XMLParser(ns_clean=True, recover=True, remove_blank_text=True, encoding="utf8")
pmids = dict(
(
(citation["pmid"], citation["id"])
for citation in info["citations"]
if "pmid" in citation and "id" in citation
)
)
if len(pmids) > 0:
pubmed_abstracts = etree.fromstring(
common.eutils.efetch(id=",".join(pmids.keys()), retmode="xml", rettype="abstract"), parser
)
for idList in pubmed_abstracts.xpath("PubmedArticle/PubmedData/ArticleIdList"):
# print etree.tostring(idList)
pmid = idList.findtext('ArticleId[@IdType="pubmed"]')
if pmid in pmids:
citation = info["citations_by_id"][pmids[pmid]]
for key_name, id_name in (("doi", "doi"), ("pmcid", "pmc"), ("pii", "pii")):
id = idList.findtext('ArticleId[@IdType="{0}"]'.format(id_name))
if key_name not in citation and id is not None:
citation[key_name] = id
# print 'KEY', key_name, id
# Create Metadata link annotation
link = document.newAccList("metadata", 100)
link["property:sourceIcon"] = utopia.get_plugin_data_as_url("images/annotation_icon.png", "image/png")
link["property:sourceTitle"] = "eLife"
link[
"property:sourceDescription"
] = """
<p>The <a href="http://www.elifesciences.org/">eLife</a> open access publishing platform.</p>
"""
# Create Metadata annotation
annotation = spineapi.Annotation()
annotation["concept"] = "DocumentMetadata"
for k in self.keys:
v = info.get(k)
if v is not None:
annotation["property:{0}".format(k)] = v
document.addAnnotation(annotation, link["scratch"])
# Create Bibliography annotations
for citation in info.get("citations", []):
annotation = spineapi.Annotation()
annotation["concept"] = "DocumentReference"
for k in self.keys:
v = citation.get(k)
if v is not None:
annotation["property:{0}".format(k)] = v
document.addAnnotation(annotation, link["scratch"])
#######################################################################################
# Apply parsed data to document
# Citations
for citation in info["citations"]:
# Find cross refs
for pre, label, post in citation.get("contexts", []):
matches = document.findInContext(pre, label, post)
# print matches
if len(matches) > 0:
try:
annotation = spineapi.Annotation()
annotation["concept"] = "ForwardCitation"
annotation["property:state"] = "found"
if "title" in citation:
annotation["property:title"] = citation["title"]
if "id" in citation:
annotation["property:bibid"] = citation["id"]
if "doi" in citation and citation["doi"].startswith("10.1371/"):
citation[
"pdf"
] = "http://www.ploscompbiol.org/article/fetchObjectAttachment.action?uri={0}&representation=PDF".format(
"info:doi/{0}".format(citation["doi"])
)
if "pmcid" in citation:
citation["pdf"] = "http://www.ncbi.nlm.nih.gov/pmc/articles/{0}/pdf/".format(
citation["pmcid"]
)
for k in (
"displayText",
"label",
"pdf",
"pmid",
"pmc",
"pii",
"doi",
"first_author_surname",
"year",
"journal",
"volume",
"page_from",
):
if k in citation:
annotation["property:{0}".format(k)] = citation[k]
# print annotation.get('property:label'), annotation.get('property:pdf')
for match in matches:
annotation.addExtent(match)
document.addAnnotation(annotation, link["scratch"])
# print citation
except:
raise
pass # FIXME
for id, table in info.get("tables", {}).iteritems():
if "caption" in table and "xml" in table:
regex = fuzz(table["caption"], strict=True)
print regex
matches = document.search(regex, spineapi.RegExp + spineapi.IgnoreCase)
if len(matches) == 1:
annotation = spineapi.Annotation()
annotation["concept"] = "Table"
annotation[
"session:upload_files"
] = "data:application/xml;name=data.xml;base64,%s" % base64.standard_b64encode(table["xml"])
annotation["session:volatile"] = "1"
annotation.addExtent(matches[0])
document.addAnnotation(annotation, link["scratch"])
else:
print "*********** failed to match table:", id
