def export_json_impl():
h = HypothesisUtils(username=username, token=api_token, group=group, max_results=100000)
params = {'group' : h.group }
rows = h.search_all(params)
annos = [HypothesisAnnotation(row) for row in rows]
# clean up bugs from old curation workflow
for anno in annos:
if anno.tags:
new_tags = []
for tag in anno.tags:
if tag in bad_tags:
new_tags.append(tag.replace('RRID:', 'RRIDCUR:')) # scibot made a mistake early, might be able to correct tags in bulk someday
else:
new_tags.append(tag) # horribly inefficient...
anno.tags = new_tags
if anno.text.startswith('RRID:'): # catch cases where the RRID was put in text instead of in tags
if 'RRIDCUR:Missing' in anno.tags or 'RRIDCUR:Unrecognized' in anno.tags:
rtag = anno.text.split(None,1)[0] # trap for cases where there is more text after an RRID...
if rtag not in anno.tags:
anno.tags.append(rtag)
print('TEXT ISSUE for %s at https://hyp.is/%s' % (anno.user, anno.id))
elif anno.exact and anno.exact.startswith('RRID:'): # this needs to go second in case of RRIDCUR:Incorrect
if anno.exact.startswith('RRID: '): # deal with nospace first
rtag = anno.exact.replace('RRID: ', 'RRID:')
else:
rtag = anno.exact
rtag = rtag.split(None,1)[0] # trap more
if rtag not in anno.tags:
if anno.user == 'scibot' and len(anno.tags) == 1 and anno.tags[0].startswith('RRID:RRID:'): # FIXME HACK
anno.tags = [rtag]
else:
pass # anything else we detect in the data doesn't need to be corrected or used to fix tags
output_json = [anno.__dict__ for anno in annos]
DATE = date.today().strftime('%Y-%m-%d')
return output_json, DATE
#!/usr/bin/env python3
from __future__ import print_function
from os import environ
from collections import defaultdict
from hypothesis import HypothesisUtils, HypothesisAnnotation
username = environ.get('RRIDBOT_USERNAME', 'USERNAME') # Hypothesis account
password = environ.get('RRIDBOT_PASSWORD', 'PASSWORD')
group = environ.get('RRIDBOT_GROUP', '__world__')
print(username, group) # sanity check
h = HypothesisUtils(username=username, password=password, group=group, max_results=5000)
h.login()
params = {'group' : h.group }
rows = h.search_all(params)
annos = [HypothesisAnnotation(row) for row in rows]
annotated_urls = defaultdict(list)
for anno in annos:
annotated_urls[anno.uri].append(anno)
html = """<html>
<head><style>
body { font-family:verdana;margin:.75in }
.anno { margin: 20px;
border-style: solid;
border-width: thin;
padding: 20px; }
.text { margin:20px }
.article { font-size:larger }
</style></head>
<body>"""
def export_impl():
h = HypothesisUtils(username=username, token=api_token, group=group, max_results=100000)
params = {'group' : h.group }
rows = h.search_all(params)
annos = [HypothesisAnnotation(row) for row in rows]
annotated_urls = defaultdict(list)
for anno in annos:
annotated_urls[anno.uri].append(anno)
fix_trailing_slash(annotated_urls)
output_rows = []
for annotated_url in annotated_urls.keys():
#print(annotated_url)
annos = annotated_urls[annotated_url]
replies = defaultdict(list)
PMID = []
for anno in annos: # gotta build the reply structure and get pmid
#print('id:', anno.id)
#print('user:'******'exact:', anno.exact)
#print('text:', anno.text)
#print('tags:', anno.tags)
#print('type:', anno.type)
#print('references:', anno.references)
if anno.references:
for reference in anno.references: # shouldn't there only be one???
replies[reference].append(anno)
PMID.extend([tag for tag in anno.tags if tag.startswith('PMID:') and '_' not in tag]) # bad tags with PMID:SCR_
#curators didn't put the pmid in as tags :(
if anno.text.startswith('PMID:'): # DANGER ZONE
if '_' in anno.text:
print('PMIDS DONT HAVE UNDERSCORES PROBABLY CURATION BUG', anno.text)
else:
PMID.append(anno.text.strip()) # because, yep, when you don't tag sometimes you get \n :/
if PMID:
if len(PMID) > 1:
print(PMID, annotated_url)
if PMID[0] == PMID[1]:
PMID = PMID[0]
print('WARNING: more than one pmid tag')
else:
raise BaseException('more than one pmid tag')
else:
PMID = PMID[0]
#print(PMID)
else:
all_tags = []
for a in annos:
all_tags.extend(a.tags)
#print('NO PMID FOR', annotated_url)
#print(set([a.user for a in annos]))
#print(all_tags)
PMID = annotated_url
RRIDs = defaultdict(list)
EXACTs = {}
CITEs = {}
#USERs = {}
for anno in annos:
RRID = None
additional = []
for tag in anno.tags:
if re.match('RRID:.+[0-9]+.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
#if re.match('RRID:.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
if RRID is not None:
raise BaseException('MORE THAN ONE RRID PER ENTRY!')
RRID = tag # :/ this works for now but ARHGHHGHASFHAS
else:
additional.append(tag) # eg Unresolved
if tag == 'RRIDCUR:Missing': # fix for bad curation process
maybe_rrid = anno.text.strip()
if re.match('RRID:.+[0-9]+', maybe_rrid): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
RRID = maybe_rrid # RRIDCUR:Missing was already added above
if RRID is not None:
EXACTs[RRID] = anno.exact.strip() if anno.exact else ''
RRIDs[RRID].extend(additional)
#USERs[RRID] = anno.user
if RRID not in CITEs:
if anno.text:
if 'Proper Citation:' in anno.text:
CITEs[RRID] = anno.text.split('Proper Citation:')[1].strip().split('<',1)[0]
if anno.id in replies:
for r_anno in replies[anno.id]:
RRIDs[RRID].extend(r_anno.tags) # not worrying about the text here
elif not anno.references and PMID not in anno.tags: # this is an independent annotation which will not be included
new = 'NONE:' + anno.id
RRIDs[new].append('')
EXACTs[new] = anno.exact
#USERs[RRID] = anno.user
for rrid, more in RRIDs.items():
#FIXME TOOOOOO SLOW
#r = requests.get('https://scicrunch.org/resolver/{RRID}.xml'.format(RRID=rrid))
#if r.status_code < 300:
#proper_citation = get_proper_citation(r.content)
#else:
#proper_citation = ''
try:
proper_citation = CITEs[rrid]
except KeyError: # FIXME this is a hack to avoid some cases of LWW for citations
proper_citation = ''
if not more:
row = [PMID, rrid, '', annotated_url, EXACTs[rrid], proper_citation]
output_rows.append(row)
else:
for val in set(more): # cull dupes
row = [PMID, rrid, val, annotated_url, EXACTs[rrid], proper_citation]
output_rows.append(row)
DATE = date.today().strftime('%Y-%m-%d')
return output_rows, DATE
def export_impl():
h = HypothesisUtils(username=username, token=api_token, group=group, max_results=100000)
params = {'group' : h.group }
rows = h.search_all(params)
annos = [HypothesisAnnotation(row) for row in rows]
annotated_urls = defaultdict(list)
for anno in annos:
annotated_urls[anno.uri].append(anno)
output_rows = []
for annotated_url in annotated_urls.keys():
#print(annotated_url)
annos = annotated_urls[annotated_url]
replies = defaultdict(list)
PMID = []
for anno in annos: # gotta build the reply structure and get pmid
#print('id:', anno.id)
#print('user:'******'exact:', anno.exact)
#print('text:', anno.text)
#print('tags:', anno.tags)
#print('type:', anno.type)
#print('references:', anno.references)
if anno.references:
for reference in anno.references: # shouldn't there only be one???
replies[reference].append(anno)
PMID.extend([tag for tag in anno.tags if tag.startswith('PMID:') and '_' not in tag]) # bad tags with PMID:SCR_
#curators didn't put the pmid in as tags :(
if anno.text.startswith('PMID:'): # DANGER ZONE
if '_' in anno.text:
print('PMIDS DONT HAVE UNDERSCORES PROBABLY CURATION BUG', anno.text)
else:
PMID.append(anno.text.strip()) # because, yep, when you don't tag sometimes you get \n :/
if PMID:
if len(PMID) > 1:
print(PMID, annotated_url)
if PMID[0] == PMID[1]:
PMID = PMID[0]
print('WARNING: more than one pmid tag')
else:
raise BaseException('more than one pmid tag')
else:
PMID = PMID[0]
#print(PMID)
else:
all_tags = []
for a in annos:
all_tags.extend(a.tags)
#print('NO PMID FOR', annotated_url)
#print(set([a.user for a in annos]))
#print(all_tags)
PMID = annotated_url
RRIDs = defaultdict(list)
EXACTs = {}
CITEs = {}
#USERs = {}
for anno in annos:
RRID = None
additional = []
for tag in anno.tags:
if re.match('RRID:.+[0-9]+.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
#if re.match('RRID:.+', tag): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
if RRID is not None:
raise BaseException('MORE THAN ONE RRID PER ENTRY!')
RRID = tag # :/ this works for now but ARHGHHGHASFHAS
else:
additional.append(tag) # eg Unresolved
if tag == 'RRIDCUR:Missing': # fix for bad curation process
maybe_rrid = anno.text.strip()
if re.match('RRID:.+[0-9]+', maybe_rrid): # ARRRRGGGGHHHHHHH ARRRRGGHHHH
RRID = maybe_rrid # RRIDCUR:Missing was already added above
if RRID is not None:
EXACTs[RRID] = anno.exact.strip() if anno.exact else ''
RRIDs[RRID].extend(additional)
#USERs[RRID] = anno.user
if RRID not in CITEs:
if anno.text:
if 'Proper Citation:' in anno.text:
CITEs[RRID] = anno.text.split('Proper Citation:')[1].strip().split('<',1)[0]
if anno.id in replies:
for r_anno in replies[anno.id]:
RRIDs[RRID].extend(r_anno.tags) # not worrying about the text here
elif not anno.references and PMID not in anno.tags: # this is an independent annotation which will not be included
new = 'NONE:' + anno.id
RRIDs[new].append('')
EXACTs[new] = anno.exact
#USERs[RRID] = anno.user
for rrid, more in RRIDs.items():
#FIXME TOOOOOO SLOW
#r = requests.get('https://scicrunch.org/resolver/{RRID}.xml'.format(RRID=rrid))
#if r.status_code < 300:
#proper_citation = get_proper_citation(r.content)
#else:
#proper_citation = ''
try:
proper_citation = CITEs[rrid]
except KeyError: # FIXME this is a hack to avoid some cases of LWW for citations
proper_citation = ''
if not more:
row = [PMID, rrid, '', annotated_url, EXACTs[rrid], proper_citation]
output_rows.append(row)
else:
for val in set(more): # cull dupes
row = [PMID, rrid, val, annotated_url, EXACTs[rrid], proper_citation]
output_rows.append(row)
DATE = date.today().strftime('%Y-%m-%d')
return output_rows, DATE
