def extract_more_verses(text, ref, point, outline):
"""
-text: string of verse numbers without book or chapter, e.g. '3, 7, 10-14, 16'.
-ref: Reference immediately preceding this string of verse numbers.
-point: outline point the verses are under.
-outline: outline list from extract().
Helper function for extract() -- creates References for verses and adds them to outline.
Creates References for verses and adds them to outline.
"""
# gets list of tuples: (verse, end_verse,)
# e.g. '3, 7, 10-14, 16' --> [(3, None,), (7, None,), (10, 14,), (16, None,)]
verses = re.findall(re.compile(r'(?P<verse>\d{1,3})(?:-(?P<end_verse>\d{1,3}))?'), text)
for verse in verses:
more_verses_ref = Reference(outline_point=point,
book=ref.book,
verse=int(verse[0]))
more_verses_ref.end_verse = int(verse[1]) if verse[1] else None
if ref.end_chapter is not None:
more_verses_ref.chapter = ref.end_chapter
else:
more_verses_ref.chapter = ref.chapter
if more_verses_ref.end_verse is not None:
more_verses_ref.end_chapter = more_verses_ref.chapter
more_verses_ref.save()
outline[-1][1].append(more_verses_ref)
return
def download(request):
query1 = FlashCard.all()
query2 = Note.all()
query3 = Reference.all()
xmlcards = [card.to_xml() for card in query1.run()]
xmlnotes = [note.to_xml() for note in query2.run()]
xmlrefs = [ref.to_xml() for ref in query3.run()]
return render_to_response('download.xml', {
'xmlcards': xmlcards,
'xmlnotes': xmlnotes,
'xmlrefs': xmlrefs
})
def read_data(filename, labeled=False, limit=False):
""" Reads the input file of references and creates a blocked list of
reference objects.
Args:
filename: the name of the file with the references.
labeled: if the reference should be labeled to an entitiy or not.
Returns:
A list of lists of reference objects grouped by block.
"""
references = []
curr_block = []
count = 0
input_file = open(filename, 'r')
for line in input_file:
line = line.strip()
if not line:
references.append(curr_block)
curr_block = []
continue
elif limit and count >= limit:
break
attrs = line.split('<>')
reference = Reference(
refid=count,
name=attrs[4],
title=attrs[5],
coauthors=attrs[2].split(':') if attrs[2] != 'undefined' else [],
venue=attrs[3],
label=attrs[1].split('_')[0] if labeled else None)
curr_block.append(reference)
count += 1
if curr_block:
references.append(curr_block)
input_file.close()
return references
def reference():
Entry = namedtuple('Entry', ['professor_email'])
ref_value = Reference.load(current_user.id)
if ref_value.ref_list:
data = {'ref_list': []}
for professor_email in ref_value.ref_list:
data['ref_list'].append(Entry(professor_email))
form = ReferenceListForm(data=data)
else:
form = ReferenceListForm()
if form.validate_on_submit():
if form.add.data:
if bool(form.professor_email.data) and Professor.load(form.professor_email.data) is not None:
ref_value.add(form.professor_email.data)
flash('Reference added', 'success')
commit()
else:
flash('Invalid email', 'danger')
return redirect(url_for('reference.reference'))
else:
for index in range(len(form.ref_list)):
if form.ref_list[index].save.data:
if bool(form.ref_list[index].professor_email.data) and Professor.load(
form.ref_list[index].professor_email.data) is not None:
ref_value.update(index, form.ref_list[index].professor_email.data)
commit()
flash('Updated successfully', 'success')
else:
flash('Invalid email', 'danger')
return redirect(url_for('reference.reference'))
if form.ref_list[index].delete.data:
ref_value.delete(index)
commit()
flash('Deleted successfully', 'success')
return redirect(url_for('reference.reference'))
return render_template('reference.html', form=form)
def insert_and_associate(self):
ref_no = Reference.insert(self.pmid, self.user)
RefTemp.delete(self.pmid)
lg_feat_hash = {}
message = ''
topic_added = {}
task_added = {}
if len(self.tasks) == 0:
return None
for task_entry in self.tasks:
task = task_entry[0]
genes = task_entry[1]
comment = task_entry[2]
if len(genes) > 0:
feat_no_list = []
topic = ''
if 'Add to' in task:
topic = 'Additional Literature'
elif 'Review' in task:
topic = task
else:
topic = 'Primary Literature'
if 'Review' in task or 'Add to' in task:
task = 'Gene Link'
message += "Curation_task = '" + task
message += "', literature_topic = '" + topic + "'"
message += ", gene = "
for name in genes:
feat_no = self.feature_no_for_name[name]
feat_no_list.append(feat_no)
## insert into ref_curation
if not task_added.has_key((feat_no, task)):
RefCuration.insert(ref_no, task, self.user, feat_no,
comment)
task_added[(feat_no, task)] = 1
message += name + '|'
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
## insert into lit_guide + litguide_feat
if topic_added.has_key(topic):
litguide_no = topic_added[topic]
else:
litguide_no = LitGuide.insert(ref_no, topic, self.user)
topic_added[topic] = litguide_no
for feat_no in feat_no_list:
key = (feat_no, litguide_no)
if lg_feat_hash.has_key(key):
continue
lg_feat_hash[key] = 1
LitGuideFeat.insert(feat_no, litguide_no)
else: ## no gene name provided
## if no gene name provided and "Add to database" was checked,
## no need to add any association
if 'Add' in task:
continue
## if it is a review, no need to add to ref_curation
if 'Review' in task:
## topic = task = 'Reviews'
LitGuide.insert(ref_no, task, self.user)
message += "Literature_topic = '" + task + "'<br>"
continue
if not task_added.has_key((0, task)):
RefCuration.insert(ref_no, task, self.user, None, comment)
task_added[(0, task)] = 1
message += "Curation_task = '" + task + "'"
## insert into lit_guide
if 'HTP' in task or 'Review' in task:
topic = ''
if 'HTP' in task:
topic = 'Omics'
else:
# 'Review' in task:
topic = task
if topic_added.has_key(topic):
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
continue
else:
lit_guide_no = LitGuide.insert(ref_no, topic,
self.user)
topic_added[topic] = lit_guide_no
message += ", literature_topic = '" + topic
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
db.session.commit()
return message
if not settings.fetcher.post_processing.expand_links:
return
lnow = time.time()
links = expand_links(set(hrefs))
log.debug("%s - %d links in %fs" %
(netloc, len(hrefs), time.time() - lnow))
# TODO: replace hrefs in content, avoiding creating the item twice
# including item url = hrefs[entry.link]
# ...and handling updates
#links = list(set(links.values()))
links = set(links.values())
for link in links:
try:
reference = Reference.get(item=item,
link=Link.get(expanded_url=link))
except Reference.DoesNotExist:
reference = Reference.create(
item=item, link=Link.get(expanded_url=link))
records += 1
except:
log.error(tb())
log.debug("%s - %d records in %fs" %
(netloc, records, time.time() - now))
writer.commit()
try:
favicon = Favicon.get(id=feed.favicon)
except Favicon.DoesNotExist:
favicon = Favicon.create(data=fetch_anyway(feed.site_url))
def extract(text):
"""
Extract from a block of text a list of 2-tuples
containing an outline point and a list of normalized, tupled verse references under that outline point.
"""
outline = []
point = OutlinePoint(level=0, string='Scripture Reading:')
point.save()
outline.append((point, [],))
for r in re.finditer(scripture_re, text):
try:
# find Roman numerals / outline points
is_bullet = False
for i in range(1,5):
if r.group(i):
point = OutlinePoint(level=i, string=r.group(i))
point.save()
outline.append((point, [],))
is_bullet = True
if is_bullet == False:
ref = Reference(outline_point=point)
outline[-1][1].append(ref)
if r.group('book'): # reference contains book name
ref.book = get_book(r.group('book'))[1]
ref.chapter = int(r.group('chapter')) if r.group('chapter') else None
ref.verse = int(r.group('verse')) if r.group('verse') else None
ref.end_chapter = int(r.group('end_chapter')) if r.group('end_chapter') else None
ref.end_verse = int(r.group('end_verse')) if r.group('end_verse') else None
if ((ref.end_verse is not None) and (ref.end_chapter is None)):
ref.end_chapter = ref.chapter
ref.save()
if r.group('more_verses'):
extract_more_verses(r.group('more_verses'), ref, point, outline)
else: # get book from previous reference
if len(outline[-1][1]) > 1:
ref.book = outline[-1][1][-2].book
else: # if this is the first reference under an outline point
ref.book = outline[-2][1][-1].book
if r.group('headless_chapter'): # reference has no book
ref.chapter = int(r.group('headless_chapter'))
ref.verse = int(r.group('headless_verse')) if r.group('headless_verse') else None
ref.end_chapter = int(r.group('headless_end_chapter')) if r.group('headless_end_chapter') else None
ref.end_verse = int(r.group('headless_end_verse')) if r.group('headless_end_verse') else None
if ((ref.end_verse is not None) and (ref.end_chapter is None)):
ref.end_chapter = ref.chapter
ref.save()
if r.group('more_headless_verses'):
extract_more_verses(r.group('more_headless_verses'), ref, point, outline)
else:
if r.group('lonely_verse'):
if len(outline[-1][1]) > 1:
ref.chapter = outline[-1][1][-2].chapter
else: # this is the first reference under an outline point
ref.chapter = outline[-2][1][-1].chapter
ref.verse = int(r.group('lonely_verse'))
ref.end_verse = int(r.group('lonely_end_verse')) if r.group('lonely_end_verse') else None
if ((ref.end_verse is not None) and (ref.end_chapter is None)):
ref.end_chapter = ref.chapter
ref.save()
if r.group('more_lonely_verses'):
extract_more_verses(r.group('more_lonely_verses'), ref, point, outline)
except InvalidReferenceException:
pass
return outline
def post(self, request):
param = QueryDict(request.body)
uuid = param.get('uuid')
title = param.get('title')
time = param.get('time')
origin = param.get('origin')
_authors = param.getlist('authors')
link = param.get('link')
_tags = param.getlist('tags')
content = param.get('content')
refer_to = param.getlist('reference')
score = param.get('score')
try:
year, month = time.split('-')
year, month = int(year), int(month)
publish_time = datetime.date(year, month, 1)
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '提供的日期{}有误'.format(time)}, status=500)
for _tag in _tags:
try:
_tag = int(_tag)
_ = ResearchTag.objects.get(research_tag_id=_tag)
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '错误的标签{}'.format(_tag)},
status=500)
tags = ResearchTag.objects.filter(
research_tag_id__in=[int(_t) for _t in _tags])
author_ids = []
for _author in _authors:
if _author.isdigit():
author_ids.append(int(_author))
elif Author.objects.filter(name=_author).exists():
a = Author.objects.get(name=_author).author_id
author_ids.append(a)
else:
a = Author(name=_author)
a.save()
author_ids.append(a.author_id)
authors = Author.objects.filter(author_id__in=author_ids)
try:
score = int(score)
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '错误的评分分数格式'}, status=500)
if not Paper.objects.filter(paper_uuid=uuid).exists():
# 新建的场合
try:
comment = PaperComment(content=content)
comment.save()
paper = Paper(paper_uuid=uuid,
title=title,
publish_origin=origin,
publish_time=publish_time,
author=authors,
link=link,
tag=tags,
comment=comment,
self_score=score)
paper.save()
redis.set(self.LATEST_KEY, str(uuid_gen.uuid4()))
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '保存失败'}, status=500)
else:
return JsonResponse({
'next':
reverse('paperdb.detail',
kwargs={'paper_uuid': paper.paper_uuid})
})
try:
# 编辑的场合
paper = Paper.objects.get(paper_uuid=uuid)
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '错误的uuid/未找到相关论文记录'}, status=404)
else:
paper.title = title
paper.publish_time = publish_time
paper.publish_origin = origin
paper.author = authors
paper.link = paper.link
paper.tag = tags
paper.self_score = score
try:
paper.save()
except Exception as e:
logger.error(traceback.format_exc(e))
return JsonResponse({'msg': '保存失败'}, status=500)
if paper.comment is None:
if content != '':
comment = PaperComment(content=content)
comment.save()
paper.comment = comment
paper.save()
elif content != paper.comment.content.replace(
'\r\n', '\n'): # traditional下的换行符出入
paper.comment.content = content
paper.comment.save()
for refer_to_paper in Paper.objects.filter(paper_uuid__in=refer_to):
if not Reference.objects.filter(
reference_src=paper,
reference_trg=refer_to_paper).exists():
reference = Reference(reference_src=paper,
reference_trg=refer_to_paper)
reference.save()
return JsonResponse({
'next':
reverse('paperdb.detail', kwargs={'paper_uuid': paper.paper_uuid})
})
if not settings.fetcher.post_processing.expand_links:
return
lnow = time.time()
links = expand_links(set(hrefs))
log.debug("%s - %d links in %fs" % (netloc, len(hrefs),time.time()-lnow))
# TODO: replace hrefs in content, avoiding creating the item twice
# including item url = hrefs[entry.link]
# ...and handling updates
#links = list(set(links.values()))
links = set(links.values())
for link in links:
try:
reference = Reference.get(item = item, link = Link.get(expanded_url = link))
except Reference.DoesNotExist:
reference = Reference.create(item = item, link = Link.get(expanded_url = link))
records += 1
except:
log.error(tb())
log.debug("%s - %d records in %fs" % (netloc, records,time.time()-now))
writer.commit()
try:
favicon = Favicon.get(id = feed.favicon)
except Favicon.DoesNotExist:
favicon = Favicon.create(data=fetch_anyway(feed.site_url))
feed.favicon = favicon
feed.save()
from models import Fiction, CompactDisc, VinylRecord, Nonfiction, Reference
catalog_seed = [
Fiction(0, "Sirens Of Titan", "Kurt Vonnegut", 1959),
Fiction(1, "The Heart Is a Lonely Hunter", "Carson McCullers", 1940),
Fiction(2, "Things Fall Apart", "Chinua Achebe", 1958),
CompactDisc(3, "In Utero", "Nirvana", 1993, 12),
CompactDisc(4, "Siamese Dream", "Smashing Pumpkins", 1993, 12),
VinylRecord(5, "Hunky Dory", "David Bowie", 1971, 11),
VinylRecord(6, "3 Feet High and Rising", "De La Soul", 1989, 16),
Nonfiction(7, "The Columbian Exchange", "Alfred Crosby", 1972),
Nonfiction(8, "Imagined Communities", "Benedict Anderson", 1983),
Nonfiction(9, "The Rise and Fall of the Plantation Complex",
"Philip D. Curtin", 1998),
Fiction(10, "Wandering", "Lu Xun", 1926),
Reference(11, "The Oxford Dictionary of Word Histories", "Linguistics",
2002),
Reference(12, "The Associated Press Style Guide 2019 Edition",
"Journalism", 2019),
Reference(13, "The Chicago Manual of Style, 17th Ed.",
"Academic Publishing", 2017),
Fiction(14, "Edges: 13 New Tales from the Borderlands of the Imagination",
"Multiple Authors, ed. Ursula K. LeGuin", 1980),
CompactDisc(15, "w h o k i l l", "tUnE-yArDs", 2011, 10),
Nonfiction(16, "Black Lamb and Grey Falcon", "Rebecca West", 1941)
]
def insert_and_associate(self):
ref_no = Reference.insert(self.pmid, self.user)
RefTemp.delete(self.pmid)
lg_feat_hash = {}
message = ''
topic_added = {}
task_added = {}
if len(self.tasks) == 0:
return None
for task_entry in self.tasks:
task = task_entry[0]
genes = task_entry[1]
comment = task_entry[2]
if len(genes) > 0:
feat_no_list = []
topic = ''
if 'Add to' in task:
topic = 'Additional Literature'
elif 'Review' in task:
topic = task
else:
topic = 'Primary Literature'
if 'Review' in task or 'Add to' in task:
task = 'Gene Link'
message += "Curation_task = '" + task
message += "', literature_topic = '" + topic + "'"
message += ", gene = "
for name in genes:
feat_no = self.feature_no_for_name[name]
feat_no_list.append(feat_no)
## insert into ref_curation
if not task_added.has_key((feat_no, task)):
RefCuration.insert(ref_no, task, self.user, feat_no, comment)
task_added[(feat_no, task)] = 1
message += name + '|'
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
## insert into lit_guide + litguide_feat
if topic_added.has_key(topic):
litguide_no = topic_added[topic]
else:
litguide_no = LitGuide.insert(ref_no, topic, self.user)
topic_added[topic] = litguide_no
for feat_no in feat_no_list:
key = (feat_no, litguide_no)
if lg_feat_hash.has_key(key):
continue
lg_feat_hash[key] = 1
LitGuideFeat.insert(feat_no, litguide_no)
else: ## no gene name provided
## if no gene name provided and "Add to database" was checked,
## no need to add any association
if 'Add' in task:
continue
## if it is a review, no need to add to ref_curation
if 'Review' in task:
## topic = task = 'Reviews'
LitGuide.insert(ref_no, task, self.user)
message += "Literature_topic = '" + task + "'<br>"
continue
if not task_added.has_key((0, task)):
RefCuration.insert(ref_no, task, self.user, None, comment)
task_added[(0, task)] = 1
message += "Curation_task = '" + task + "'"
## insert into lit_guide
if 'HTP' in task or 'Review' in task:
topic = ''
if 'HTP' in task:
topic = 'Omics'
else:
# 'Review' in task:
topic = task
if topic_added.has_key(topic):
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
continue
else:
lit_guide_no = LitGuide.insert(ref_no, topic, self.user)
topic_added[topic] = lit_guide_no
message += ", literature_topic = '" + topic
if comment:
message += ", comment = '" + comment + "'"
message += "<br>"
db.session.commit()
return message
def save_references(request):
"""
SGD_features.tab format looks like this:
0) Primary SGDID (mandatory)
1) Feature type (mandatory)
2) Feature qualifier (optional)
3) Feature name (optional)
4) Standard gene name (optional)
5) Alias (optional, multiples separated by |)
6) Parent feature name (optional)
7) Secondary SGDID (optional, multiples separated by |)
8) Chromosome (optional)
9) Start_coordinate (optional)
10) Stop_coordinate (optional)
11) Strand (optional)
12) Genetic position (optional)
13) Coordinate version (optional)
14) Sequence version (optional)
15) Description (optional)
"""
featuresFile = request.FILES['features_file']
features = [x.split('\t') for x in featuresFile.read().splitlines()]
for line in features:
reference = Reference()
reference.sgdid = line[0]
reference.feature_type = line[1]
if line[2]:
reference.qualifier = line[2]
if line[3]:
reference.feature_name = line[3]
if line[4]:
reference.standard_name = line[4]
if line[5]:
reference.aliases = line[5]
if line[6]:
reference.parent_name = line[6]
if line[7]:
reference.secondary_sgdid = line[7]
if line[15]:
reference.description = line[15]
reference.createdDate = datetime.datetime.now()
reference.modifiedDate = datetime.datetime.now()
reference.save()
return HttpResponseRedirect('/strains/')
def books(request):
query = Reference.all()
imgIDs = [ref.pic.key() for ref in query.run() if ref.pic is not None]
return render_to_response('library.html', {'images': imgIDs})