def post(self, unused):
self.response.headers['Content-type'] = 'text/javascript'
user = users.get_current_user()
annos = self.request.get('annos', '').strip()
format = self.request.get('format',
utils.guess_format(annos, force_tabs=True))
if not annos:
simplejson.dumps({'status': 'fail'})
return
name = user and user.nickname() or ""
title = self.request.get('title') or ""
anno_id = hashlib.md5(annos + title).hexdigest()
# dont save unless it's new.
a = Annotation.gql(
"WHERE anno_id = :anno_id AND title = :title AND author = :author",
anno_id=anno_id,
title=title,
author=user).get()
if a is None:
a = Annotation(title=title,
author=user,
content=annos.strip(),
anno_id=anno_id,
format=format)
a.put()
self.response.out.write(
simplejson.dumps({
'status': 'success',
'anno_id': anno_id
}))
def review_index_page(user_id):
try:
user = User.objects.get(id=user_id)
except Exception as e:
return redirect('/404')
doc_map = dict()
annotations = Annotation.objects(user=user).order_by('-created_at')
for annotation in annotations:
try:
# for situation in which the annotated document was deleted
doc = annotation.doc
except Exception:
continue
if not (doc.id in doc_map):
sent_total = Sent.objects(doc=doc).count()
annotation_sent_total = Annotation.objects(doc=doc, user=user, type='sentence').count()
doc_map[doc.id] = {
'doc': doc,
'sent_total': sent_total,
'annotation_sent_total': annotation_sent_total,
'progress': annotation_sent_total / sent_total * 100,
'annotation_total': Annotation.objects(doc=doc, user=user).count(),
'review_total': AnnotationReview.objects(doc=doc, user=g.user).count(),
}
return render_template('review/index.html', doc_map=doc_map, user=user, g=g)
def annotations(request):
if request.method == "POST":
received_annotation = json.loads(request.body)
findex = received_annotation['findex']
stuid = received_annotation['stuid']
ftype = received_annotation.get('ftype', 0)
atype = received_annotation.get('atype', 0)
mid = received_annotation.get('mid', 0)
del received_annotation['findex']
del received_annotation['stuid']
received_annotation.pop('ftype', None)
received_annotation.pop('atype', None)
received_annotation.pop('mid', None)
# del received_annotation['ftype']
# del received_annotation['atype']
annotation = Annotation(user_id=request.user.id,
ftype=ftype,
findex=findex,
stuid=stuid,
atype=atype,
mid=mid,
annotation=json.dumps(received_annotation))
annotation.save()
return HttpResponseSeeOtherRedirect('/annotate/annotations/' +
str(annotation.id))
# 取得目前使用者所有標註資料。(理論上不會執行到這邊，應該會從 search 僅取得目前頁面的標註資料)
annotations = [
a for a in Annotation.objects.filter(
user_id=request.user.id).order_by('id')
]
rows = []
for annotation in annotations:
rows.append(json.loads(annotation.annotation))
return JsonResponse(rows, safe=False)
def create_tables():
from models import Image, Imageuri, ImageFragment, Metadata, Annotation
Image.create_table()
Imageuri.create_table()
ImageFragment.create_table()
Metadata.create_table()
Annotation.create_table()
def add_annotation(self, metadata, key, value):
""" Adds an Annotation to a Metadata Object
of one namespace
if the key is already included the value
will be updated. if the key is not there, a new annotation
will be created
"""
try:
Annotation.get(imgmeta=metadata, an_key=key)
newimgmetaversion = metadata.version
newimgmetaversion = newimgmetaversion + 1
meta_query = metadata.update(version=newimgmetaversion)
meta_query.execute()
anno_query = Annotation.update(an_value=value).where(imgmeta=metadata, an_key=key)
anno_query.execute()
return
except Annotation.DoesNotExist:
imganno = Annotation.create(
imgmeta = metadata,
an_key = key,
an_value = value,)
imganno.save()
return imganno
def get(self, anno_id=''):
user = users.get_current_user()
if anno_id == "":
a = Annotation.all().order("-date").get()
else:
a = Annotation.all().filter('anno_id = ', anno_id).get()
if a is None: a = Annotation()
user_tmpl = user_stuff(self.request.uri)
history = [x for x in Annotation.all().order("-date") if x.title][:12]
name = user and user.nickname() or ""
user_history = [x for x in Annotation.gql("WHERE author = :author", \
author=user) if x.title][:12] if user else None
bar_hists = a.get_bar_hists()
query = Annotation.all()
gobe_examples = [Annotation.by_anno_id('8316edf46cc9b4ec845596e91997630f'),
Annotation.by_anno_id('50f939d37ebf65f4588ba5bc435e7bdf'),
Annotation.by_anno_id('e58139270757f77ef25ddcfa0df8e61b'),
]
self.response.out.write(render("index.html", user_tmpl, anno_name=name, anno_id=a.anno_id or "",
anno_content=a.content or "", anno_title=a.title or "",
bar_hists = bar_hists,
gobe_examples=gobe_examples,
history=history, user_history=user_history))
def annotate(request, id):
u = User.objects.get(id=1)
aud = AudioMeta.objects.get(id=id)
t = request.POST['t']
a = open("/home/leegao/webapps/notetag/notetag/test.txt", "w")
a.write(t)
a.close()
m = request.POST['data']
an = Annotation(user_id=u, aud_id=aud, time=int(t), msg=m)
an.save()
return HttpResponse(content="0")
def annotate(request, id):
u = User.objects.get(id=1)
aud = AudioMeta.objects.get(id=id)
t = request.POST["t"]
a = open("/home/leegao/webapps/notetag/notetag/test.txt", "w")
a.write(t)
a.close()
m = request.POST["data"]
an = Annotation(user_id=u, aud_id=aud, time=int(t), msg=m)
an.save()
return HttpResponse(content="0")
def get(self, anno_id):
# they can request a certain format by ending with .gobe whatever.
self.response.headers['Content-type'] = 'text/plain'
if anno_id == "":
a = Annotation.all().order("-date").get()
elif anno_id != "all":
a = Annotation.all().filter('anno_id = ', anno_id).get()
if a is None: a = Annotation()
content = a.content
if a.format and a.format != "gobe":
content = [x.strip() for x in content.split("\n")]
content = utils.main(content, a.format, force_tabs=True)
self.response.out.write(content or "")
def __init__(self,
pkl_path='./data/pkl/annotations_clustering.pkl',
redundant=True):
import os, pickle, logging
from models import Annotation
from mongoengine import connect
import config
connect(**config.Config.MONGODB_SETTINGS)
self.redundant = redundant
dumps = []
if os.path.exists(pkl_path):
dumps = pickle.load(open(pkl_path, "rb"))
else:
print('generate pkl' + pkl_path)
annotations = Annotation.objects(type='sentence')
for annotation in tqdm(annotations):
try:
row = annotation.dump()
user = annotation.user
row['user_name'] = '{} {}'.format(user.first_name,
user.last_name)
dumps.append(row)
except Exception as e:
logging.exception(e)
pickle.dump(dumps, open(pkl_path, "wb"))
random.shuffle(dumps)
self.annotations = dumps
self.build_map()
def delete_duplicate_annotations():
annotations = Annotation.objects().all()
progress = 0
total = annotations.count()
for annotation in annotations:
progress += 1
if progress % 100 == 0:
print('progress {}/{}'.format(progress, total))
try:
targets = Annotation.objects.filter(
doc=annotation.doc,
sent=annotation.sent,
index=annotation.index,
user=annotation.user,
type=annotation.type,
anchor_offset=annotation.anchor_offset)
if targets.count() >= 2:
print('count >= 2! :', targets.count())
if targets[0].id != annotation.id:
targets[0].delete()
else:
targets[1].delete()
except:
pass
def download_dataset():
docs = Doc.objects.filter()
data = []
for doc in tqdm(docs):
annotations = Annotation.objects(doc=doc)
for annotation in annotations:
data.append({
'annotator': annotation.user.username,
'doc_id': doc.seq,
'version': doc.type,
'turker_id': annotation.user.turker_id,
'sentence_index': annotation.index,
'sentence': annotation.entire_text,
'annotation_anchor_offset': annotation.anchor_offset,
'annotation_focus_offset': annotation.focus_offset,
'annotation_target_text': annotation.target_text,
'annotation_type': annotation.type,
'attributes': annotation.basket,
'title': doc.title,
'source': doc.source,
'created_at': annotation.created_at,
})
dataset_path = os.path.abspath(os.path.dirname(__file__) + '/dataset.json')
data_json = json.dumps(data, default=json_util.default, indent=2)
with open(dataset_path, 'w', encoding='utf-8') as f:
f.write(data_json)
return send_file(dataset_path, as_attachment=True)
def put_review_annotation(annotation_id):
data = request.get_json()
basket = data['basket']
try:
annotation = Annotation.objects().get(id=annotation_id)
except Exception:
return Response('404', status=404)
try:
annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
except AnnotationReview.DoesNotExist:
annotation_review = AnnotationReview(annotation=annotation, user=g.user)
review_basket = dict()
for key in basket:
if '-review' in key:
review_basket[key] = basket[key]
annotation_review.doc = annotation.doc
annotation_review.ip = request.remote_addr
annotation_review.basket = review_basket
annotation_review.updated_at = datetime.datetime.now
annotation_review.save()
annotation.basket = utils.merge_dict(annotation.basket, review_basket)
return json.dumps({
'annotation': annotation.dump(),
})
def index_v2_page(doc_type):
item_per_page = 50
page = request.args.get('p', 1)
page = int(page)
total = Doc.objects.filter(type=doc_type).count()
total_page = math.ceil(total / item_per_page)
paginator = Pagination(Doc.objects(type=doc_type).order_by('seq'), page, 50)
docs = paginator.items
docs_data = []
for doc in docs:
item = doc.dump()
item['sent_total'] = Sent.objects(doc=doc).count()
item['progress'] = Annotation.objects(doc=doc, user=g.user, type='sentence').count()
docs_data.append(item)
pagination = {
'page': page,
'total_page': total_page,
'left': max(1, page - 5),
'right': min(page + 5, total_page),
}
return render_template('index.html', type=doc_type, docs=docs_data, g=g, pagination=pagination)
def import_csv(filename):
result = []
with codecs.open(filename, 'r', 'utf-8') as f:
word_columns = []
index_columns = []
sentence_column = 0
reader = csv.reader(f, delimiter=';')
for i, row in enumerate(reader):
if i == 0: # header row
for j, r in enumerate(row):
if r.startswith('w'):
word_columns.append(j)
if r.startswith('index'):
index_columns.append(j)
if r.startswith('full'):
sentence_column = j
else: # other rows
pk = int(row[0])
words = [row[c] for c in word_columns]
indices = [row[c] for c in index_columns]
sentence = row[sentence_column]
result.append(Annotation(pk, words, indices, sentence))
return result
def download_dataset_AMT_v3():
docs = Doc.objects.filter(type='mturk_v3')
data = []
for doc in tqdm(docs):
annotations = Annotation.objects(doc=doc)
for annotation in annotations:
data.append({
'annotator': annotation.user.username,
'version': doc.type,
'turker_id': annotation.user.turker_id,
'doc_id': str(doc.id),
'sentence_index': annotation.index,
'sentence': annotation.entire_text,
'basket': annotation.basket,
'source': doc.source,
'created_at': annotation.created_at,
})
dataset_path = os.path.abspath(os.path.dirname(__file__) + '/dataset_AMT_v3.json')
data_json = json.dumps(data, default=json_util.default, indent=2)
with open(dataset_path, 'w', encoding='utf-8') as f:
f.write(data_json)
return send_file(dataset_path, as_attachment=True)
def ins_ann(topic_id):
"""
Request Includes annotations and corresponding text.
Should insert annotation.
params:
"""
try:
data = json.loads(request.data)
ann = Annotation.create(parent=topic_id,
start_index=data['start_index'],
end_index=data['end_index'],
start_paragraph=data['start_paragraph'],
end_paragraph=data['end_paragraph'],
body=data['body'],
deltas=0)
ann.save()
except json.JSONDecodeError as e:
stderr.write("Unable to parse JSON request from /annotations/ \
reason: {}".format(str(e)))
except IntegrityError as e:
stderr.write("Integrity error when inserting annotation: {}".format(
str(e)))
abort(400)
"""Do something with annotations."""
return Response("Hi", status=200, mimetype="text/plain")
def get(self, anno_id):
# they can request a certain format by ending with .gobe whatever.
self.response.headers['Content-type'] = 'text/plain';
if anno_id == "":
a = Annotation.all().order("-date").get()
elif anno_id != "all":
a = Annotation.all().filter('anno_id = ', anno_id).get()
if a is None: a = Annotation()
content = a.content
if a.format and a.format != "gobe":
content = [x.strip() for x in content.split("\n")]
content = utils.main(content, a.format, force_tabs=True)
self.response.out.write(content or "")
def _addAnnotation(sentence,
annotation,
user,
prevSentence=None,
nextSentence=None):
a = Annotation(sentence, annotation, user, prevSentence, nextSentence)
db_session.add(a)
db_session.commit()
def get_ann_meta(topic_id):
"""Return annotation meta data."""
annotations = Annotation.select().where(
Annotation.parent == topic_id).order_by(Annotation.deltas)
data = [model_to_dict(ann) for ann in annotations]
return jsonify(data)
def index():
user: User = current_user
facial_exprssions_translations = None
if user.language == 'english':
facial_exprssions_translations = utils.english_fe
elif user.language == 'persian':
facial_exprssions_translations = utils.persian_fe
elif user.language == 'filipino':
facial_exprssions_translations = utils.filipino_fe
print(user.language, '**********')
print(user.username, ' %%%%%%%%')
if request.method == "POST": # if the request is post (i.e. new video annotated?)
# print(request.form)
form = request.form
annotation: Annotation = Annotation()
annotation.emotion = form['emotion']
annotation.anger_score = form['anger']
annotation.contempt_score = form['contempt']
annotation.disgust_score = form['disgust']
annotation.annoyed_score = form['annoyed']
annotation.gender = form['gender']
annotation.filename = form.get("token")
user.add_video(form.get("token"))
social_signals = ','.join([ss for ss in form.getlist('socialsignal')])
extra_ss = form.get("extra")
## TODO: should be tested
if len(extra_ss) > 0:
social_signals += ',%s' % extra_ss
annotation.confidence = form['confidence']
annotation.annotator_culture = user.culture
annotation.annotator_language = user.language
annotation.annotator_individuality = user.individuality
annotation.social_signals = social_signals
db.session.add(annotation)
db.session.commit()
vid = utils.get_random_video(user.culture, user.get_annotated_videos())
completed = utils.get_completed_videos(user.culture, user.get_annotated_videos())
if vid == "FINISHED":
return render_template('thankyou.html')
print("Annotation %s created :)" % annotation)
return render_template('index.html', context={'video':vid, 'language': user.language, 'completed': completed,
'expressions': facial_exprssions_translations})
vid = utils.get_random_video(user.culture, user.get_annotated_videos())
completed = utils.get_completed_videos(user.culture, user.get_annotated_videos())
if vid == "FINISHED":
return render_template('thankyou.html')
return render_template('index.html', context={'video':vid, 'language': user.language, 'completed': completed,
'expressions': facial_exprssions_translations})
def delete_review_annotation(annotation_id):
try:
annotation = Annotation.objects().get(id=annotation_id)
annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
annotation_review.delete()
except AnnotationReview.DoesNotExist:
return Response('not found', status=404)
return Response('success', status=200)
def delete_annotation(annotation_id):
try:
annotation = Annotation.objects().get(id=annotation_id)
except Annotation.DoesNotExist:
return Response('not found', status=404)
if annotation.user.id != g.user.id:
return Response('permission error', status=403)
annotation.delete()
return Response('success', status=200)
def get_attribute_reason():
"""
attribute_reason = {
'Knowledge_Awareness': {
'I_did_not_know_the_information': [],
},
}
"""
attribute_reason = defaultdict(lambda: defaultdict(lambda: []))
dumps = []
bin_path = './data/pkl/annotations_with_sent.pkl'
if os.path.exists(bin_path):
dumps = pickle.load(open(bin_path, "rb"))
else:
print('generate ' + bin_path)
annotations = Annotation.objects(type='sentence')
for annotation in tqdm(annotations):
try:
row = annotation.dump()
row['sentence'] = annotation.sent.text
row['user_name'] = annotation.user.first_name + ' ' + annotation.user.last_name
dumps.append(row)
except Exception as e:
logging.exception(e)
pickle.dump(dumps, open(bin_path, "wb"))
random.shuffle(dumps)
reason_set = set()
for annotation in tqdm(dumps):
basket = annotation['basket']
for attribute_key in basket:
option = basket[attribute_key]
if not ('reason' in option):
continue
attribute_value = option['value']
reason = option['reason']
if not reason:
continue
option['user'] = annotation['user']
option['user_name'] = annotation['user_name']
option['sentence'] = annotation['sentence']
# reason_key = '{}-{}'.format(option['user'], ''.join(tokenize_and_lemmatize(reason)))
# if reason_key in reason_set:
# continue
# reason_set.add(reason_key)
attribute_reason[attribute_key][attribute_value].append(option)
return attribute_reason
def post(self, unused):
self.response.headers['Content-type'] = 'text/javascript';
user = users.get_current_user()
annos = self.request.get('annos', '').strip()
format = self.request.get('format', utils.guess_format(annos, force_tabs=True))
if not annos:
simplejson.dumps({'status': 'fail'})
return
name = user and user.nickname() or ""
title = self.request.get('title') or ""
anno_id = hashlib.md5(annos + title).hexdigest()
# dont save unless it's new.
a = Annotation.gql("WHERE anno_id = :anno_id AND title = :title AND author = :author",
anno_id=anno_id, title=title, author=user).get()
if a is None:
a = Annotation(title=title, author=user, content=annos.strip(), anno_id=anno_id,
format=format)
a.put()
self.response.out.write(simplejson.dumps({'status':'success', 'anno_id': anno_id}))
def index1(request):
if request.method=='POST':
print request.body
inputToDB=json.loads(request.body)
json.dumps("annotMe.txt");
print inputToDB
myTemp=open('san.txt','w')
myTemp.write(str(inputToDB))
myTemp.close()
for anData in inputToDB:
a=Annotation(x=anData['x'],y=anData['y'],msg=anData['msg'])
print a.msg
#making a persistent storage
a.save()
# a=inputToDB
#print a
#dict = simplejson.JSONDecoder().decode( JSONdata )
#print request.POST['data']
print '---------Request'
#print request
return HttpResponse("Ur request is processed!!!")
def ins_com(annotation_id):
"""Insert comment on annotation."""
ann_comm = None
try:
data = json.loads(request.data)
ann = Annotation.get_by_id(annotation_id)
comm = Comment(parent=annotation_id, body=data['comment'], deltas=0)
comm.save()
except json.JSONDecodeError as e:
stderr.write("Unable to parse JSON request when attempting to \
add comment, reason: {}".format(str(e)))
return Response("Hello", status=200, mimetype="text/plain")
def get_annotation(doc_id):
try:
doc = Doc.objects().get(id=doc_id)
annotations = Annotation.objects(doc=doc, user=g.user)
except Exception as e:
return Response('not found', status=404)
data = []
for annotation in annotations:
data.append(annotation.dump())
return json.dumps({
'annotations': data,
})
def mod_ann_delta(annotation_id, up_or_down):
"""Add a delta to an annotation or comment"""
try:
ann = Annotation.get_by_id(annotation_id)
if up_or_down == 'true':
ann.deltas += 1
else:
if ann.deltas > 0:
ann.deltas -= 1
ann.save()
except pw.DoesNotExist as e:
stderr.write("Unable to find annotation_id in db: {}").format(str(e))
return Response("Yo", status=200, mimetype="text/plain")
def get_ann_text(topic_id, annotation_id):
"""Get text for annotation and comments."""
data = ''
try:
annotations = Annotation.select().where(
Annotation.parent == topic_id
and Annotation.id == annotation_id).order_by(Annotation.deltas)
data = [model_to_dict(ann) for ann in annotations][0]
except pw.DoesNotExist as e:
stderr.write(
"Unable to find topic_id or annotation_id in db: {}").format(
str(e))
except IndexError:
stderr.write("Index out of bounds, that's embarressing")
return jsonify(data)
def put_annotation(annotation_id):
data = request.get_json()
basket = data['basket']
try:
annotation = Annotation.objects().get(id=annotation_id)
except Exception:
return Response('not found', status=404)
if annotation.user.id != g.user.id:
return Response('permission error', status=403)
annotation.basket = basket
annotation.updated_at = datetime.datetime.now
annotation.save()
return json.dumps({
'annotation': annotation.dump(),
})
def add_answer(request, stimulus_index, answer_pk):
response = {}
#Add the answer to the database
answer = Answer.objects.get(pk=answer_pk)
Annotation(question_nr=stimulus_index, answer=answer).save()
#Check and return whether the answer is also correct
correct_pk = open(STIMULI_FILE_LOCATION).readlines()[int(
stimulus_index)].strip().split('\t')[1]
response['correct'] = correct_pk == answer_pk
print('Received', request.POST)
if response['correct']:
minimum_streak_to_enter_highscore = calculate_minimum_streak_to_enter_highscore(
)
#First, see whether this is info to add to an existing streak
if request.POST['streak_id'] not in [
None, 'undefined', 'null', False, 'false'
]:
current_streak = Streak.objects.filter(
pk=request.POST['streak_id'])[0]
current_streak.length = request.POST['streak_length']
current_streak.save()
response['streak_id'] = current_streak.pk
#if not, check whether you made the highscore
elif int(request.POST['streak_length']
) >= minimum_streak_to_enter_highscore:
#If so, add the streak to the database
new_streak = Streak(playername='Anonymous',
length=int(request.POST['streak_length']))
new_streak.save()
response['streak_id'] = new_streak.pk
response['ask_for_name'] = True
#else, do nothing specail
else:
response['streak_id'] = False
return HttpResponse(json.dumps(response))
def image_detail(imageid):
"""
function that is when a specific image is requested
the metadata of the requested image is loaded (if there is one)
and given to the template
"""
try:
image = Image.get(id=imageid)
imgfrag = ImageFragment.get(image=image, x=0, y=0, visible=True)
imgmeta = Metadata.get(imgfrag=imgfrag, namespace="http://www.w3.org/ns/ma-ont#")
annos = Annotation.get(imgmeta=imgmeta)
return render_template('image_detail.html', img=image,imgmeta=imgmeta)
except Imageuri.DoesNotExist:
flash('Die Bildnummer wurde nicht gefunden. Bitte waehle ein anderes Bild.', 'error')
return render_template('images.html')
except ImageFragment.DoesNotExist:
return render_template('image_detail.html', img=image)
def change_all_attribute_key():
annotations = Annotation.objects().all()
for annotation in tqdm(annotations):
basket = annotation.basket
new_basket = dict()
for key in basket:
new_key = key
value = basket[key]
if key == 'Disputability_of_the_sentence':
new_key = 'Disputability'
elif key == 'Perceived_Author_Credibility_for_the_upcoming_sentences':
new_key = 'Perceived_Author_Credibility'
elif key == 'Acceptance_of_the_sentence_as_true':
new_key = 'Acceptance'
new_basket[new_key] = value
annotation.basket = new_basket
annotation.save()
def get_annotations():
dumps = []
bin_path = './data/bin/annotations_time.bin'
if os.path.exists(bin_path):
dumps = pickle.load(open(bin_path, "rb"))
else:
print('generate ' + bin_path)
annotations = Annotation.objects(type='sentence')
for annotation in tqdm(annotations):
try:
if not ('Acceptance' in annotation['basket']):
continue
dump = annotation.dump()
dump['is_turk'] = not annotation.user.is_active
dumps.append(dump)
except Exception as e:
logging.exception(e)
pickle.dump(dumps, open(bin_path, "wb"))
random.shuffle(dumps)
return dumps
def get_review_annotation(user_id, doc_id):
try:
doc = Doc.objects.get(id=doc_id)
user = User.objects.get(id=user_id)
except Exception as e:
return redirect('/404')
annotations = Annotation.objects(doc=doc, user=user)
data = []
for annotation in annotations:
try:
annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
annotation.basket = utils.merge_dict(annotation.basket, annotation_review.basket)
except AnnotationReview.DoesNotExist:
pass
data.append(annotation.dump())
return json.dumps({
'annotations': data,
})
def export_dataset_v3():
docs = Doc.objects.filter(type='mturk_v3')
data = []
for doc in tqdm(docs):
sents = Sent.objects(doc=doc)
annotations = Annotation.objects(doc=doc)
if sents.count() != annotations.count():
continue
source = doc.source
if 'aljazeera' not in source and 'foxnews' not in source and 'theguardian' not in source:
continue
if not is_ok(annotations):
continue
for annotation in annotations:
data.append({
'annotator': annotation.user.username,
'version': doc.type,
'turker_id': annotation.user.turker_id,
'doc_id': str(doc.id),
'sentence_index': annotation.index,
'sentence': annotation.entire_text,
'basket': annotation.basket,
'source': doc.source,
'created_at': annotation.created_at,
})
dataset_path = os.path.abspath(
os.path.dirname(__file__) + '/../data/dataset_AMT_v3.json')
data_json = json.dumps(data, default=json_util.default)
with open(dataset_path, 'w', encoding='utf-8') as f:
f.write(data_json)
def lcia_dispatcher(product, method):
"""
same again
"""
uR, annot = random_result("float")
# save the result
R = RealResult(data_type = "float",
unit = uR.unit,
mean_value = uR.value,
st_dev = uR.stdev,
confidence_interval_5 = uR.conf5(),
confidence_interval_95 = uR.conf95())
R.save()
A=Annotation(uuid=R.uuid,relation=method.name,annotation=product.name + " | " + annot)
A.save()
Aa=Annotation(uuid=R.uuid,relation='created',annotation=now())
Aa.save()
return R
def module_dispatcher(prop, chemical, product=None):
"""
first argument is the property to be computed-- should be an instance of models.Property
second argument is the chemical the property is computed about-- instance of models.Chemica
third argument is the product system in scope-- instance of models.Product. not implemented.
This function is only called when a new result is required. Invoke the appropriate module
and return the result! make a random one for now.
"""
M = prop.module
# these should all return a ( uResult , annotation )
uR, annot = {
'Module 01': (None, 'dummy'),
'Module 02': (None, 'dummy'),
'ECHA CombinedTox': echa_tox_lookup(prop, chemical)
}.get(M.name,random_result(prop.data_type))
if uR is None:
return annot
# save the result
R = RealResult(data_type = prop.data_type,
unit = uR.unit,
mean_value = uR.value,
st_dev = uR.stdev,
confidence_interval_5 = uR.conf5(),
confidence_interval_95 = uR.conf95())
R.save()
A=Annotation(uuid=R.uuid,relation='result',annotation=annot)
A.save()
Aa=Annotation(uuid=R.uuid,relation='created',annotation=now())
Aa.save()
return R
def ImportGFF(specimen, file):
fileName = file.split('/')
fileName = fileName[-1].split('.')[0]
specimenObject = GetSpecimen(specimen)
gff, created = GFF.objects.get_or_create(Specimen=specimenObject, FileName=fileName)
print gff.FileName
#Get the specimen this annotation file is for
gff.Specimen = GetSpecimen(specimen)
#Set default version type
gff.GFFVersion = 2
#Open file and check for version type
#TODO: Grab GFF Info and store in database
annotationFile = open(file, 'r')
counter = 0
for line in annotationFile.readlines():
counter += 1
if line.startswith("##"):
if "gff-version" in line:
temp = line.split(' ')
gff.GFFVersion = temp[1]
if counter == 10:
break
annotationFile.close()
gff.save()
#Grab a list of chromosomes related to this specimen
validChromosomes = GetRelatedChromosomes(gff.Specimen)
annotations = {}
#In GFF format, each line is an annotation.
#Read in each annotation and split at tabs (as per gff spec)
#Then store as needed into an Annotation object which will be pushed onto the annotations stack
#TODO: Bust out version reading into methods
if gff.GFFVersion == 2:
print "BEGINNING READING OF VERSION 2 FILE...\n"
annotationFile = open(file, 'r')
counter = 0
for line in annotationFile.readlines():
counter += 1
if not line.startswith('#'):
elements = line.split('\t')
annotation = Annotation()
#TODO: Handle when values come back null from not finding a matching chromosome!
annotation.Specimen = specimen
annotation.Chromosome = ParseChromosomeName(validChromosomes,
elements[0]) #Related validChromosomes, chromosome
annotation.ID = counter
annotation.Source = elements[1]
annotation.Feature = elements[2]
annotation.Start = elements[3]
annotation.End = elements[4]
#Do checking for Score
if elements[5] == '.':
annotation.Score = None
else:
annotation.Score = elements[5]
#Do checking for Strand
if elements[6] == '.':
annotation.Strand = None
else:
annotation.Strand = elements[6]
#Do checking for frame
if elements[7] == '.':
annotation.Frame = None
else:
annotation.Frame = elements[7]
#Check for existence of attribute and possible extra length
if len(elements) >= 9:
annotation.Attribute = elements[8:]
else:
annotation.Attribute = None
if annotation.Chromosome in annotations:
#appent to the current list
annotations[annotation.Chromosome].append(annotation)
else:
#Create the chromosome entry in the annotations dictionary
annotations[annotation.Chromosome] = []
annotations[annotation.Chromosome].append(annotation)
if counter % 10000 == 0:
sys.stdout.write('.')
#print "RESULTS: ", annotation.Specimen, annotation.Source, annotation.Feature, annotation.Start, annotation.End, annotation.Score, annotation.Strand, annotation.Frame, annotation.Attribute
print "DONE READING FILE!"
else:
print "THIS GFF VERSION IS NOT SUPPORTED: Version", gff.GFFVersion
print "BEGINNING SORTING LIST..."
#Sort the list of annotations read in by their start value (this could probably be optimized by using an always ordered list and inserting in order above)
#annotations = sorted(annotations, key = lambda annotation: annotation.Start)
for sublist in annotations:
annotations[sublist] = sorted(annotations[sublist], key=lambda annotation: int(annotation.Start))
print "DONE SORTING!"
chunkAndStoreAnnotations(gff, annotations)
def functional_table():
form = FunctionClassFilterForm()
form.function_class.choices = [('cog', 'Cog'),
('pfam', 'Pfam'),
('tigrfam', 'TigrFam'),
('all', 'All')
]
form.select_sample_groups.choices = [(sample_set.name, sample_set.name) for sample_set in SampleSet.query.all()]
type_identifiers = []
if form.validate_on_submit():
function_class = form.function_class.data
if function_class == 'all':
function_class = None
limit = form.limit.data
if limit == 'all':
limit = None
else:
limit = int(limit)
filter_alternative = form.filter_alternative.data
if filter_alternative == 'filter_with_type_identifiers':
for type_identifier in form.type_identifiers.entries:
if type_identifier.data != '':
type_identifiers.append(type_identifier.data)
elif filter_alternative == 'filter_with_search':
search_string = form.search_annotations
if search_string.data != '':
q = _search_query(search_string.data)
type_identifiers = [a.type_identifier for a in q.all()]
sample_sets = form.select_sample_groups.data
if len(sample_sets) > 0:
samples = [sample.scilifelab_code for sample in Sample.all_from_sample_sets(sample_sets)]
else:
samples = None
download_action = False
if form.submit_download.data:
download_action = True
download_select = form.download_select.data
else:
function_class=None
limit=20
samples = None
download_action = False
if len(form.type_identifiers) == 0:
form.type_identifiers.append_entry()
if type_identifiers == []:
type_identifiers = None
samples, table = Annotation.rpkm_table(limit=limit, samples=samples, function_class=function_class, type_identifiers=type_identifiers)
samples = sorted(samples, key=lambda x: x.scilifelab_code)
sample_scilifelab_codes = [sample.scilifelab_code for sample in samples]
if download_action:
if download_select == 'Gene List':
# Fetch all contributing genes for all the annotations in the table
annotation_ids = [annotation.id for annotation, sample in table.items()]
genes_per_annotation = Annotation.genes_per_annotation(annotation_ids)
csv_output = '\n'.join(
[','.join([gene.name, annotation.type_identifier]) \
for gene, annotation in genes_per_annotation])
r = make_response(csv_output)
r.headers["Content-Disposition"] = "attachment; filename=gene_list.csv"
r.headers["Content-Type"] = "text/csv"
return r
return render_template('functional_table.html',
table=table,
samples=samples,
sample_scilifelab_codes = sample_scilifelab_codes,
form=form
)
def get_annotation(self, imgmeta):
return Annotation.select().where(imgmeta=imgmeta)
def _create_or_update(self, anno=None):
create = anno is None
if create:
anno = Annotation()
body = self._get_request_body()
anno.context_id = body['contextId']
anno.collection_id = body['collectionId']
anno.uri = body['uri']
anno.media = body['media']
anno.user_id = body['user']['id']
anno.user_name = body['user']['name']
anno.is_private = False if len(body.get('permissions', {}).get('read', [])) == 0 else True
anno.text = body.get('text', '')
anno.quote = body.get('quote', '')
anno.json = json.dumps(body)
if 'parent' in body and body['parent'] != '0':
anno.parent_id = int(body['parent'])
anno.save()
if create and anno.parent_id:
parent_anno = Annotation.objects.get(pk=int(body['parent']))
parent_anno.total_comments = F('total_comments') + 1
parent_anno.save()
if not create:
anno.tags.clear()
for tag_name in body.get('tags', []):
tag_object, created = AnnotationTags.objects.get_or_create(name=tag_name.strip())
anno.tags.add(tag_object)
return anno
def _handle_post_annotation_request(user, document, request):
if not user.is_authenticated:
return HttpResponseForbidden()
annotation = Annotation()
annotation.annotator = user
annotation.content = sanitize(request.POST['annotation_content'])
annotation.document_this_annotation_belongs = document
annotation.page_index = request.POST["page_id"].split("_")[2]
annotation.height_percent = request.POST["height_percent"]
annotation.width_percent = request.POST["width_percent"]
annotation.top_percent = request.POST["top_percent"]
annotation.left_percent = request.POST["left_percent"]
annotation.frame_color = request.POST["frame_color"]
annotation.is_public = True if request.POST[
"is_public"] == 'true' else False
annotation.save()
# send notification to the document uploader
if annotation.annotator.pk != document.owner.pk:
notify.send(
sender=annotation.annotator,
recipient=document.owner,
action_object=annotation,
verb='post annotation',
redirect_url=annotation.url,
image_url=annotation.annotator.portrait_url,
description=h.handle(annotation.content),
is_public=annotation.is_public,
)
# send notification to the collectors, i.e., followers
for user in document.collectors.all():
if annotation.annotator.pk != user.pk and document.owner.pk != user.pk:
notify.send(
sender=annotation.annotator,
recipient=user,
action_object=annotation,
verb='post annotation',
redirect_url=annotation.url,
image_url=annotation.annotator.portrait_url,
description=h.handle(annotation.content),
is_public=annotation.is_public,
)
context = {
"document": document,
'annotation': annotation,
'ANONYMOUS_USER_PORTRAIT_URL': settings.ANONYMOUS_USER_PORTRAIT_URL,
"new_annotation_id": annotation.id,
}
return JsonResponse(
{
'new_annotationdiv_html':
render(request, "file_viewer/one_annotation_div.html",
context).content,
'new_annotation_id':
annotation.id,
'new_annotation_uuid':
str(annotation.clean_uuid),
'new_annotation_json':
annotation,
},
encoder=AnnotationEncoder)
def test_annotation_rpkm_table(self):
annotation_types = [("Cog", {'class': Cog}),
("Pfam", {'class': Pfam}),
("TigrFam", {'class': TigrFam}),
("EcNumber", {'class': EcNumber})]
nr_annotation_types = len(annotation_types)
annotation_sources = {}
for annotation_type, type_d in annotation_types:
annotation_sources[annotation_type]= AnnotationSource(
annotation_type,
"v1.0",
"rpsblast",
"e_value=0.000001"
)
sample1 = Sample("P1993_101", None, None)
sample2 = Sample("P1993_102", None, None)
nr_samples = 2
for i in range(50):
gene1 = Gene("gene1{}".format(i), None)
gene2 = Gene("gene2{}".format(i), None)
gene_count1 = GeneCount(gene1, sample1, 0.001)
gene_count2 = GeneCount(gene1, sample2, 0.01)
gene_count3 = GeneCount(gene2, sample1, 0.002)
gene_count4 = GeneCount(gene2, sample2, 0.02)
for annotation_type, type_d in annotation_types:
if annotation_type == 'Cog':
type_id = str(i)
type_id = "0"*(4-len(type_id))+type_id
annotation = type_d['class'](annotation_type.upper() + type_id, "H")
elif annotation_type == 'EcNumber':
if i > 25:
type_id = "0.0.2.{}".format(i)
else:
type_id = "0.0.0.{}".format(i)
annotation = type_d['class'](type_id)
else:
type_id = str(i)
type_id = "0"*(4-len(type_id))+type_id
annotation = type_d['class'](annotation_type.upper() + type_id)
annotation_mode = i % 3
gene_annotations = []
if annotation_mode in [0,1]:
gene_annotations.append(GeneAnnotation(
annotation,
gene1,
annotation_sources[annotation_type]
))
if annotation_mode in [1,2]:
gene_annotations.append(GeneAnnotation(
annotation,
gene2,
annotation_sources[annotation_type]
))
self.session.add_all(gene_annotations)
self.session.add(gene1)
self.session.add(gene2)
self.session.commit()
refresh_all_mat_views()
samples, rows = Annotation.rpkm_table()
assert len(samples) == 2
assert len(rows) == 20 # Default limit
samples, rows = Annotation.rpkm_table(limit=100)
assert len(samples) == 2
assert len(rows) == 100
samples, rows = Annotation.rpkm_table(limit=None)
assert len(samples) == 2
assert len(rows) == nr_annotation_types * 50
for annotation, sample_d in rows.items():
# sample_d should be a ordered dict
assert ["P1993_101", "P1993_102"] == [sample.scilifelab_code for sample in sample_d.keys()]
rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]
rpkms_flat = []
for rpkm_row in rpkms:
rpkms_flat += rpkm_row
assert len(rpkms_flat) == nr_annotation_types * nr_samples * 50
# Annotations sorted by total rpkm over all samples
# and the rpkm values should be summed over all genes for that annotation
# there should be roughly equal numbers of the three different counts
for i, row in enumerate(rpkms[:67]):
assert row == [0.003, 0.03]
for row in rpkms[69:130]:
assert row == [0.002, 0.02]
for row in rpkms[150:200]:
assert row == [0.001, 0.01]
# possible to filter on function classes
for annotation_type, type_d in annotation_types:
samples, rows = Annotation.rpkm_table(limit=None, function_class=annotation_type.lower())
assert len(rows) == 50
for key in rows.keys():
assert annotation_type[:3].lower() == key.annotation_type[:3]
# possible to filter on samples
for sample in [sample1, sample2]:
samples, rows = Annotation.rpkm_table(samples=[sample.scilifelab_code], limit=None)
assert len(rows) == 200
assert len(samples) == 1
assert samples[0] == sample
for annotation, sample_d in rows.items():
assert list(sample_d.keys()) == [sample]
rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]
if sample.scilifelab_code == "P1993_101":
for i, row in enumerate(rpkms[:65]):
assert row == [0.003]
for row in rpkms[69:130]:
assert row == [0.002]
for row in rpkms[150:200]:
assert row == [0.001]
else:
for row in rpkms[:67]:
assert row == [0.03]
for row in rpkms[69:130]:
assert row == [0.02]
for row in rpkms[150:200]:
assert row == [0.01]
# possible to filter on sample and function class at the same time
for annotation_type, type_d in annotation_types:
for sample in [sample1, sample2]:
samples, rows = Annotation.rpkm_table(limit=None, function_class=annotation_type.lower(), samples=[sample.scilifelab_code])
assert len(rows) == 50
for key in rows.keys():
assert annotation_type.lower()[:3] == key.annotation_type[:3]
assert len(samples) == 1
assert samples[0] == sample
for annotation, sample_d in rows.items():
assert list(sample_d.keys()) == [sample]
rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]
if sample.scilifelab_code == "P1993_101":
for row in rpkms[:9]:
assert row == [0.003]
for row in rpkms[19:29]:
assert row == [0.002]
for row in rpkms[39:]:
assert row == [0.001]
else:
for row in rpkms[:9]:
assert row == [0.03]
for row in rpkms[19:29]:
assert row == [0.02]
for row in rpkms[39:]:
assert row == [0.01]
# possible to filter on individual annotations
annotation_ids = ["COG0001", "TIGRFAM0004", "COG0003", "PFAM0002", "0.0.2.26"]
for r in range(5):
for type_identifiers in itertools.combinations(annotation_ids, r+1):
samples, rows = Annotation.rpkm_table(limit=None, type_identifiers=list(type_identifiers))
assert len(samples) == 2
assert len(rows) == len(type_identifiers)
assert set([key.type_identifier for key in rows.keys()]) == set(type_identifiers)
def test_annotation(self):
annotation = Annotation("COG0001")
self.session.add(annotation)
self.session.commit()
assert Annotation.query.first() is annotation
#Test the many to many relationship
reference_assembly = ReferenceAssembly("version 1")
gene = Gene("gene1", reference_assembly)
gene2 = Gene("gene2", reference_assembly)
gene3 = Gene("gene3", reference_assembly)
annotation2 = Annotation("COG0002", description="This cog is really really good")
# Test having multiple genes to one annotation
annotation_source = AnnotationSource("Cog", "v1.0", "rpsblast", "e_value=0.000001")
gene_annotation1 = GeneAnnotation(annotation_source = annotation_source, e_value=0.0000001)
gene_annotation2 = GeneAnnotation(annotation_source = annotation_source)
gene_annotation1.gene = gene
gene_annotation2.gene = gene2
gene_annotation1.annotation = annotation
gene_annotation2.annotation = annotation
self.session.add(annotation)
self.session.add(gene3)
self.session.add(gene_annotation1)
self.session.add(gene_annotation2)
self.session.commit()
annotation_01 = Annotation.query.filter_by(type_identifier="COG0001").first()
assert len(annotation_01.genes) == 2
assert gene in annotation_01.genes
assert gene2 in annotation_01.genes
assert annotation in Gene.query.filter_by(name="gene1").first().annotations
assert annotation in Gene.query.filter_by(name="gene2").first().annotations
assert len(Gene.query.filter_by(name="gene3").first().annotations) == 0
# Genes for annotation method
genes_for_annotation = Annotation.genes_per_annotation([annotation.id])
assert len(genes_for_annotation) == 2
assert (gene, annotation) in genes_for_annotation
assert (gene2, annotation) in genes_for_annotation
# Add the second annotation
self.session.add(annotation2)
self.session.commit()
q = Annotation.query.filter(Annotation.description.contains("good"))
annotation_02 = q.all()
assert len(annotation_02) == 1
assert annotation_02[0] == annotation2
# Test having multiple annotations to one gene
gene_annotation3 = GeneAnnotation(annotation2, gene, annotation_source, e_value = 1e-14)
self.session.add(gene_annotation3)
self.session.commit()
assert len(Gene.query.filter_by(name="gene1").first().annotations) == 2
assert annotation in Gene.query.filter_by(name="gene1").first().annotations
assert annotation2 in Gene.query.filter_by(name="gene1").first().annotations
assert gene_annotation1.e_value > gene_annotation3.e_value
assert gene.e_value_for(annotation) > gene.e_value_for(annotation2)
# gene -> annotation
# gene2 -> annotation
# gene -> annotation2
# Genes for annotation method
genes_for_annotation = Annotation.genes_per_annotation([annotation.id])
assert len(genes_for_annotation) == 2
assert (gene, annotation) in genes_for_annotation
assert (gene2, annotation) in genes_for_annotation
genes_for_annotation = Annotation.genes_per_annotation([annotation2.id])
assert len(genes_for_annotation) == 1
assert (gene, annotation2) in genes_for_annotation
genes_for_annotation = Annotation.genes_per_annotation([annotation.id, annotation2.id])
assert len(genes_for_annotation) == 3
assert (gene, annotation) in genes_for_annotation
assert (gene, annotation2) in genes_for_annotation
assert (gene2, annotation) in genes_for_annotation
annotation3 = Annotation("COG0003", description=("This cog is really really good. I assure you, "
"really quite good. Among its capabilities I have to mention that its utterly suitable for "
"testing the description string, including the short description."))
assert len(annotation3.description) > 103
assert annotation3.short_description[-3:] == "..."
assert len(annotation3.short_description) == 103
assert annotation3.description[:100] == annotation3.short_description[:100]
