示例#1
0
    def post(self, unused):
        self.response.headers['Content-type'] = 'text/javascript'
        user = users.get_current_user()
        annos = self.request.get('annos', '').strip()
        format = self.request.get('format',
                                  utils.guess_format(annos, force_tabs=True))
        if not annos:
            simplejson.dumps({'status': 'fail'})
            return
        name = user and user.nickname() or ""
        title = self.request.get('title') or ""
        anno_id = hashlib.md5(annos + title).hexdigest()

        # dont save unless it's new.
        a = Annotation.gql(
            "WHERE anno_id = :anno_id AND title = :title AND author = :author",
            anno_id=anno_id,
            title=title,
            author=user).get()
        if a is None:
            a = Annotation(title=title,
                           author=user,
                           content=annos.strip(),
                           anno_id=anno_id,
                           format=format)
            a.put()

        self.response.out.write(
            simplejson.dumps({
                'status': 'success',
                'anno_id': anno_id
            }))
示例#2
0
def review_index_page(user_id):
    try:
        user = User.objects.get(id=user_id)
    except Exception as e:
        return redirect('/404')

    doc_map = dict()
    annotations = Annotation.objects(user=user).order_by('-created_at')

    for annotation in annotations:
        try:
            # for situation in which the annotated document was deleted
            doc = annotation.doc
        except Exception:
            continue

        if not (doc.id in doc_map):
            sent_total = Sent.objects(doc=doc).count()
            annotation_sent_total = Annotation.objects(doc=doc, user=user, type='sentence').count()
            doc_map[doc.id] = {
                'doc': doc,
                'sent_total': sent_total,
                'annotation_sent_total': annotation_sent_total,
                'progress': annotation_sent_total / sent_total * 100,
                'annotation_total': Annotation.objects(doc=doc, user=user).count(),
                'review_total': AnnotationReview.objects(doc=doc, user=g.user).count(),
            }

    return render_template('review/index.html', doc_map=doc_map, user=user, g=g)
示例#3
0
def annotations(request):
    if request.method == "POST":
        received_annotation = json.loads(request.body)
        findex = received_annotation['findex']
        stuid = received_annotation['stuid']
        ftype = received_annotation.get('ftype', 0)
        atype = received_annotation.get('atype', 0)
        mid = received_annotation.get('mid', 0)
        del received_annotation['findex']
        del received_annotation['stuid']
        received_annotation.pop('ftype', None)
        received_annotation.pop('atype', None)
        received_annotation.pop('mid', None)
        # del received_annotation['ftype']
        # del received_annotation['atype']
        annotation = Annotation(user_id=request.user.id,
                                ftype=ftype,
                                findex=findex,
                                stuid=stuid,
                                atype=atype,
                                mid=mid,
                                annotation=json.dumps(received_annotation))
        annotation.save()
        return HttpResponseSeeOtherRedirect('/annotate/annotations/' +
                                            str(annotation.id))

    # 取得目前使用者所有標註資料。(理論上不會執行到這邊,應該會從 search 僅取得目前頁面的標註資料)
    annotations = [
        a for a in Annotation.objects.filter(
            user_id=request.user.id).order_by('id')
    ]
    rows = []
    for annotation in annotations:
        rows.append(json.loads(annotation.annotation))
    return JsonResponse(rows, safe=False)
示例#4
0
文件: app.py 项目: annebaui/srimato
def create_tables():
    from models import Image, Imageuri, ImageFragment, Metadata, Annotation
    Image.create_table()
    Imageuri.create_table()
    ImageFragment.create_table()
    Metadata.create_table()
    Annotation.create_table()
示例#5
0
 def add_annotation(self, metadata, key, value):
 
     """ Adds an Annotation to a Metadata Object
         of one namespace
         if the key is already included the value
         will be updated. if the key is not there, a new annotation
         will be created
     """ 
     
     try: 
         Annotation.get(imgmeta=metadata, an_key=key)
         newimgmetaversion = metadata.version
         newimgmetaversion = newimgmetaversion + 1
         meta_query = metadata.update(version=newimgmetaversion)
         meta_query.execute()
         
         anno_query = Annotation.update(an_value=value).where(imgmeta=metadata, an_key=key)
         anno_query.execute()
         return
     except Annotation.DoesNotExist:
         imganno = Annotation.create(
                             imgmeta = metadata,
                             an_key = key,
                             an_value = value,)
         imganno.save()
         return imganno
示例#6
0
    def get(self, anno_id=''):
        user = users.get_current_user()
        if anno_id == "":
            a = Annotation.all().order("-date").get()
        else:
            a = Annotation.all().filter('anno_id = ', anno_id).get()
        if a is None: a = Annotation()
        user_tmpl = user_stuff(self.request.uri)
        history = [x for x in Annotation.all().order("-date") if x.title][:12]
        name = user and user.nickname() or ""
        user_history = [x for x in Annotation.gql("WHERE author = :author", \
                              author=user) if x.title][:12] if user else None

        bar_hists = a.get_bar_hists()

        query = Annotation.all()
        gobe_examples = [Annotation.by_anno_id('8316edf46cc9b4ec845596e91997630f'),
                         Annotation.by_anno_id('50f939d37ebf65f4588ba5bc435e7bdf'),
                         Annotation.by_anno_id('e58139270757f77ef25ddcfa0df8e61b'),
                        ]

        self.response.out.write(render("index.html", user_tmpl, anno_name=name, anno_id=a.anno_id or "",
                                       anno_content=a.content or "", anno_title=a.title or "",
                                       bar_hists = bar_hists,
                                       gobe_examples=gobe_examples,
                                       history=history, user_history=user_history))
示例#7
0
def annotate(request, id):
    u = User.objects.get(id=1)
    aud = AudioMeta.objects.get(id=id)
    t = request.POST['t']
    a = open("/home/leegao/webapps/notetag/notetag/test.txt", "w")
    a.write(t)
    a.close()
    m = request.POST['data']
    an = Annotation(user_id=u, aud_id=aud, time=int(t), msg=m)
    an.save()
    return HttpResponse(content="0")
示例#8
0
文件: views.py 项目: itsachen/notetag
def annotate(request, id):
    u = User.objects.get(id=1)
    aud = AudioMeta.objects.get(id=id)
    t = request.POST["t"]
    a = open("/home/leegao/webapps/notetag/notetag/test.txt", "w")
    a.write(t)
    a.close()
    m = request.POST["data"]
    an = Annotation(user_id=u, aud_id=aud, time=int(t), msg=m)
    an.save()
    return HttpResponse(content="0")
示例#9
0
    def get(self, anno_id):
        # they can request a certain format by ending with .gobe whatever.
        self.response.headers['Content-type'] = 'text/plain'

        if anno_id == "":
            a = Annotation.all().order("-date").get()
        elif anno_id != "all":
            a = Annotation.all().filter('anno_id = ', anno_id).get()
        if a is None: a = Annotation()
        content = a.content

        if a.format and a.format != "gobe":
            content = [x.strip() for x in content.split("\n")]
            content = utils.main(content, a.format, force_tabs=True)

        self.response.out.write(content or "")
示例#10
0
    def __init__(self,
                 pkl_path='./data/pkl/annotations_clustering.pkl',
                 redundant=True):
        import os, pickle, logging
        from models import Annotation
        from mongoengine import connect
        import config

        connect(**config.Config.MONGODB_SETTINGS)
        self.redundant = redundant

        dumps = []
        if os.path.exists(pkl_path):
            dumps = pickle.load(open(pkl_path, "rb"))
        else:
            print('generate pkl' + pkl_path)
            annotations = Annotation.objects(type='sentence')
            for annotation in tqdm(annotations):
                try:
                    row = annotation.dump()
                    user = annotation.user
                    row['user_name'] = '{} {}'.format(user.first_name,
                                                      user.last_name)
                    dumps.append(row)
                except Exception as e:
                    logging.exception(e)
            pickle.dump(dumps, open(pkl_path, "wb"))

        random.shuffle(dumps)
        self.annotations = dumps
        self.build_map()
示例#11
0
def delete_duplicate_annotations():
    annotations = Annotation.objects().all()

    progress = 0
    total = annotations.count()
    for annotation in annotations:
        progress += 1
        if progress % 100 == 0:
            print('progress {}/{}'.format(progress, total))

        try:
            targets = Annotation.objects.filter(
                doc=annotation.doc,
                sent=annotation.sent,
                index=annotation.index,
                user=annotation.user,
                type=annotation.type,
                anchor_offset=annotation.anchor_offset)

            if targets.count() >= 2:
                print('count >= 2! :', targets.count())
                if targets[0].id != annotation.id:
                    targets[0].delete()
                else:
                    targets[1].delete()
        except:
            pass
示例#12
0
def download_dataset():
    docs = Doc.objects.filter()

    data = []
    for doc in tqdm(docs):
        annotations = Annotation.objects(doc=doc)
        for annotation in annotations:
            data.append({
                'annotator': annotation.user.username,
                'doc_id': doc.seq,
                'version': doc.type,
                'turker_id': annotation.user.turker_id,
                'sentence_index': annotation.index,
                'sentence': annotation.entire_text,
                'annotation_anchor_offset': annotation.anchor_offset,
                'annotation_focus_offset': annotation.focus_offset,
                'annotation_target_text': annotation.target_text,
                'annotation_type': annotation.type,
                'attributes': annotation.basket,
                'title': doc.title,
                'source': doc.source,
                'created_at': annotation.created_at,
            })

    dataset_path = os.path.abspath(os.path.dirname(__file__) + '/dataset.json')
    data_json = json.dumps(data, default=json_util.default, indent=2)
    with open(dataset_path, 'w', encoding='utf-8') as f:
        f.write(data_json)

    return send_file(dataset_path, as_attachment=True)
示例#13
0
def put_review_annotation(annotation_id):
    data = request.get_json()
    basket = data['basket']

    try:
        annotation = Annotation.objects().get(id=annotation_id)
    except Exception:
        return Response('404', status=404)
    try:
        annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
    except AnnotationReview.DoesNotExist:
        annotation_review = AnnotationReview(annotation=annotation, user=g.user)

    review_basket = dict()
    for key in basket:
        if '-review' in key:
            review_basket[key] = basket[key]

    annotation_review.doc = annotation.doc
    annotation_review.ip = request.remote_addr
    annotation_review.basket = review_basket
    annotation_review.updated_at = datetime.datetime.now
    annotation_review.save()

    annotation.basket = utils.merge_dict(annotation.basket, review_basket)

    return json.dumps({
        'annotation': annotation.dump(),
    })
示例#14
0
def index_v2_page(doc_type):
    item_per_page = 50
    page = request.args.get('p', 1)
    page = int(page)

    total = Doc.objects.filter(type=doc_type).count()
    total_page = math.ceil(total / item_per_page)
    paginator = Pagination(Doc.objects(type=doc_type).order_by('seq'), page, 50)
    docs = paginator.items

    docs_data = []
    for doc in docs:
        item = doc.dump()
        item['sent_total'] = Sent.objects(doc=doc).count()
        item['progress'] = Annotation.objects(doc=doc, user=g.user, type='sentence').count()

        docs_data.append(item)

    pagination = {
        'page': page,
        'total_page': total_page,
        'left': max(1, page - 5),
        'right': min(page + 5, total_page),
    }

    return render_template('index.html', type=doc_type, docs=docs_data, g=g, pagination=pagination)
示例#15
0
def import_csv(filename):
    result = []

    with codecs.open(filename, 'r', 'utf-8') as f:
        word_columns = []
        index_columns = []
        sentence_column = 0

        reader = csv.reader(f, delimiter=';')
        for i, row in enumerate(reader):
            if i == 0:  # header row
                for j, r in enumerate(row):
                    if r.startswith('w'):
                        word_columns.append(j)
                    if r.startswith('index'):
                        index_columns.append(j)
                    if r.startswith('full'):
                        sentence_column = j
            else:  # other rows
                pk = int(row[0])
                words = [row[c] for c in word_columns]
                indices = [row[c] for c in index_columns]
                sentence = row[sentence_column]
                result.append(Annotation(pk, words, indices, sentence))

    return result
示例#16
0
def download_dataset_AMT_v3():
    docs = Doc.objects.filter(type='mturk_v3')

    data = []
    for doc in tqdm(docs):
        annotations = Annotation.objects(doc=doc)
        for annotation in annotations:
            data.append({
                'annotator': annotation.user.username,
                'version': doc.type,
                'turker_id': annotation.user.turker_id,
                'doc_id': str(doc.id),
                'sentence_index': annotation.index,
                'sentence': annotation.entire_text,
                'basket': annotation.basket,
                'source': doc.source,
                'created_at': annotation.created_at,
            })

    dataset_path = os.path.abspath(os.path.dirname(__file__) + '/dataset_AMT_v3.json')
    data_json = json.dumps(data, default=json_util.default, indent=2)
    with open(dataset_path, 'w', encoding='utf-8') as f:
        f.write(data_json)

    return send_file(dataset_path, as_attachment=True)
示例#17
0
def ins_ann(topic_id):
    """
    Request Includes annotations and corresponding text.

    Should insert annotation.

    params:

    """
    try:
        data = json.loads(request.data)
        ann = Annotation.create(parent=topic_id,
                                start_index=data['start_index'],
                                end_index=data['end_index'],
                                start_paragraph=data['start_paragraph'],
                                end_paragraph=data['end_paragraph'],
                                body=data['body'],
                                deltas=0)
        ann.save()
    except json.JSONDecodeError as e:
        stderr.write("Unable to parse JSON request from /annotations/ \
                      reason: {}".format(str(e)))
    except IntegrityError as e:
        stderr.write("Integrity error when inserting annotation: {}".format(
            str(e)))
        abort(400)
    """Do something with annotations."""
    return Response("Hi", status=200, mimetype="text/plain")
示例#18
0
    def get(self, anno_id):
        # they can request a certain format by ending with .gobe whatever.
        self.response.headers['Content-type'] = 'text/plain';

        if anno_id == "":
            a = Annotation.all().order("-date").get()
        elif anno_id != "all":
            a = Annotation.all().filter('anno_id = ', anno_id).get()
        if a is None: a = Annotation()
        content = a.content

        if a.format and a.format != "gobe":
            content = [x.strip() for x in content.split("\n")]
            content = utils.main(content, a.format, force_tabs=True)

        self.response.out.write(content or "")
示例#19
0
def _addAnnotation(sentence,
                   annotation,
                   user,
                   prevSentence=None,
                   nextSentence=None):
    a = Annotation(sentence, annotation, user, prevSentence, nextSentence)
    db_session.add(a)
    db_session.commit()
示例#20
0
def get_ann_meta(topic_id):
    """Return annotation meta data."""

    annotations = Annotation.select().where(
        Annotation.parent == topic_id).order_by(Annotation.deltas)
    data = [model_to_dict(ann) for ann in annotations]

    return jsonify(data)
示例#21
0
def index():
    user: User = current_user
    facial_exprssions_translations = None
    if user.language == 'english':
        facial_exprssions_translations = utils.english_fe
    elif user.language == 'persian':
        facial_exprssions_translations = utils.persian_fe
    elif user.language == 'filipino':
        facial_exprssions_translations = utils.filipino_fe

    print(user.language, '**********')
    print(user.username, '  %%%%%%%%')

    if request.method == "POST":  # if the request is post (i.e. new video annotated?)
        # print(request.form)
        form = request.form



        annotation: Annotation = Annotation()
        annotation.emotion = form['emotion']
        annotation.anger_score = form['anger']
        annotation.contempt_score = form['contempt']
        annotation.disgust_score = form['disgust']
        annotation.annoyed_score = form['annoyed']
        annotation.gender = form['gender']
        annotation.filename = form.get("token")
        user.add_video(form.get("token"))

        social_signals = ','.join([ss for ss in form.getlist('socialsignal')])
        extra_ss = form.get("extra")
        ## TODO: should be tested
        if len(extra_ss) > 0:
            social_signals += ',%s' % extra_ss
        annotation.confidence = form['confidence']
        annotation.annotator_culture = user.culture
        annotation.annotator_language = user.language
        annotation.annotator_individuality = user.individuality
        annotation.social_signals = social_signals


        db.session.add(annotation)
        db.session.commit()
        vid = utils.get_random_video(user.culture, user.get_annotated_videos())
        completed = utils.get_completed_videos(user.culture, user.get_annotated_videos())
        if vid == "FINISHED":
            return render_template('thankyou.html')
        print("Annotation %s created :)" % annotation)

        return render_template('index.html', context={'video':vid, 'language': user.language, 'completed': completed,
                                                      'expressions': facial_exprssions_translations})

    vid = utils.get_random_video(user.culture, user.get_annotated_videos())
    completed = utils.get_completed_videos(user.culture, user.get_annotated_videos())
    if vid == "FINISHED":
        return render_template('thankyou.html')
    return render_template('index.html', context={'video':vid, 'language': user.language, 'completed': completed,
                                                  'expressions': facial_exprssions_translations})
示例#22
0
def delete_review_annotation(annotation_id):
    try:
        annotation = Annotation.objects().get(id=annotation_id)
        annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
        annotation_review.delete()
    except AnnotationReview.DoesNotExist:
        return Response('not found', status=404)

    return Response('success', status=200)
示例#23
0
def delete_annotation(annotation_id):
    try:
        annotation = Annotation.objects().get(id=annotation_id)
    except Annotation.DoesNotExist:
        return Response('not found', status=404)

    if annotation.user.id != g.user.id:
        return Response('permission error', status=403)
    annotation.delete()
    return Response('success', status=200)
def get_attribute_reason():
    """
    attribute_reason = {
        'Knowledge_Awareness': {
            'I_did_not_know_the_information': [],
        },
    }
    """
    attribute_reason = defaultdict(lambda: defaultdict(lambda: []))
    dumps = []
    bin_path = './data/pkl/annotations_with_sent.pkl'
    if os.path.exists(bin_path):
        dumps = pickle.load(open(bin_path, "rb"))
    else:
        print('generate ' + bin_path)
        annotations = Annotation.objects(type='sentence')
        for annotation in tqdm(annotations):
            try:
                row = annotation.dump()
                row['sentence'] = annotation.sent.text
                row['user_name'] = annotation.user.first_name + ' ' + annotation.user.last_name
                dumps.append(row)
            except Exception as e:
                logging.exception(e)
        pickle.dump(dumps, open(bin_path, "wb"))

    random.shuffle(dumps)

    reason_set = set()
    for annotation in tqdm(dumps):
        basket = annotation['basket']

        for attribute_key in basket:
            option = basket[attribute_key]
            if not ('reason' in option):
                continue

            attribute_value = option['value']
            reason = option['reason']

            if not reason:
                continue

            option['user'] = annotation['user']
            option['user_name'] = annotation['user_name']
            option['sentence'] = annotation['sentence']

            # reason_key = '{}-{}'.format(option['user'], ''.join(tokenize_and_lemmatize(reason)))
            # if reason_key in reason_set:
            #     continue
            # reason_set.add(reason_key)

            attribute_reason[attribute_key][attribute_value].append(option)

    return attribute_reason
示例#25
0
    def post(self, unused):
        self.response.headers['Content-type'] = 'text/javascript';
        user = users.get_current_user()
        annos = self.request.get('annos', '').strip()
        format = self.request.get('format', utils.guess_format(annos, force_tabs=True))
        if not annos:
            simplejson.dumps({'status': 'fail'})
            return
        name = user and user.nickname() or ""
        title = self.request.get('title') or ""
        anno_id = hashlib.md5(annos + title).hexdigest()

        # dont save unless it's new.
        a = Annotation.gql("WHERE anno_id = :anno_id AND title = :title AND author = :author",
                           anno_id=anno_id, title=title, author=user).get()
        if a is None:
            a = Annotation(title=title, author=user, content=annos.strip(), anno_id=anno_id,
                          format=format)
            a.put()

        self.response.out.write(simplejson.dumps({'status':'success', 'anno_id': anno_id}))
示例#26
0
def index1(request):
    if request.method=='POST':
        print request.body
        inputToDB=json.loads(request.body)
        json.dumps("annotMe.txt");
        print inputToDB
        myTemp=open('san.txt','w')
        myTemp.write(str(inputToDB))
        myTemp.close()        
        for anData in inputToDB:
            a=Annotation(x=anData['x'],y=anData['y'],msg=anData['msg'])
            print a.msg
	    #making a persistent storage
            a.save()
       # a=inputToDB
        #print a
        #dict = simplejson.JSONDecoder().decode( JSONdata ) 
        #print request.POST['data']
        print '---------Request'
        #print request
    return HttpResponse("Ur request is processed!!!") 
示例#27
0
def ins_com(annotation_id):
    """Insert comment on annotation."""
    ann_comm = None
    try:
        data = json.loads(request.data)
        ann = Annotation.get_by_id(annotation_id)
        comm = Comment(parent=annotation_id, body=data['comment'], deltas=0)
        comm.save()
    except json.JSONDecodeError as e:
        stderr.write("Unable to parse JSON request when attempting to \
                      add comment, reason: {}".format(str(e)))

    return Response("Hello", status=200, mimetype="text/plain")
示例#28
0
def get_annotation(doc_id):
    try:
        doc = Doc.objects().get(id=doc_id)
        annotations = Annotation.objects(doc=doc, user=g.user)
    except Exception as e:
        return Response('not found', status=404)
    data = []
    for annotation in annotations:
        data.append(annotation.dump())

    return json.dumps({
        'annotations': data,
    })
示例#29
0
def mod_ann_delta(annotation_id, up_or_down):
    """Add a delta to an annotation or comment"""
    try:
        ann = Annotation.get_by_id(annotation_id)
        if up_or_down == 'true':
            ann.deltas += 1
        else:
            if ann.deltas > 0:
                ann.deltas -= 1
        ann.save()
    except pw.DoesNotExist as e:
        stderr.write("Unable to find annotation_id in db: {}").format(str(e))

    return Response("Yo", status=200, mimetype="text/plain")
示例#30
0
def get_ann_text(topic_id, annotation_id):
    """Get text for annotation and comments."""
    data = ''
    try:
        annotations = Annotation.select().where(
            Annotation.parent == topic_id
            and Annotation.id == annotation_id).order_by(Annotation.deltas)
        data = [model_to_dict(ann) for ann in annotations][0]
    except pw.DoesNotExist as e:
        stderr.write(
            "Unable to find topic_id or annotation_id in db: {}").format(
                str(e))
    except IndexError:
        stderr.write("Index out of bounds, that's embarressing")

    return jsonify(data)
示例#31
0
def put_annotation(annotation_id):
    data = request.get_json()
    basket = data['basket']
    try:
        annotation = Annotation.objects().get(id=annotation_id)
    except Exception:
        return Response('not found', status=404)
    if annotation.user.id != g.user.id:
        return Response('permission error', status=403)
    annotation.basket = basket
    annotation.updated_at = datetime.datetime.now
    annotation.save()

    return json.dumps({
        'annotation': annotation.dump(),
    })
示例#32
0
def add_answer(request, stimulus_index, answer_pk):

    response = {}

    #Add the answer to the database
    answer = Answer.objects.get(pk=answer_pk)
    Annotation(question_nr=stimulus_index, answer=answer).save()

    #Check and return whether the answer is also correct
    correct_pk = open(STIMULI_FILE_LOCATION).readlines()[int(
        stimulus_index)].strip().split('\t')[1]
    response['correct'] = correct_pk == answer_pk

    print('Received', request.POST)

    if response['correct']:
        minimum_streak_to_enter_highscore = calculate_minimum_streak_to_enter_highscore(
        )

        #First, see whether this is info to add to an existing streak
        if request.POST['streak_id'] not in [
                None, 'undefined', 'null', False, 'false'
        ]:

            current_streak = Streak.objects.filter(
                pk=request.POST['streak_id'])[0]
            current_streak.length = request.POST['streak_length']
            current_streak.save()
            response['streak_id'] = current_streak.pk

        #if not, check whether you made the highscore
        elif int(request.POST['streak_length']
                 ) >= minimum_streak_to_enter_highscore:

            #If so, add the streak to the database
            new_streak = Streak(playername='Anonymous',
                                length=int(request.POST['streak_length']))
            new_streak.save()
            response['streak_id'] = new_streak.pk
            response['ask_for_name'] = True

        #else, do nothing specail
        else:
            response['streak_id'] = False

    return HttpResponse(json.dumps(response))
示例#33
0
文件: views.py 项目: annebaui/srimato
def image_detail(imageid):

    """
        function that is when a specific image is requested
        the metadata of the requested image is loaded (if there is one)
        and given to the template
    """

    try: 
        image = Image.get(id=imageid)
        imgfrag = ImageFragment.get(image=image, x=0, y=0, visible=True)
        imgmeta = Metadata.get(imgfrag=imgfrag, namespace="http://www.w3.org/ns/ma-ont#")
        annos = Annotation.get(imgmeta=imgmeta)
        return render_template('image_detail.html', img=image,imgmeta=imgmeta)
    except Imageuri.DoesNotExist:
        flash('Die Bildnummer wurde nicht gefunden. Bitte waehle ein anderes Bild.', 'error')
        return render_template('images.html')
    except ImageFragment.DoesNotExist:
        return render_template('image_detail.html', img=image)
示例#34
0
def change_all_attribute_key():
    annotations = Annotation.objects().all()

    for annotation in tqdm(annotations):
        basket = annotation.basket
        new_basket = dict()

        for key in basket:
            new_key = key
            value = basket[key]
            if key == 'Disputability_of_the_sentence':
                new_key = 'Disputability'
            elif key == 'Perceived_Author_Credibility_for_the_upcoming_sentences':
                new_key = 'Perceived_Author_Credibility'
            elif key == 'Acceptance_of_the_sentence_as_true':
                new_key = 'Acceptance'
            new_basket[new_key] = value

        annotation.basket = new_basket
        annotation.save()
示例#35
0
def get_annotations():
    dumps = []
    bin_path = './data/bin/annotations_time.bin'
    if os.path.exists(bin_path):
        dumps = pickle.load(open(bin_path, "rb"))
    else:
        print('generate ' + bin_path)
        annotations = Annotation.objects(type='sentence')
        for annotation in tqdm(annotations):
            try:
                if not ('Acceptance' in annotation['basket']):
                    continue
                dump = annotation.dump()
                dump['is_turk'] = not annotation.user.is_active
                dumps.append(dump)
            except Exception as e:
                logging.exception(e)
        pickle.dump(dumps, open(bin_path, "wb"))

    random.shuffle(dumps)
    return dumps
示例#36
0
def get_review_annotation(user_id, doc_id):
    try:
        doc = Doc.objects.get(id=doc_id)
        user = User.objects.get(id=user_id)
    except Exception as e:
        return redirect('/404')
    annotations = Annotation.objects(doc=doc, user=user)

    data = []
    for annotation in annotations:
        try:
            annotation_review = AnnotationReview.objects().get(annotation=annotation, user=g.user)
            annotation.basket = utils.merge_dict(annotation.basket, annotation_review.basket)
        except AnnotationReview.DoesNotExist:
            pass

        data.append(annotation.dump())

    return json.dumps({
        'annotations': data,
    })
示例#37
0
def export_dataset_v3():
    docs = Doc.objects.filter(type='mturk_v3')

    data = []
    for doc in tqdm(docs):
        sents = Sent.objects(doc=doc)
        annotations = Annotation.objects(doc=doc)

        if sents.count() != annotations.count():
            continue

        source = doc.source
        if 'aljazeera' not in source and 'foxnews' not in source and 'theguardian' not in source:
            continue

        if not is_ok(annotations):
            continue

        for annotation in annotations:
            data.append({
                'annotator': annotation.user.username,
                'version': doc.type,
                'turker_id': annotation.user.turker_id,
                'doc_id': str(doc.id),
                'sentence_index': annotation.index,
                'sentence': annotation.entire_text,
                'basket': annotation.basket,
                'source': doc.source,
                'created_at': annotation.created_at,
            })

    dataset_path = os.path.abspath(
        os.path.dirname(__file__) + '/../data/dataset_AMT_v3.json')
    data_json = json.dumps(data, default=json_util.default)
    with open(dataset_path, 'w', encoding='utf-8') as f:
        f.write(data_json)
示例#38
0
def lcia_dispatcher(product, method):
    """
    same again
    """
    uR, annot = random_result("float")
    
    # save the result  
    R = RealResult(data_type = "float",
                   unit = uR.unit,
                   mean_value = uR.value,
                   st_dev = uR.stdev,
                   confidence_interval_5 = uR.conf5(),
                   confidence_interval_95 = uR.conf95())

    R.save()
    A=Annotation(uuid=R.uuid,relation=method.name,annotation=product.name + " | " + annot)
    A.save()
    Aa=Annotation(uuid=R.uuid,relation='created',annotation=now())
    Aa.save()

    return R
示例#39
0
def module_dispatcher(prop, chemical, product=None):
    """
    first argument is the property to be computed-- should be an instance of models.Property
    second argument is the chemical the property is computed about-- instance of models.Chemica
    third argument is the product system in scope-- instance of models.Product. not implemented.
    
    This function is only called when a new result is required.  Invoke the appropriate module
    and return the result!  make a random one for now.
    """
    
    M = prop.module
    # these should all return a ( uResult , annotation )
    uR, annot = {
            'Module 01': (None, 'dummy'),
            'Module 02': (None, 'dummy'),
            'ECHA CombinedTox': echa_tox_lookup(prop, chemical)
            }.get(M.name,random_result(prop.data_type))
    
    if uR is None:
        return annot
    
    # save the result  
    R = RealResult(data_type = prop.data_type,
                   unit = uR.unit,
                   mean_value = uR.value,
                   st_dev = uR.stdev,
                   confidence_interval_5 = uR.conf5(),
                   confidence_interval_95 = uR.conf95())

    R.save()
    A=Annotation(uuid=R.uuid,relation='result',annotation=annot)
    A.save()
    Aa=Annotation(uuid=R.uuid,relation='created',annotation=now())
    Aa.save()

    return R
示例#40
0
def ImportGFF(specimen, file):
    fileName = file.split('/')
    fileName = fileName[-1].split('.')[0]

    specimenObject = GetSpecimen(specimen)

    gff, created = GFF.objects.get_or_create(Specimen=specimenObject, FileName=fileName)
    print gff.FileName
    #Get the specimen this annotation file is for
    gff.Specimen = GetSpecimen(specimen)
    #Set default version type
    gff.GFFVersion = 2

    #Open file and check for version type
    #TODO: Grab GFF Info and store in database
    annotationFile = open(file, 'r')
    counter = 0
    for line in annotationFile.readlines():
        counter += 1

        if line.startswith("##"):
            if "gff-version" in line:
                temp = line.split(' ')
                gff.GFFVersion = temp[1]

        if counter == 10:
            break
    annotationFile.close()
    gff.save()

    #Grab a list of chromosomes related to this specimen
    validChromosomes = GetRelatedChromosomes(gff.Specimen)

    annotations = {}

    #In GFF format, each line is an annotation. 
    #Read in each annotation and split at tabs (as per gff spec)
    #Then store as needed into an Annotation object which will be pushed onto the annotations stack
    #TODO: Bust out version reading into methods
    if gff.GFFVersion == 2:
        print "BEGINNING READING OF VERSION 2 FILE...\n"
        annotationFile = open(file, 'r')
        counter = 0
        for line in annotationFile.readlines():
            counter += 1
            if not line.startswith('#'):
                elements = line.split('\t')

                annotation = Annotation()
                #TODO: Handle when values come back null from not finding a matching chromosome!
                annotation.Specimen = specimen
                annotation.Chromosome = ParseChromosomeName(validChromosomes,
                                                            elements[0]) #Related validChromosomes, chromosome
                annotation.ID = counter
                annotation.Source = elements[1]
                annotation.Feature = elements[2]
                annotation.Start = elements[3]
                annotation.End = elements[4]

                #Do checking for Score
                if elements[5] == '.':
                    annotation.Score = None
                else:
                    annotation.Score = elements[5]

                #Do checking for Strand
                if elements[6] == '.':
                    annotation.Strand = None
                else:
                    annotation.Strand = elements[6]

                #Do checking for frame
                if elements[7] == '.':
                    annotation.Frame = None
                else:
                    annotation.Frame = elements[7]

                #Check for existence of attribute and possible extra length
                if len(elements) >= 9:
                    annotation.Attribute = elements[8:]
                else:
                    annotation.Attribute = None

                if annotation.Chromosome in annotations:
                    #appent to the current list
                    annotations[annotation.Chromosome].append(annotation)
                else:
                    #Create the chromosome entry in the annotations dictionary
                    annotations[annotation.Chromosome] = []
                    annotations[annotation.Chromosome].append(annotation)

                if counter % 10000 == 0:
                    sys.stdout.write('.')

                    #print "RESULTS: ", annotation.Specimen, annotation.Source, annotation.Feature, annotation.Start, annotation.End, annotation.Score, annotation.Strand, annotation.Frame, annotation.Attribute

        print "DONE READING FILE!"
    else:
        print "THIS GFF VERSION IS NOT SUPPORTED: Version", gff.GFFVersion

    print "BEGINNING SORTING LIST..."
    #Sort the list of annotations read in by their start value (this could probably be optimized by using an always ordered list and inserting in order above)
    #annotations = sorted(annotations, key = lambda annotation: annotation.Start)
    for sublist in annotations:
        annotations[sublist] = sorted(annotations[sublist], key=lambda annotation: int(annotation.Start))
    print "DONE SORTING!"

    chunkAndStoreAnnotations(gff, annotations)
示例#41
0
def functional_table():
    form = FunctionClassFilterForm()
    form.function_class.choices = [('cog', 'Cog'),
                    ('pfam', 'Pfam'),
                    ('tigrfam', 'TigrFam'),
                    ('all', 'All')
                ]

    form.select_sample_groups.choices = [(sample_set.name, sample_set.name) for sample_set in  SampleSet.query.all()]

    type_identifiers = []
    if form.validate_on_submit():
        function_class = form.function_class.data
        if function_class == 'all':
            function_class = None
        limit = form.limit.data
        if limit == 'all':
            limit = None
        else:
            limit = int(limit)

        filter_alternative = form.filter_alternative.data
        if filter_alternative == 'filter_with_type_identifiers':
            for type_identifier in form.type_identifiers.entries:
                if type_identifier.data != '':
                    type_identifiers.append(type_identifier.data)
        elif filter_alternative == 'filter_with_search':
            search_string = form.search_annotations
            if search_string.data != '':
                q = _search_query(search_string.data)
                type_identifiers = [a.type_identifier for a in q.all()]


        sample_sets = form.select_sample_groups.data
        if len(sample_sets) > 0:
            samples = [sample.scilifelab_code for sample in Sample.all_from_sample_sets(sample_sets)]
        else:
            samples = None

        download_action = False
        if form.submit_download.data:
            download_action = True
            download_select = form.download_select.data
    else:
        function_class=None
        limit=20
        samples = None
        download_action = False

    if len(form.type_identifiers) == 0:
        form.type_identifiers.append_entry()

    if type_identifiers == []:
        type_identifiers = None

    samples, table = Annotation.rpkm_table(limit=limit, samples=samples, function_class=function_class, type_identifiers=type_identifiers)
    samples = sorted(samples, key=lambda x: x.scilifelab_code)
    sample_scilifelab_codes = [sample.scilifelab_code for sample in samples]
    if download_action:
        if download_select == 'Gene List':
            # Fetch all contributing genes for all the annotations in the table
            annotation_ids = [annotation.id for annotation, sample in table.items()]
            genes_per_annotation = Annotation.genes_per_annotation(annotation_ids)
            csv_output = '\n'.join(
                    [','.join([gene.name, annotation.type_identifier]) \
                            for gene, annotation in genes_per_annotation])
            r = make_response(csv_output)
            r.headers["Content-Disposition"] = "attachment; filename=gene_list.csv"
            r.headers["Content-Type"] = "text/csv"
            return r
    return render_template('functional_table.html',
            table=table,
            samples=samples,
            sample_scilifelab_codes = sample_scilifelab_codes,
            form=form
        )
示例#42
0
 def get_annotation(self, imgmeta):
     return Annotation.select().where(imgmeta=imgmeta)
示例#43
0
    def _create_or_update(self, anno=None):
        create = anno is None
        if create:
            anno = Annotation()

        body = self._get_request_body()
        anno.context_id = body['contextId']
        anno.collection_id = body['collectionId']
        anno.uri = body['uri']
        anno.media = body['media']
        anno.user_id = body['user']['id']
        anno.user_name = body['user']['name']
        anno.is_private = False if len(body.get('permissions', {}).get('read', [])) == 0 else True
        anno.text = body.get('text', '')
        anno.quote = body.get('quote', '')
        anno.json = json.dumps(body)

        if 'parent' in body and body['parent'] != '0':
            anno.parent_id = int(body['parent'])
        anno.save()

        if create and anno.parent_id:
            parent_anno = Annotation.objects.get(pk=int(body['parent']))
            parent_anno.total_comments = F('total_comments') + 1
            parent_anno.save()

        if not create:
            anno.tags.clear()
        for tag_name in body.get('tags', []):
            tag_object, created = AnnotationTags.objects.get_or_create(name=tag_name.strip())
            anno.tags.add(tag_object)

        return anno
示例#44
0
def _handle_post_annotation_request(user, document, request):
    if not user.is_authenticated:
        return HttpResponseForbidden()

    annotation = Annotation()
    annotation.annotator = user
    annotation.content = sanitize(request.POST['annotation_content'])
    annotation.document_this_annotation_belongs = document
    annotation.page_index = request.POST["page_id"].split("_")[2]
    annotation.height_percent = request.POST["height_percent"]
    annotation.width_percent = request.POST["width_percent"]
    annotation.top_percent = request.POST["top_percent"]
    annotation.left_percent = request.POST["left_percent"]
    annotation.frame_color = request.POST["frame_color"]
    annotation.is_public = True if request.POST[
        "is_public"] == 'true' else False
    annotation.save()

    # send notification to the document uploader
    if annotation.annotator.pk != document.owner.pk:
        notify.send(
            sender=annotation.annotator,
            recipient=document.owner,
            action_object=annotation,
            verb='post annotation',
            redirect_url=annotation.url,
            image_url=annotation.annotator.portrait_url,
            description=h.handle(annotation.content),
            is_public=annotation.is_public,
        )
    # send notification to the collectors, i.e., followers
    for user in document.collectors.all():
        if annotation.annotator.pk != user.pk and document.owner.pk != user.pk:
            notify.send(
                sender=annotation.annotator,
                recipient=user,
                action_object=annotation,
                verb='post annotation',
                redirect_url=annotation.url,
                image_url=annotation.annotator.portrait_url,
                description=h.handle(annotation.content),
                is_public=annotation.is_public,
            )

    context = {
        "document": document,
        'annotation': annotation,
        'ANONYMOUS_USER_PORTRAIT_URL': settings.ANONYMOUS_USER_PORTRAIT_URL,
        "new_annotation_id": annotation.id,
    }
    return JsonResponse(
        {
            'new_annotationdiv_html':
            render(request, "file_viewer/one_annotation_div.html",
                   context).content,
            'new_annotation_id':
            annotation.id,
            'new_annotation_uuid':
            str(annotation.clean_uuid),
            'new_annotation_json':
            annotation,
        },
        encoder=AnnotationEncoder)
示例#45
0
    def test_annotation_rpkm_table(self):
        annotation_types = [("Cog", {'class': Cog}),
                ("Pfam", {'class': Pfam}),
                ("TigrFam", {'class': TigrFam}),
                ("EcNumber", {'class': EcNumber})]

        nr_annotation_types = len(annotation_types)
        annotation_sources = {}
        for annotation_type, type_d in annotation_types:
            annotation_sources[annotation_type]= AnnotationSource(
                    annotation_type,
                    "v1.0",
                    "rpsblast",
                    "e_value=0.000001"
                )

        sample1 = Sample("P1993_101", None, None)
        sample2 = Sample("P1993_102", None, None)
        nr_samples = 2
        for i in range(50):
            gene1 = Gene("gene1{}".format(i), None)
            gene2 = Gene("gene2{}".format(i), None)

            gene_count1 = GeneCount(gene1, sample1, 0.001)
            gene_count2 = GeneCount(gene1, sample2, 0.01)
            gene_count3 = GeneCount(gene2, sample1, 0.002)
            gene_count4 = GeneCount(gene2, sample2, 0.02)

            for annotation_type, type_d in annotation_types:
                if annotation_type == 'Cog':
                    type_id = str(i)
                    type_id = "0"*(4-len(type_id))+type_id
                    annotation = type_d['class'](annotation_type.upper() + type_id, "H")
                elif annotation_type == 'EcNumber':
                    if i > 25:
                        type_id = "0.0.2.{}".format(i)
                    else:
                        type_id = "0.0.0.{}".format(i)
                    annotation = type_d['class'](type_id)
                else:
                    type_id = str(i)
                    type_id = "0"*(4-len(type_id))+type_id
                    annotation = type_d['class'](annotation_type.upper() + type_id)

                annotation_mode = i % 3
                gene_annotations = []
                if annotation_mode in [0,1]:
                    gene_annotations.append(GeneAnnotation(
                            annotation,
                            gene1,
                            annotation_sources[annotation_type]
                        ))
                if annotation_mode in [1,2]:
                    gene_annotations.append(GeneAnnotation(
                            annotation,
                            gene2,
                            annotation_sources[annotation_type]
                        ))
                self.session.add_all(gene_annotations)

            self.session.add(gene1)
            self.session.add(gene2)
        self.session.commit()
        refresh_all_mat_views()
        samples, rows = Annotation.rpkm_table()
        assert len(samples) == 2
        assert len(rows) == 20 # Default limit
        samples, rows = Annotation.rpkm_table(limit=100)
        assert len(samples) == 2
        assert len(rows) == 100
        samples, rows = Annotation.rpkm_table(limit=None)
        assert len(samples) == 2
        assert len(rows) == nr_annotation_types * 50

        for annotation, sample_d in rows.items():
            # sample_d should be a ordered dict
            assert ["P1993_101", "P1993_102"] == [sample.scilifelab_code for sample in sample_d.keys()]
        rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]

        rpkms_flat = []
        for rpkm_row in rpkms:
            rpkms_flat += rpkm_row

        assert len(rpkms_flat) == nr_annotation_types * nr_samples * 50

        # Annotations sorted by total rpkm over all samples
        # and the rpkm values should be summed over all genes for that annotation
        # there should be roughly equal numbers of the three different counts
        for i, row in enumerate(rpkms[:67]):
            assert row == [0.003, 0.03]
        for row in rpkms[69:130]:
            assert row == [0.002, 0.02]
        for row in rpkms[150:200]:
            assert row == [0.001, 0.01]

        # possible to filter on function classes
        for annotation_type, type_d in annotation_types:
            samples, rows = Annotation.rpkm_table(limit=None, function_class=annotation_type.lower())
            assert len(rows) == 50
            for key in rows.keys():
                assert annotation_type[:3].lower() == key.annotation_type[:3]

        # possible to filter on samples
        for sample in [sample1, sample2]:
            samples, rows = Annotation.rpkm_table(samples=[sample.scilifelab_code], limit=None)
            assert len(rows) == 200
            assert len(samples) == 1
            assert samples[0] == sample
            for annotation, sample_d in rows.items():
                assert list(sample_d.keys()) == [sample]

            rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]
            if sample.scilifelab_code == "P1993_101":
                for i, row in enumerate(rpkms[:65]):
                    assert row == [0.003]
                for row in rpkms[69:130]:
                    assert row == [0.002]
                for row in rpkms[150:200]:
                    assert row == [0.001]
            else:
                for row in rpkms[:67]:
                    assert row == [0.03]
                for row in rpkms[69:130]:
                    assert row == [0.02]
                for row in rpkms[150:200]:
                    assert row == [0.01]

        # possible to filter on sample and function class at the same time
        for annotation_type, type_d in annotation_types:
            for sample in [sample1, sample2]:
                samples, rows = Annotation.rpkm_table(limit=None, function_class=annotation_type.lower(), samples=[sample.scilifelab_code])
                assert len(rows) == 50
                for key in rows.keys():
                    assert annotation_type.lower()[:3] == key.annotation_type[:3]

                assert len(samples) == 1
                assert samples[0] == sample
                for annotation, sample_d in rows.items():
                    assert list(sample_d.keys()) == [sample]

                rpkms = [[rpkm for sample, rpkm in sample_d.items()] for annotation, sample_d in rows.items()]
                if sample.scilifelab_code == "P1993_101":
                    for row in rpkms[:9]:
                        assert row == [0.003]
                    for row in rpkms[19:29]:
                        assert row == [0.002]
                    for row in rpkms[39:]:
                        assert row == [0.001]
                else:
                    for row in rpkms[:9]:
                        assert row == [0.03]
                    for row in rpkms[19:29]:
                        assert row == [0.02]
                    for row in rpkms[39:]:
                        assert row == [0.01]

        # possible to filter on individual annotations
        annotation_ids = ["COG0001", "TIGRFAM0004", "COG0003", "PFAM0002", "0.0.2.26"]

        for r in range(5):
            for type_identifiers in itertools.combinations(annotation_ids, r+1):

                samples, rows = Annotation.rpkm_table(limit=None, type_identifiers=list(type_identifiers))
                assert len(samples) == 2
                assert len(rows) == len(type_identifiers)
                assert set([key.type_identifier for key in rows.keys()]) == set(type_identifiers)
示例#46
0
    def test_annotation(self):
        annotation = Annotation("COG0001")
        self.session.add(annotation)
        self.session.commit()

        assert Annotation.query.first() is annotation

        #Test the many to many relationship
        reference_assembly = ReferenceAssembly("version 1")
        gene = Gene("gene1", reference_assembly)
        gene2 = Gene("gene2", reference_assembly)
        gene3 = Gene("gene3", reference_assembly)

        annotation2 = Annotation("COG0002", description="This cog is really really good")
        # Test having multiple genes to one annotation
        annotation_source = AnnotationSource("Cog", "v1.0", "rpsblast", "e_value=0.000001")
        gene_annotation1 = GeneAnnotation(annotation_source = annotation_source, e_value=0.0000001)
        gene_annotation2 = GeneAnnotation(annotation_source = annotation_source)

        gene_annotation1.gene = gene
        gene_annotation2.gene = gene2

        gene_annotation1.annotation = annotation
        gene_annotation2.annotation = annotation

        self.session.add(annotation)
        self.session.add(gene3)
        self.session.add(gene_annotation1)
        self.session.add(gene_annotation2)
        self.session.commit()

        annotation_01 = Annotation.query.filter_by(type_identifier="COG0001").first()
        assert len(annotation_01.genes) == 2
        assert gene in annotation_01.genes
        assert gene2 in annotation_01.genes
        assert annotation in Gene.query.filter_by(name="gene1").first().annotations
        assert annotation in Gene.query.filter_by(name="gene2").first().annotations
        assert len(Gene.query.filter_by(name="gene3").first().annotations) == 0

        # Genes for annotation method
        genes_for_annotation = Annotation.genes_per_annotation([annotation.id])
        assert len(genes_for_annotation) == 2
        assert (gene, annotation) in genes_for_annotation
        assert (gene2, annotation)  in genes_for_annotation

        # Add the second annotation
        self.session.add(annotation2)
        self.session.commit()
        q =  Annotation.query.filter(Annotation.description.contains("good"))
        annotation_02 = q.all()
        assert len(annotation_02) == 1
        assert annotation_02[0] == annotation2

        # Test having multiple annotations to one gene
        gene_annotation3 = GeneAnnotation(annotation2, gene, annotation_source, e_value = 1e-14)
        self.session.add(gene_annotation3)
        self.session.commit()

        assert len(Gene.query.filter_by(name="gene1").first().annotations) == 2
        assert annotation in Gene.query.filter_by(name="gene1").first().annotations
        assert annotation2 in Gene.query.filter_by(name="gene1").first().annotations

        assert gene_annotation1.e_value > gene_annotation3.e_value
        assert gene.e_value_for(annotation) > gene.e_value_for(annotation2)

        # gene -> annotation
        # gene2 -> annotation
        # gene -> annotation2

        # Genes for annotation method
        genes_for_annotation = Annotation.genes_per_annotation([annotation.id])
        assert len(genes_for_annotation) == 2
        assert (gene, annotation) in genes_for_annotation
        assert (gene2, annotation) in genes_for_annotation

        genes_for_annotation = Annotation.genes_per_annotation([annotation2.id])
        assert len(genes_for_annotation) == 1
        assert (gene, annotation2) in genes_for_annotation

        genes_for_annotation = Annotation.genes_per_annotation([annotation.id, annotation2.id])
        assert len(genes_for_annotation) == 3
        assert (gene, annotation) in genes_for_annotation
        assert (gene, annotation2) in genes_for_annotation
        assert (gene2, annotation) in genes_for_annotation

        annotation3 = Annotation("COG0003", description=("This cog is really really good. I assure you, "
            "really quite good. Among its capabilities I have to mention that its utterly suitable for "
            "testing the description string, including the short description."))

        assert len(annotation3.description) > 103
        assert annotation3.short_description[-3:] == "..."
        assert len(annotation3.short_description) == 103
        assert annotation3.description[:100] == annotation3.short_description[:100]