def bulk_annotate(request,cid,sid=None):
'''bulk annotate will allow a user to apply annotations, in masse, to a collection or set
:param cid: the collection id to use
:param sid: the report set id, if a subset of reports is being used
'''
if sid !=None:
report_set = get_report_set(sid)
collection = report_set.collection
reports = report_set.reports.all()
else:
collection = get_report_collection(cid)
reports = collection.report_set.all()
if has_collection_edit_permission(request,collection):
# POST indicates creating a bulk annotation
if request.method == "POST":
# Does the user want to bulk annotate only unlabeled sets?
unlabeled_only = request.POST.get('unlabeled_only',None)
# What annotations does the user want to do in bulk?
selection_keys = [x for x in request.POST.keys() if re.search("^whatisit[||]", x)]
selections = []
seen_annotations = []
for selection_key in selection_keys:
name = selection_key.replace("whatisit||","")
label = request.POST.get(selection_key,"")
if label != "":
allowed_annotation = AllowedAnnotation.objects.get(name=name,
label=label)
# If the user doesn't want to include already labeled reports, exclude
if unlabeled_only != None:
reports = reports.exclude(reports_annotated__annotator=request.user).distinct()
for report in reports:
# Update user annotation sent to celery to run async
update_user_annotation.apply_async(kwargs={'user':request.user.id,
'allowed_annotation':allowed_annotation.id,
'report':report.id})
messages.info(request,"Annotation %s:%s task running for %s reports" %(name,label,reports.count()))
return view_report_collection(request,cid)
# Otherwise just render the form
else:
allowed_annotations = get_allowed_annotations(collection,return_objects=False)
context = {"collection": collection,
"allowed_annotations": allowed_annotations}
if sid != None:
context['sid'] = sid
return render(request, "annotate/bulk_annotate.html", context)
return HttpResponseRedirect(collection.get_absolute_url())
def download_annotation_set(request, sid, uid, return_json=False):
'''download annotation set will download annotations for a particular user
and report set. Default returns a text/csv file, if return_json is true,
returns json file
:param sid: the report set id
:param uid: the user id to download
:param return_json: return a Json response instead (default is False)
'''
report_set = get_report_set(sid)
collection = report_set.collection
# Does the user have permission to edit?
requester = request.user
if requester == collection.owner or requester in collection.contributors.all(
):
user = User.objects.get(id=uid)
df = get_reportset_annotations(report_set, user)
if not return_json:
response = HttpResponse(df.to_csv(sep="\t"),
content_type='text/csv')
export_name = "%s_%s_annotations.tsv" % (report_set.id,
user.username)
response['Content-Disposition'] = 'attachment; filename="%s"' % (
export_name)
return response
return JsonResponse(df.to_json(orient="records"))
messages.info(request,
"You do not have permission to perform this action.")
return redirect('report_collection_details', cid=collection.id)
def new_set_annotator(request,sid):
'''creates a new Credential for a user and a particular
collection, with default state TESTING to ensure tests first.
:param sid: the report_set id
'''
report_set = get_report_set(sid)
collection = report_set.collection
if has_collection_edit_permission(request,collection):
if request.method == "POST":
user_id = request.POST.get('user',None)
if user_id:
user = get_user(request,user_id)
credential = Credential.objects.create(user=user,
report_set=report_set)
credential.save()
messages.success(request, 'Credential for user %s added, user will need to test before annotating.' %(user))
return redirect('edit_set_annotators',sid=report_set.id)
def edit_set_annotators(request,sid):
'''edit_set_annotators allows a collection owner to add/remove users to an annotation set
this means that the user has asked for and been given permission to annotate the collection.
:param sid: the report set id
'''
report_set = get_report_set(sid)
collection = report_set.collection
if has_collection_edit_permission(request,collection):
# Get list of allowed annotators for set, not in set (to add)
has_credential = has_credentials(report_set,status="PASSED")
denied_credential = has_credentials(report_set,status="DENIED")
testing_credential = has_credentials(report_set,status="TESTING")
# Get credentials for allowed annotators
credentials = get_credentials(has_credential,report_set)
users_with_credentials = [x.user for x in credentials]
# Get list of allowed annotators for set, allowed in set (if want to remove)
contenders = get_credential_contenders(report_set)
# Get list of failed annotators for set (if want to retest)
failures = get_credentials(denied_credential,
report_set,
status="DENIED")
# Get list of testing annotators for set (if want to pass without testing)
testing = get_credentials(testing_credential,
report_set,
status="TESTING")
# Remove contenders that are allowed annotation
contenders = [x for x in contenders if x not in users_with_credentials]
context = {'annotators':credentials,
'collection':collection,
'testing':testing,
'contenders':contenders,
'report_set':report_set,
'failures':failures}
return render(request, 'reports/report_collection_annotators.html', context)
# Does not have permission, return to collection
messages.info(request, "You do not have permission to edit annotators for this collection.")
return view_report_collection(request,cid)
def remove_set_annotator(request,sid,uid):
'''completely remove a user from an annotation set.
:param sid: the report_set id
:param uid: the user id
'''
report_set = get_report_set(sid)
collection = report_set.collection
if has_collection_edit_permission(request,collection):
annotator = get_user(request,uid)
credential = Credential.objects.get(user=annotator,
report_set=report_set)
credential.delete()
messages.info(request, "User %s has been removed from set annotators." %(annotator))
return render(request, 'reports/report_collection_annotators.html', context)
# Does not have permission, return to collection
messages.info(request, "You do not have permission to edit annotators for this collection.")
return view_report_collection(request,cid)
def annotate_set(request,sid,show_count=True):
'''annotate_set will allow the user to annotate a custom selected set
'''
report_set = get_report_set(sid)
collection = report_set.collection
user = request.user
# Only continue annotation if the user is approved (and not expired)
user_status = get_annotation_status(report_set=report_set,
user=user)
# No credential exists, for user (this should not happen)
if user_status == None:
messages.info(request,"You must ask for permission to annotate a collection first.")
return HttpResponseRedirect(collection.get_absolute_url())
# Approved means we continue
if user_status == "PASSED":
if has_collection_annotate_permission(request,collection):
# If the user has un-annotated reports, return those first
reports = report_set.reports.all().exclude(reports_annotated__annotator=user).distinct()
if len(reports) == 0:
reports = report_set.reports.all()
return annotate_random(request,
cid=collection.id,
sid=sid,
reports=reports)
elif user_status == "TESTING":
# Send the user to the testing view, will grant permission/deny after test
return test_annotator(request=request,
sid=report_set.id)
else: #denied or other
messages.info(request,"You are not allowed to perform this action.")
return HttpResponseRedirect(collection.get_absolute_url())
def download_reports(request, cid, sid=None, return_json=False):
'''download reports returns a tsv download for an entire collection of reports (not recommended)
or for reports within a collection (recommended)
:param cid: the collection id
:param sid: the report set if, if provided use that report set.
:param return_json: return a Json response instead (default is False)
'''
if sid != None:
report_set = get_report_set(sid)
collection = report_set.collection
reports = report_set.reports.all()
export_name = "%s_reports_set.tsv" % (report_set.id)
else:
collection = get_report_collection(cid)
reports = collection.report_set.all()
export_name = "%s_reports.tsv" % (collection.id)
# Does the user have permission to edit?
requester = request.user
if requester == collection.owner or requester in collection.contributors.all(
):
df = pandas.DataFrame(columns=["report_id", "report_text"])
df["report_id"] = [r.report_id for r in reports]
df["report_text"] = [r.report_text for r in reports]
if not return_json:
response = HttpResponse(df.to_csv(sep="\t"),
content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="%s"' % (
export_name)
return response
else:
return JsonResponse(df.to_json(orient="records"))
messages.info(request,
"You do not have permission to perform this action.")
return redirect('report_collection_details', cid=collection.id)
def change_set_annotator(request,sid,uid,status):
'''change the status of a set annotator to one of
PASSED,DENIED,TESTING
:param sid: the report_set id
:param uid: the user id
'''
report_set = get_report_set(sid)
collection = report_set.collection
if has_collection_edit_permission(request,collection):
annotator = get_user(request,uid)
credential = Credential.objects.get(user=annotator,
report_set=report_set)
credential.status = status
credential.save()
messages.info(request,"User %s status changed to %s" %(annotator,status.lower()))
# If the user is passed, add to annotators
if status == "PASSED":
report_set.annotators.add(annotator)
elif status in ["DENIED","TESTING"]:
report_set.annotators.remove(annotator)
report_set.save()
# Alert the user of the status change
message = """Your request to annotate set %s for
collection %s has updated status %s""" %(report_set.name,
collection.name,
status.lower())
notify.send(collection.owner, recipient=annotator, verb=message)
return redirect('edit_set_annotators',sid=report_set.id)
# Does not have permission, return to collection
messages.info(request, "You do not have permission to edit annotators for this collection.")
return view_report_collection(request,cid)
def test_annotator(request, sid, rid=None):
'''test_annotator will ask an annotator to answer the specified number of
questions to be granted access to annotate a collection. The data is stored
in a session, and the session expires (and is reset) when the user has viewed
the total number of required questions
:param sid: the id of the report set
:param rid: the report id that was tested
'''
from whatisit.apps.wordfish.views import (annotate_report_testing,
annotate_set, get_permissions,
view_report_collection)
from whatisit.apps.users.models import Credential
if rid != None: # scoring is needed
completed_report = get_report(rid)
user = request.user
report_set = get_report_set(sid)
context = {'collection': report_set.collection}
permissions = get_permissions(request, context)
# Double check that user has permission to annotate
if permissions["annotate_permission"] == True:
# Also check that user hasn't previously failed.
credential = Credential.objects.get(user=user, report_set=report_set)
# Double check user status
user_status = get_annotation_status(report_set=report_set, user=user)
# Only continue if user_status is TESTING
if user_status == "TESTING":
# Get session variable
session = get_testing_session(user=user, report_set=report_set)
session_key = 'reports_testing_%s' % (session.id)
# testing set is stored in request.session with corresponding id
testing_set = request.session.get(session_key, None)
testing_report = None
testing_correct = session.correct
testing_incorrect = session.incorrect
# Total reports required for taking, and for passing
N = report_set.number_tests
N_pass = report_set.passing_tests
###########################################################
# NEW TESTING SESSION
###########################################################
if testing_set == None:
messages.info(
request,
'''This is the start of the test. You will be asked to annotate %s reports.
You should click on the correct label below, and then you can
use your right arrow key (or the arrow at the bottom) to submit your answer.
''' % (N))
# Randomly select N reports from the set
testing_reports = select_random_reports(
reports=report_set.reports.all(), N=N)
testing_reports = list(testing_reports)
# Remove the first for testing
testing_report = testing_reports.pop(0)
request.session[session_key] = testing_reports
###########################################################
# RESUME TESTING SESSION
###########################################################
else:
# If the testing set session is not empty, get any updated answers
if request.method == "POST":
# Get the correct annotations from the database
correct_annotations = get_annotations(
user=report_set.gold_standard, report=completed_report)
answers = summarize_annotations(
correct_annotations)['labels']
post_keys = list(request.POST.keys())
for post_key in post_keys:
# The annotation labels have a '||'
if re.search('[||]', post_key):
new_annotation = request.POST[post_key]
user_selected = False
if new_annotation == "on":
selected_name, selected_label = post_key.split(
'||')
user_selected = True
# If we have an answer
option_chosen = ''
if selected_name in answers:
correct_answer = answers[selected_name]
# The user selected the right answer
if user_selected and correct_answer == selected_label:
testing_correct += 1
# The user selected, but not right answer
elif user_selected and correct_answer != selected_label:
testing_incorrect += 1
# The user didn't select, is right answer
elif not user_selected and correct_answer == selected_label:
testing_incorrect += 1
# The user didn't select, is not right answer
elif not user_selected and correct_answer != selected_label:
continue
# If the total number tests (correct and incorrect)
if len(testing_set) == 0:
# Did the user pass?
if testing_correct >= N_pass:
user_status = credential.status = "PASSED"
else:
user_status = credential.status = "DENIED"
credential.save()
# Update the counts
session.correct = testing_correct
session.incorrect = testing_incorrect
session.save()
# If user status is (still) TESTING, start or continue
if user_status == "TESTING":
# If we didn't select a testing report
if testing_report == None:
testing_report = testing_set.pop(0)
request.session[session_key] = testing_set
# Update the user with remaining reports
remaining_tests = N - (testing_correct + testing_incorrect)
messages.info(
request, 'You have %s reports remaining in this session' %
(remaining_tests))
# Get allowed annotations for set
testing_annotations = get_testing_annotations(report_set)
# Start testing the user
return annotate_report_testing(
request=request,
rid=testing_report.id,
sid=report_set.id,
report=testing_report,
allowed_annotations=testing_annotations)
# If the user status was approved, either previously or aboved, move on to annotation
elif user_status == "PASSED":
messages.info(
request,
"Congratulations, you have passed the testing! You are now annotating the collection set."
)
request = delete_testing_session(
request, session) # deletes session, sets reports to None
# Add the user to the set
if request.user not in report_set.annotators.all():
report_set.annotators.add(request.user)
report_set.save()
return annotate_set(request, report_set.id)
elif user_status == "DENIED":
messages.info(request,
"You are not qualified to annotate this set.")
session = get_testing_session(user=user, report_set=report_set)
request = delete_testing_session(
request, session) # deletes session, sets reports to None
return view_report_collection(request, report_set.collection.id)
# User does not have permission to annotate, return to collection view
else:
messages.info(
request, "You do not have permissions to perform this operation.")
# Denied, or not annotate permission returns to see collection
return view_report_collection(request, report_set.collection.id)