def get(self):
headers = {'Content-Type': 'text/html'}
user_id = session['user_id'];
user = User.objects(id=user_id).first()
assignments = Assignment.objects()
for assignment in assignments:
assignment['owner_name'] = assignment.owner.name
assignment['owner_id'] = str(assignment.owner.id)
assignment['ds_name'] = assignment.dataset.name
assignment['complete'] = assignment.statuses.get(str(user_id), False)
return make_response(
render_template(
"annotator.html",
data={
"assignments" : assignments,
"user" : user
},
logged_in=('user_id' in session)
),
200,
headers
)
def get(self):
user = User.objects(id=session['user_id']).first()
my_datasets = Dataset.objects(owner=user)
public_datasets = Dataset.objects(privacy='public', owner__ne=user)
authorized_datasets = Dataset.objects(privacy='private', collaborators__in=[user])
classes = [class_.name for class_ in Class.objects(owner=user)]
assignments = list(map(self._collect_assignment_data, Assignment.objects(owner=user)))
return make_response(render_template(
"instructor.html",
data={
"user" : json.dumps(user.to_json()),
"my_datasets" : my_datasets,
"public_datasets" : public_datasets,
"authorized_datasets" : authorized_datasets,
"classes" : classes,
"assignments" : assignments
},
logged_in=('user_id' in session)
),
200,
{'Content-Type': 'text/html'}
)
def get(self):
headers = {'Content-Type': 'application/json'}
args = parser.parse_args()
assignment_id = args['assignment_id']
query_content = args['query_content']
documents = []
assignment = Assignment.objects(id=assignment_id).first()
query = Query.objects(assignment=assignment,content=query_content).first()
doc_scores = query.doc_scores
for doc_name in doc_scores:
score = doc_scores[doc_name]
full_doc_name = doc_name + ".txt"
# get the document and count rel_num
document = Document.objects(dataset=assignment.dataset,name=full_doc_name).first()
rel_num = Annotation.objects(document=document,query=query,judgement='relevant').count()
irrel_num = Annotation.objects(document=document,query=query,judgement='irrelevant').count()
documents.append({
'name' : full_doc_name,
'score' : score,
'rel_num' : rel_num,
'irrel_num' : irrel_num
})
documents = sorted(documents, key=lambda k: k['score'], reverse=True)
return make_response(jsonify(documents), 200, headers)
def post(self):
args = parser.parse_args()
assignment_id = args['assignment_id']
assignment = Assignment.objects(id=assignment_id).first()
name = args['name']
query = args['query']
ranker = args['ranker']
params = args['params']
dataset = args['dataset']
deadline = args['deadline']
doc_scores = args['doc_scores']
assignment.name = name
assignment.query = query
assignment.ranker = ranker
assignment.params = params
assignment.deadline = deadline
if len(doc_scores) != 0:
assignment.doc_scores = doc_scores
assignment.save()
return jsonify(assignment)
def get(self, owner_id, assignment_name):
headers = {'Content-Type': 'text/html'}
args = parser.parse_args()
owner = User.objects(id=owner_id).first()
user_id = session['user_id']
user = User.objects(id=user_id).first()
assignment = Assignment.objects(name=assignment_name) \
.filter(owner=owner).first()
# Allow re-submission
#if assignment.statuses[str(user_id)]:
# return redirect("/annotator")
assignment.owner_name = assignment.owner.name
assignment.ds_name = assignment.dataset.name
assignment.ds_owner_id = str(assignment.dataset.owner.id)
queries = Query.objects(assignment=assignment,
creator=user,
submitted=False)
return make_response(
render_template("assignment.html",
user=user,
assignment=assignment,
queries=queries), 200, headers)
def post(self):
headers = {'Content-Type': 'application/json'}
args = parser.parse_args()
assignment_id = args['assignment_id']
assignment = Assignment.objects(id=assignment_id).first()
annotations = args['annotations']
user_id = session['user_id']
# Allow re-submission
#if assignment.statuses[str(user_id)]:
# return make_response(jsonify("Failed: Already submitted assignment"), 200)
annotator = User.objects(id=user_id).first()
for query_content in annotations:
query = Query.objects(assignment=assignment,
content=query_content).first()
apq = annotations[query_content]
for file_name in apq:
label = apq[file_name]
dataset = assignment.dataset
document = Document.objects(dataset=dataset) \
.filter(name=file_name).first()
a = Annotation()
a.annotator = annotator
a.document = document
a.judgement = label
a.query = query
a.save()
Query.objects(id=query.id).update(submitted=True)
# student has completed the assignment
cstatus = assignment.statuses
cstatus[str(user_id)] = True
Assignment.objects(id=assignment_id).update(statuses=cstatus)
return make_response(jsonify("succeed"), 200, headers)
def get(self):
check_role('instructor')
datasets = [(d.name, d.author.name) for d in DataSet.objects()]
instructor = User.objects(email=current_user.email).first()
assignments = {}
for assignment in Assignment.objects(instructor=instructor):
assignments[(assignment.name, assignment.instructor.name)] = (
self.get_judgement_score(assignment))
return make_response(
render_template('instructor.html',
datasets=datasets,
assignments=assignments))
def store_annotations(annotations, assignment_id):
assignment = Assignment.objects(id=assignment_id).first()
for query_id in annotations:
query = Query.objects(id=query_id).first()
for doc_id in annotations[query_id]:
result = annotations[query_id][doc_id]
judgement = False
if result == 'T':
judgement = True
doc = Document.objects(id=doc_id).first()
user = User.objects(email=current_user.email).first()
try:
Annotation(user, query, doc, judgement, assignment).save()
except (NotUniqueError, ValidationError):
return False
AnnotatorAPI.query_nsa_filter(assignment)
return True
def get(self, instructor_name, assignment_name):
# TODO: annotation view
check_role('annotator')
instructor = User.objects(name=instructor_name).first()
assignment = Assignment.objects(name=assignment_name,
instructor=instructor).first()
if not assignment:
flash('No Such Assignment!')
return redirect(url_for('annotatorapi'))
queries = {}
assignment_score = Score.objects(assignment=assignment)
for q in assignment.queries_need_to_show:
scores = assignment_score.filter(query=q)
queries[q] = [score.document for score in scores]
return make_response(
render_template('assignment.html',
assignment=assignment,
queries=queries))
def post(self):
check_role('instructor')
args = parser.parse_args()
queries = [q for q in args['query[]'] if q != '']
if not queries:
flash('Empty Query!')
return redirect(url_for('instructorapi'))
ranker = args['ranker']
assignment_name = args['assignment_name']
dataset_name = args['ds_name']
params = self.generate_params_dict(ranker, args['param[]'])
num_results = args['num_results']
instructor = User.objects(email=current_user.email).first()
dataset = DataSet.objects(name=dataset_name).first()
assignment = Assignment(name=assignment_name,
instructor=instructor,
data_set=dataset,
ranker=ranker,
params=params,
num_results=num_results)
try:
assignment.save()
except (NotUniqueError, ValidationError):
flash('Invalid Input!')
return redirect(url_for('instructorapi'))
q = self.generate_queries(queries)
assignment.update(queries=q)
assignment.update(queries_need_to_show=q)
try:
self.search(assignment, dataset_name, queries, ranker, params,
num_results)
except Exception as e:
print(e)
flash('Assignment Created!')
return redirect(url_for('instructorapi'))
def post(self):
args = parser.parse_args()
query_content = args['query']
assignment_id = args['assignment_id']
user_id = args['user_id']
doc_scores = args['doc_scores']
assignment = Assignment.objects(id=assignment_id).first()
creator = User.objects(id=user_id).first()
query = Query()
query.content = query_content
query.assignment = assignment
query.doc_scores = doc_scores
query.creator = creator
query.submitted = False
query.save()
return "OK"
def get(self):
check_role('annotator')
return make_response(
render_template('annotator.html',
datasets=DataSet.objects(),
assignments=Assignment.objects()))
def post(self):
args = parser.parse_args()
dataset_id = args['dataset']
dataset = Dataset.objects(id=dataset_id).first()
assignments = Assignment.objects(dataset=dataset_id)
assignment_ids = [a.id for a in assignments]
queries = Query.objects(assignment__in=assignment_ids, submitted=True)
query_ids = [q.id for q in queries]
metadata_filepath = cfg["anno_dataset_base_path"] + str(
dataset.owner.gitlab_id) + "/" + dataset.name + "/metadata.data"
doc_ids = self._get_doc_ids(metadata_filepath)
to_write = []
valid_query_num = 0
for query_id in query_ids:
annotations = Annotation.objects(query=query_id)
if len(annotations) < 40:
continue
judgements = {}
for a in annotations:
doc_id = doc_ids[a.document.name]
judge_score = 1 if a.judgement == "relevant" else 0
if doc_id in judgements:
judgements[doc_id].append(judge_score)
else:
judgements[doc_id] = [judge_score]
overall_judgements = {}
is_valid = False
for doc_id in judgements:
judgem = int(
round(
float(sum(judgements[doc_id])) /
len(judgements[doc_id])))
if judgem > 0:
overall_judgements[doc_id] = judgem
is_valid = True
if is_valid:
entries = {"docs": [], "query_id": query_id}
for doc_id in overall_judgements:
entry = (valid_query_num, doc_id,
overall_judgements[doc_id])
entries["docs"].append(entry)
to_write.append(entries)
valid_query_num += 1
old_dataset_path = cfg["anno_dataset_base_path"] + str(
dataset.owner.gitlab_id) + "/" + dataset.name
new_dataset_path = cfg["perm_dataset_base_path"]
self._copy_dataset(old_dataset_path,
new_dataset_path + '/' + dataset.name)
self._update_dataset_config(new_dataset_path, dataset.name)
path = cfg["perm_dataset_base_path"] + '/' + dataset.name + "/"
qrels_filepath = path + dataset.name + "-qrels.txt"
queries_filepath = path + dataset.name + "-queries.txt"
self._write_query_files(to_write, queries_filepath, qrels_filepath)
response = {
"status": "success",
"queries_filepath": queries_filepath,
"qrels_filepath": qrels_filepath
}
return make_response(jsonify(response))
def post(self):
headers = {'Content-Type': 'application/json'}
args = parser.parse_args()
name = args['name']
ranker = args['ranker']
params = args['params']
dataset = args['dataset']
deadline = args['deadline']
doc_scores = args['doc_scores']
num_results = args['num_results']
assignment = Assignment()
assignment.name = name
assignment.owner = User.objects(id=session['user_id']).first()
assignment.ranker = ranker
assignment.params = params
assignment.dataset = Dataset.objects(id=dataset).first()
assignment.annotators = User.objects()
assignment.statuses = {
str(anno.id): False
for anno in assignment.annotators
}
assignment.deadline = deadline
assignment.num_results = num_results
assignment.save()
return str(assignment.id)