def agree_tags(delta, column):
"""
egytokenes címkézési feladatokra számol egyetértést
:param delta: az összevetett adat
:param column: az az oszlop, amelyre egyetértést akarunk számolni
:return:
"""
by_field = reverse_tags(delta, column)
task = AnnotationTask(data=by_field)
oa = task.avg_Ao() # observed agreement
s = task.S() # Bennett, Albert and Goldstein S (1954) all categories are equally likely
pi = task.pi() # Scott pi (1955) single distribution
kappa = task.kappa() # Cohen kappa (1960) individual coder distribution
w_kappa = task.weighted_kappa()
alpha = task.alpha() # Krippendorff alpha (1980)
return oa, s, pi, kappa, w_kappa, alpha
def status_view(request, task_id=None):
"""
Renders the evaluation tasks status page for staff users.
"""
LOGGER.info('Rendering evaluation task overview for user "{0}".'.format(
request.user.username))
# Check if user is member in WMT13 group. If so, redirect to wmt13 app.
if request.user.groups.filter(name="WMT13").exists():
LOGGER.info('Redirecting user "{0}" to WMT13 overview.'.format(
request.user.username))
return redirect('appraise.wmt13.views.overview')
if task_id:
task = get_object_or_404(EvaluationTask, task_id=task_id)
headers = task.get_status_header()
status = []
for user in task.users.all():
status.append((user.username, task.get_status_for_user(user)))
scores = None
result_data = []
raw_result_data = Counter()
users = list(task.users.all())
for item in EvaluationItem.objects.filter(task=task):
results = []
for user in users:
qset = EvaluationResult.objects.filter(user=user, item=item)
if qset.exists():
category = str(qset[0].results)
results.append((user.id, item.id, category))
raw_result_data[qset[0].raw_result] += 1
if len(results) == len(users):
result_data.extend(results)
_raw_results = []
_keys = raw_result_data.keys()
_total_results = float(sum(raw_result_data.values()))
for key in sorted(_keys):
value = raw_result_data[key]
_raw_results.append((key, value, 100 * value / _total_results))
try:
# Computing inter-annotator agreement only makes sense for more
# than one coder -- otherwise, we only display result_data...
if len(users) > 1:
# Check if we can safely use NLTK's AnnotationTask class.
try:
from nltk.metrics.agreement import AnnotationTask
chk = AnnotationTask(data=[('b', '1', 'k'), ('a', '1',
'k')])
assert (chk == 1.0)
except AssertionError:
LOGGER.debug('Fixing outdated version of AnnotationTask.')
from appraise.utils import AnnotationTask
# We have to sort annotation data to prevent StopIterator errors.
result_data.sort()
annotation_task = AnnotationTask(result_data)
scores = (annotation_task.alpha(), annotation_task.kappa(),
annotation_task.S(), annotation_task.pi())
except ZeroDivisionError:
scores = None
except ImportError:
scores = None
dictionary = {
'combined': task.get_status_for_users(),
'commit_tag': COMMIT_TAG,
'headers': headers,
'scores': scores,
'raw_results': _raw_results,
'status': status,
'task_id': task.task_id,
'task_name': task.task_name,
'title': 'Evaluation Task Status',
}
return render(request, 'evaluation/status_task.html', dictionary)
else:
evaluation_tasks = {}
for task_type_id, task_type in APPRAISE_TASK_TYPE_CHOICES:
# We collect a list of task descriptions for this task_type.
evaluation_tasks[task_type] = []
# Super users see all EvaluationTask items, even non-active ones.
if request.user.is_superuser:
_tasks = EvaluationTask.objects.filter(task_type=task_type_id)
else:
_tasks = EvaluationTask.objects.filter(task_type=task_type_id,
active=True)
# Loop over the QuerySet and compute task description data.
for _task in _tasks:
if not APPRAISE_TASK_CACHE.has_key(_task.task_id):
APPRAISE_TASK_CACHE[_task.task_id] = {}
_cache = APPRAISE_TASK_CACHE[_task.task_id]
if not _cache.has_key(request.user.username):
_update_task_cache(_task, request.user)
_task_data = _cache[request.user.username]
# Append new task description to current task_type list.
evaluation_tasks[task_type].append(_task_data)
# If there are no tasks descriptions for this task_type, we skip it.
if len(evaluation_tasks[task_type]) == 0:
evaluation_tasks.pop(task_type)
dictionary = {
'active_page': "STATUS",
'commit_tag': COMMIT_TAG,
'evaluation_tasks': evaluation_tasks,
'title': 'Evaluation Task Status',
}
return render(request, 'evaluation/status.html', dictionary)