def _compute_global_stats():
"""
Computes some global statistics for the WMT14 evaluation campaign.
"""
global_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
# Check how many HITs have been completed. We now consider a HIT to be
# completed once it has been annotated by one or more annotators.
#
# Before we required `hit.users.count() >= 3` for greater overlap.
hits_completed = HIT.objects.filter(mturk_only=False,
completed=True).count()
# Check any remaining active HITs which are not yet marked complete.
for hit in HIT.objects.filter(active=True,
mturk_only=False,
completed=False):
if hit.users.count() >= 1:
hits_completed = hits_completed + 1
hit.completed = True
hit.save()
# Compute remaining HITs for all language pairs.
hits_remaining = HIT.compute_remaining_hits()
# Compute number of results contributed so far.
ranking_results = RankingResult.objects.filter(
item__hit__completed=True, item__hit__mturk_only=False).count()
# Aggregate information about participating groups.
groups = set()
for user in users:
for group in user.groups.all():
if group.name == 'WMT14' or group.name.startswith('eng2') \
or group.name.endswith('2eng'):
continue
groups.add(group)
# Compute average/total duration over all results.
durations = RankingResult.objects.all().values_list('duration', flat=True)
total_time = sum([datetime_to_seconds(x) for x in durations])
avg_time = total_time / float(hits_completed or 1)
avg_user_time = total_time / float(3 * hits_completed or 1)
global_stats.append(('Users', users.count()))
global_stats.append(('Groups', len(groups)))
global_stats.append(('HITs completed', hits_completed))
global_stats.append(('HITs remaining', hits_remaining))
global_stats.append(('Ranking results', ranking_results))
global_stats.append(('System comparisons', 10 * ranking_results))
global_stats.append(('Average duration', seconds_to_timedelta(avg_time)))
global_stats.append(('Average duration (single user)',
seconds_to_timedelta(avg_user_time)))
global_stats.append(('Total duration', seconds_to_timedelta(total_time)))
return global_stats
def _compute_global_stats():
"""
Computes some global statistics for the WMT14 evaluation campaign.
"""
global_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
# Check how many HITs have been completed. We now consider a HIT to be
# completed once it has been annotated by one or more annotators.
#
# Before we required `hit.users.count() >= 3` for greater overlap.
hits_completed = HIT.objects.filter(mturk_only=False, completed=True).count()
# Check any remaining active HITs which are not yet marked complete.
for hit in HIT.objects.filter(active=True, mturk_only=False, completed=False):
if hit.users.count() >= 1:
hits_completed = hits_completed + 1
hit.completed = True
hit.save()
# Compute remaining HITs for all language pairs.
hits_remaining = HIT.compute_remaining_hits()
# Compute number of results contributed so far.
ranking_results = RankingResult.objects.filter(
item__hit__completed=True, item__hit__mturk_only=False).count()
# Aggregate information about participating groups.
groups = set()
for user in users:
for group in user.groups.all():
if group.name == 'WMT14' or group.name.startswith('eng2') \
or group.name.endswith('2eng'):
continue
groups.add(group)
# Compute average/total duration over all results.
durations = RankingResult.objects.all().values_list('duration', flat=True)
total_time = sum([datetime_to_seconds(x) for x in durations])
avg_time = total_time / float(hits_completed or 1)
avg_user_time = total_time / float(3 * hits_completed or 1)
global_stats.append(('Users', users.count()))
global_stats.append(('Groups', len(groups)))
global_stats.append(('HITs completed', hits_completed))
global_stats.append(('HITs remaining', hits_remaining))
global_stats.append(('Ranking results', ranking_results))
global_stats.append(('System comparisons', 10 * ranking_results))
global_stats.append(('Average duration', seconds_to_timedelta(avg_time)))
global_stats.append(('Average duration (single user)',
seconds_to_timedelta(avg_user_time)))
global_stats.append(('Total duration', seconds_to_timedelta(total_time)))
return global_stats
def _compute_group_stats():
"""
Computes group statistics for the WMT14 evaluation campaign.
"""
group_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
# Aggregate information about participating groups.
groups = set()
for user in users:
for group in user.groups.all():
if group.name == 'WMT14' or group.name.startswith('eng2') \
or group.name.endswith('2eng'):
continue
groups.add(group)
# The following dictionary defines the number of HITs each group should
# have completed during the WMT14 evaluation campaign.
group_hit_requirements = {
'MSR': 0, 'JHU': 0, 'PROMT': 800, 'KIT': 400, 'UM': 0,
'DCU-Prompsit-UA': 200, 'RWTH': 100, 'CIS/IMS': 100, 'Eu-Bridge': 200,
'IIT-Bombay': 300, 'YSDA': 300, 'UU': 300, 'UA-Prompsit': 100,
'IMS-TTT': 100, 'UFAL': 600, 'AFRL': 300, 'UEDIN': 1900,
'Stanford': 300, 'UB-Grial': 100, 'QCRI': 300, 'LIMSI': 100,
'USAAR': 300, 'IPN-UPV': 200, 'CMU': 200, 'TALP-UPC': 200,
'SFU': 0,
}
for group in groups:
_name = group.name
if not _name in group_hit_requirements.keys():
continue
_group_stats = HIT.compute_status_for_group(group)
_total = _group_stats[0]
_required = group_hit_requirements[_name]
_delta = _total - _required
_data = (_total, _required, _delta)
if _data[0] > 0:
group_stats.append((_name, _data))
# Sort by number of remaining HITs.
group_stats.sort(key=lambda x: x[1][2])
# Add totals at the bottom.
global_total = sum([x[1][0] for x in group_stats])
global_required = sum([x[1][1] for x in group_stats])
global_delta = global_total - global_required
global_data = (global_total, global_required, global_delta)
group_stats.append(("Totals", global_data))
return group_stats
def _compute_user_stats():
"""
Computes user statistics for the WMT14 evaluation campaign.
"""
user_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
for user in users:
_user_stats = HIT.compute_status_for_user(user)
_name = user.username
_avg_time = seconds_to_timedelta(_user_stats[1])
_total_time = seconds_to_timedelta(_user_stats[2])
_data = (_name, _user_stats[0], _avg_time, _total_time)
if _data[0] > 0:
user_stats.append(_data)
# Sort by total number of completed HITs.
user_stats.sort(key=lambda x: x[1])
user_stats.reverse()
return user_stats
def _compute_user_stats():
"""
Computes user statistics for the WMT14 evaluation campaign.
"""
user_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
for user in users:
_user_stats = HIT.compute_status_for_user(user)
_name = user.username
_avg_time = seconds_to_timedelta(_user_stats[1])
_total_time = seconds_to_timedelta(_user_stats[2])
_data = (_name, _user_stats[0], _avg_time, _total_time)
if _data[0] > 0:
user_stats.append(_data)
# Sort by total number of completed HITs.
user_stats.sort(key=lambda x: x[1])
user_stats.reverse()
return user_stats
def _compute_language_pair_stats():
"""
Computes HIT statistics per language pair.
"""
language_pair_stats = []
# Running compute_remaining_hits() will also update completion status for HITs.
for choice in LANGUAGE_PAIR_CHOICES:
_code = choice[0]
_name = choice[1]
_remaining_hits = HIT.compute_remaining_hits(language_pair=_code)
_completed_hits = HIT.objects.filter(completed=True,
mturk_only=False,
language_pair=_code).count()
_total_hits = _remaining_hits + _completed_hits
_data = (_name, (_remaining_hits,
100 * _remaining_hits / float(_total_hits or 1)),
(_completed_hits,
100 * _completed_hits / float(_total_hits or 1)))
language_pair_stats.append(_data)
return language_pair_stats
def _compute_language_pair_stats():
"""
Computes HIT statistics per language pair.
"""
language_pair_stats = []
# Running compute_remaining_hits() will also update completion status for HITs.
for choice in LANGUAGE_PAIR_CHOICES:
_code = choice[0]
_name = choice[1]
_remaining_hits = HIT.compute_remaining_hits(language_pair=_code)
_completed_hits = HIT.objects.filter(completed=True, mturk_only=False,
language_pair=_code).count()
_total_hits = _remaining_hits + _completed_hits
_data = (
_name,
(_remaining_hits, 100 * _remaining_hits/float(_total_hits or 1)),
(_completed_hits, 100 * _completed_hits/float(_total_hits or 1))
)
language_pair_stats.append(_data)
return language_pair_stats
# Properly set DJANGO_SETTINGS_MODULE environment variable.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
PROJECT_HOME = os.path.normpath(os.getcwd() + "/..")
sys.path.append(PROJECT_HOME)
# We have just added appraise to the system path list, hence this works.
from django.contrib.auth.models import User, Group
from appraise.wmt14.models import HIT
# Compute user statistics for all users.
user_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
for user in users:
_user_stats = HIT.compute_status_for_user(user)
_name = user.username
_email = user.email
_group = "UNDEFINED"
for _g in user.groups.all():
if _g.name.startswith("eng2") \
or _g.name.endswith("2eng") \
or _g.name == "WMT14":
continue
_group = _g.name
break
_data = (_name, _email, _group, _user_stats[0], _user_stats[2])
user_stats.append(_data)
def _compute_group_stats():
"""
Computes group statistics for the WMT14 evaluation campaign.
"""
group_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
# Aggregate information about participating groups.
groups = set()
for user in users:
for group in user.groups.all():
if group.name == 'WMT14' or group.name.startswith('eng2') \
or group.name.endswith('2eng'):
continue
groups.add(group)
# The following dictionary defines the number of HITs each group should
# have completed during the WMT14 evaluation campaign.
group_hit_requirements = {
'MSR': 0,
'JHU': 0,
'PROMT': 800,
'KIT': 400,
'UM': 0,
'DCU-Prompsit-UA': 200,
'RWTH': 100,
'CIS/IMS': 100,
'Eu-Bridge': 200,
'IIT-Bombay': 300,
'YSDA': 300,
'UU': 300,
'UA-Prompsit': 100,
'IMS-TTT': 100,
'UFAL': 600,
'AFRL': 300,
'UEDIN': 1900,
'Stanford': 300,
'UB-Grial': 100,
'QCRI': 300,
'LIMSI': 100,
'USAAR': 300,
'IPN-UPV': 200,
'CMU': 200,
'TALP-UPC': 200,
'SFU': 0,
}
for group in groups:
_name = group.name
if not _name in group_hit_requirements.keys():
continue
_group_stats = HIT.compute_status_for_group(group)
_total = _group_stats[0]
_required = group_hit_requirements[_name]
_delta = _total - _required
_data = (_total, _required, _delta)
if _data[0] > 0:
group_stats.append((_name, _data))
# Sort by number of remaining HITs.
group_stats.sort(key=lambda x: x[1][2])
# Add totals at the bottom.
global_total = sum([x[1][0] for x in group_stats])
global_required = sum([x[1][1] for x in group_stats])
global_delta = global_total - global_required
global_data = (global_total, global_required, global_delta)
group_stats.append(("Totals", global_data))
return group_stats
def overview(request):
"""
Renders the evaluation tasks overview.
"""
LOGGER.info('Rendering WMT14 HIT overview for user "{0}".'.format(
request.user.username or "Anonymous"))
# Re-initialise random number generator.
seed(None)
# Collect available language pairs for the current user.
language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
language_pairs = request.user.groups.filter(name__in=language_codes)
hit_data = []
total = [0, 0, 0]
for language_pair in language_pairs:
hit = _compute_next_task_for_user(request.user, language_pair)
user_status = HIT.compute_status_for_user(request.user, language_pair)
for i in range(3):
total[i] = total[i] + user_status[i]
if hit:
# Convert status seconds back into datetime.time instances.
for i in range(2):
user_status[i + 1] = seconds_to_timedelta(
int(user_status[i + 1]))
hit_data.append((hit.get_language_pair_display(),
hit.get_absolute_url(), hit.hit_id, user_status))
# Convert total seconds back into datetime.timedelta instances.
total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))
# Remove microseconds to get a nicer timedelta rendering in templates.
total[1] = total[1] - timedelta(microseconds=total[1].microseconds)
total[2] = seconds_to_timedelta(int(total[2]))
group = None
for _group in request.user.groups.all():
if _group.name == 'WMT14' \
or _group.name.startswith('eng2') \
or _group.name.endswith('2eng'):
continue
group = _group
break
if group is not None:
group_name = group.name
group_status = HIT.compute_status_for_group(group)
for i in range(2):
group_status[i + 1] = seconds_to_timedelta(int(group_status[i +
1]))
else:
group_status = None
group_name = None
LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
request.user.username or "Anonymous",
u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))
# Compute admin URL for super users.
admin_url = None
if request.user.is_superuser:
admin_url = reverse('admin:index')
dictionary = {
'active_page': "OVERVIEW",
'commit_tag': COMMIT_TAG,
'hit_data': hit_data,
'total': total,
'group_name': group_name,
'group_status': group_status,
'admin_url': admin_url,
'title': 'WMT14 Dashboard',
}
return render(request, 'wmt14/overview.html', dictionary)
# Hotfix potentially wrong ISO codes; we are using ISO-639-3.
iso_639_2_to_3_mapping = {'cze': 'ces', 'fre': 'fra', 'ger': 'deu'}
for part2_code, part3_code in iso_639_2_to_3_mapping.items():
language_pair = language_pair.replace(part2_code, part3_code)
try:
_total = _total + 1
_hit_xml = tostring(_child, encoding="utf-8").decode('utf-8')
if args.dry_run_enabled:
_ = HIT(block_id=block_id, hit_xml=_hit_xml,
language_pair=language_pair, mturk_only=args.mturk_only)
else:
# Use get_or_create() to avoid exact duplicates. We do allow
# them for WMT14 to measure intra-annotator agreement...
h = HIT(block_id=block_id, hit_xml=_hit_xml,
language_pair=language_pair, mturk_only=args.mturk_only)
h.save()
# pylint: disable-msg=W0703
except Exception, msg:
print msg
_errors = _errors + 1
print
print '[{0}]'.format(_hits_file)
print 'Successfully imported {0} HITs, encountered errors for ' \
'{1} HITs.'.format(_total, _errors)
print
block_id = _child.attrib["block-id"]
language_pair = '{0}2{1}'.format(_child.attrib["source-language"],
_child.attrib["target-language"])
# Hotfix potentially wrong ISO codes; we are using ISO-639-3.
iso_639_2_to_3_mapping = {'cze': 'ces', 'fre': 'fra', 'ger': 'deu'}
for part2_code, part3_code in iso_639_2_to_3_mapping.items():
language_pair = language_pair.replace(part2_code, part3_code)
try:
_total = _total + 1
_hit_xml = tostring(_child, encoding="utf-8").decode('utf-8')
if args.dry_run_enabled:
_ = HIT(block_id=block_id,
hit_xml=_hit_xml,
language_pair=language_pair,
mturk_only=args.mturk_only)
else:
# Use get_or_create() to avoid exact duplicates. We do allow
# them for WMT14 to measure intra-annotator agreement...
h = HIT(block_id=block_id,
hit_xml=_hit_xml,
language_pair=language_pair,
mturk_only=args.mturk_only)
h.save()
# pylint: disable-msg=W0703
except Exception, msg:
print msg
_errors = _errors + 1
usage: export_wmt14_status.py
Exports HIT status for all language pairs.
"""
from datetime import datetime
import os
import sys
if __name__ == "__main__":
# Properly set DJANGO_SETTINGS_MODULE environment variable.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
PROJECT_HOME = os.path.normpath(os.getcwd() + "/..")
sys.path.append(PROJECT_HOME)
# We have just added appraise to the system path list, hence this works.
from appraise.wmt14.models import HIT, LANGUAGE_PAIR_CHOICES
remaining_hits = {}
for language_pair in [x[0] for x in LANGUAGE_PAIR_CHOICES]:
remaining_hits[language_pair] = HIT.compute_remaining_hits(
language_pair=language_pair)
print
print '[{0}]'.format(datetime.now().strftime("%c"))
for k, v in remaining_hits.items():
print '{0}: {1:03d}'.format(k, v)
print
Author: Christian Federmann <*****@*****.**>
usage: export_wmt14_status.py
Exports HIT status for all language pairs.
"""
from datetime import datetime
import os
import sys
if __name__ == "__main__":
# Properly set DJANGO_SETTINGS_MODULE environment variable.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
PROJECT_HOME = os.path.normpath(os.getcwd() + "/..")
sys.path.append(PROJECT_HOME)
# We have just added appraise to the system path list, hence this works.
from appraise.wmt14.models import HIT, LANGUAGE_PAIR_CHOICES
remaining_hits = {}
for language_pair in [x[0] for x in LANGUAGE_PAIR_CHOICES]:
remaining_hits[language_pair] = HIT.compute_remaining_hits(
language_pair=language_pair)
print
print '[{0}]'.format(datetime.now().strftime("%c"))
for k, v in remaining_hits.items():
print '{0}: {1:03d}'.format(k, v)
print
def overview(request):
"""
Renders the evaluation tasks overview.
"""
LOGGER.info('Rendering WMT14 HIT overview for user "{0}".'.format(
request.user.username or "Anonymous"))
# Re-initialise random number generator.
seed(None)
# Collect available language pairs for the current user.
language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
language_pairs = request.user.groups.filter(name__in=language_codes)
hit_data = []
total = [0, 0, 0]
for language_pair in language_pairs:
hit = _compute_next_task_for_user(request.user, language_pair)
user_status = HIT.compute_status_for_user(request.user, language_pair)
for i in range(3):
total[i] = total[i] + user_status[i]
if hit:
# Convert status seconds back into datetime.time instances.
for i in range(2):
user_status[i+1] = seconds_to_timedelta(int(user_status[i+1]))
hit_data.append(
(hit.get_language_pair_display(), hit.get_absolute_url(),
hit.hit_id, user_status)
)
# Convert total seconds back into datetime.timedelta instances.
total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))
# Remove microseconds to get a nicer timedelta rendering in templates.
total[1] = total[1] - timedelta(microseconds=total[1].microseconds)
total[2] = seconds_to_timedelta(int(total[2]))
group = None
for _group in request.user.groups.all():
if _group.name == 'WMT14' \
or _group.name.startswith('eng2') \
or _group.name.endswith('2eng'):
continue
group = _group
break
if group is not None:
group_name = group.name
group_status = HIT.compute_status_for_group(group)
for i in range(2):
group_status[i+1] = seconds_to_timedelta(int(group_status[i+1]))
else:
group_status = None
group_name = None
LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
request.user.username or "Anonymous",
u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))
# Compute admin URL for super users.
admin_url = None
if request.user.is_superuser:
admin_url = reverse('admin:index')
dictionary = {
'active_page': "OVERVIEW",
'commit_tag': COMMIT_TAG,
'hit_data': hit_data,
'total': total,
'group_name': group_name,
'group_status': group_status,
'admin_url': admin_url,
'title': 'WMT14 Dashboard',
}
return render(request, 'wmt14/overview.html', dictionary)
# Properly set DJANGO_SETTINGS_MODULE environment variable.
os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
PROJECT_HOME = os.path.normpath(os.getcwd() + "/..")
sys.path.append(PROJECT_HOME)
# We have just added appraise to the system path list, hence this works.
from django.contrib.auth.models import User, Group
from appraise.wmt14.models import HIT
# Compute user statistics for all users.
user_stats = []
wmt14 = Group.objects.get(name='WMT14')
users = wmt14.user_set.all()
for user in users:
_user_stats = HIT.compute_status_for_user(user)
_name = user.username
_email = user.email
_group = "UNDEFINED"
for _g in user.groups.all():
if _g.name.startswith("eng2") \
or _g.name.endswith("2eng") \
or _g.name == "WMT14":
continue
_group = _g.name
break
_data = (_name, _email, _group, _user_stats[0], _user_stats[2])
user_stats.append(_data)
