Esempio n. 1
0
def _compute_user_stats():
    """
    Computes user statistics for the WMT15 evaluation campaign.
    """
    user_stats = []
    
    wmt15_group = Group.objects.filter(name='WMT15')
    wmt15_users = []
    if wmt15_group.exists():
        wmt15_users = wmt15_group[0].user_set.all()
    
    for user in wmt15_users:
        _user_stats = HIT.compute_status_for_user(user)
        _name = user.username
        _avg_time = seconds_to_timedelta(_user_stats[1])
        _total_time = seconds_to_timedelta(_user_stats[2])
        _data = (_name, _user_stats[0], _avg_time, _total_time)
        
        if _data[0] > 0:
            user_stats.append(_data)
    
    # Sort by total number of completed HITs.
    user_stats.sort(key=lambda x: x[1])
    user_stats.reverse()
    
    return user_stats
Esempio n. 2
0
def _compute_user_stats():
    """
    Computes user statistics for the WMT13 evaluation campaign.
    """
    user_stats = []
    wmt13 = Group.objects.get(name='WMT13')
    users = wmt13.user_set.all()
    
    for user in users:
        _user_stats = HIT.compute_status_for_user(user)
        _name = user.username
        _avg_time = seconds_to_timedelta(_user_stats[1])
        _total_time = seconds_to_timedelta(_user_stats[2])
        _data = (_name, _user_stats[0], _avg_time, _total_time)
        
        if _data[0] > 0:
            user_stats.append(_data)
    
    # Sort by total number of completed HITs.
    user_stats.sort(key=lambda x: x[1])
    user_stats.reverse()
    
    # Only show top 25 contributors.
    user_stats = user_stats[:25]
    
    return user_stats
Esempio n. 3
0
def _compute_global_stats():
    """
    Computes some global statistics for the WMT14 evaluation campaign.
    """
    global_stats = []
    wmt14 = Group.objects.get(name='WMT14')
    users = wmt14.user_set.all()

    # Check how many HITs have been completed.  We now consider a HIT to be
    # completed once it has been annotated by one or more annotators.
    #
    # Before we required `hit.users.count() >= 3` for greater overlap.
    hits_completed = HIT.objects.filter(mturk_only=False,
                                        completed=True).count()

    # Check any remaining active HITs which are not yet marked complete.
    for hit in HIT.objects.filter(active=True,
                                  mturk_only=False,
                                  completed=False):
        if hit.users.count() >= 1:
            hits_completed = hits_completed + 1
            hit.completed = True
            hit.save()

    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()

    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.filter(
        item__hit__completed=True, item__hit__mturk_only=False).count()

    # Aggregate information about participating groups.
    groups = set()
    for user in users:
        for group in user.groups.all():
            if group.name == 'WMT14' or group.name.startswith('eng2') \
              or group.name.endswith('2eng'):
                continue

            groups.add(group)

    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list('duration', flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)
    avg_user_time = total_time / float(3 * hits_completed or 1)

    global_stats.append(('Users', users.count()))
    global_stats.append(('Groups', len(groups)))
    global_stats.append(('HITs completed', hits_completed))
    global_stats.append(('HITs remaining', hits_remaining))
    global_stats.append(('Ranking results', ranking_results))
    global_stats.append(('System comparisons', 10 * ranking_results))
    global_stats.append(('Average duration', seconds_to_timedelta(avg_time)))
    global_stats.append(('Average duration (single user)',
                         seconds_to_timedelta(avg_user_time)))
    global_stats.append(('Total duration', seconds_to_timedelta(total_time)))

    return global_stats
def _compute_global_stats():
    """
    Computes some global statistics for the WMT14 evaluation campaign.
    """
    global_stats = []
    wmt14 = Group.objects.get(name='WMT14')
    users = wmt14.user_set.all()
    
    # Check how many HITs have been completed.  We now consider a HIT to be
    # completed once it has been annotated by one or more annotators.
    #
    # Before we required `hit.users.count() >= 3` for greater overlap.
    hits_completed = HIT.objects.filter(mturk_only=False, completed=True).count()
    
    # Check any remaining active HITs which are not yet marked complete.
    for hit in HIT.objects.filter(active=True, mturk_only=False, completed=False):
        if hit.users.count() >= 1:
            hits_completed = hits_completed + 1
            hit.completed = True
            hit.save()
    
    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()
    
    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.filter(
      item__hit__completed=True, item__hit__mturk_only=False).count()
    
    # Aggregate information about participating groups.
    groups = set()
    for user in users:
        for group in user.groups.all():
            if group.name == 'WMT14' or group.name.startswith('eng2') \
              or group.name.endswith('2eng'):
                continue
            
            groups.add(group)
    
    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list('duration', flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)
    avg_user_time = total_time / float(3 * hits_completed or 1)
    
    global_stats.append(('Users', users.count()))
    global_stats.append(('Groups', len(groups)))
    global_stats.append(('HITs completed', hits_completed))
    global_stats.append(('HITs remaining', hits_remaining))
    global_stats.append(('Ranking results', ranking_results))
    global_stats.append(('System comparisons', 10 * ranking_results))
    global_stats.append(('Average duration', seconds_to_timedelta(avg_time)))
    global_stats.append(('Average duration (single user)',
      seconds_to_timedelta(avg_user_time)))
    global_stats.append(('Total duration', seconds_to_timedelta(total_time)))
    
    return global_stats
Esempio n. 5
0
def status(request):
    """
    Renders the status overview.
    """
    LOGGER.info('Rendering WMT13 HIT status for user "{0}".'.format(request.user.username or "Anonymous"))

    # Compute some global stats.
    global_stats = []
    wmt13 = Group.objects.get(name="WMT13")
    users = wmt13.user_set.all()

    # Check how many HITs have been completed.
    hits_completed = 0
    for hit in HIT.objects.all():
        if hit.users.count() >= 3:
            hits_completed = hits_completed + 1

    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()

    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.all().count()

    # Aggregate information about participating groups.
    groups = set()
    for user in users:
        for group in user.groups.all():
            if group.name == "WMT13" or group.name.startswith("eng2") or group.name.endswith("2eng"):
                continue

            groups.add(group)

    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list("duration", flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)

    global_stats.append(("Users", users.count()))
    global_stats.append(("Groups", len(groups)))
    global_stats.append(("HITs completed", hits_completed))
    global_stats.append(("HITs remaining", hits_remaining))
    global_stats.append(("Ranking results", ranking_results))
    global_stats.append(("System comparisons", 10 * ranking_results))
    global_stats.append(("Average duration", seconds_to_timedelta(avg_time)))
    global_stats.append(("Total duration", seconds_to_timedelta(total_time)))

    dictionary = {
        "active_page": "STATUS",
        "global_stats": global_stats,
        "commit_tag": COMMIT_TAG,
        "title": "WMT13 Status",
    }

    return render(request, "wmt13/status.html", dictionary)
Esempio n. 6
0
def overview(request):
    """
    Renders the evaluation tasks overview.
    """
    LOGGER.info('Rendering WMT14 HIT overview for user "{0}".'.format(
        request.user.username or "Anonymous"))

    # Re-initialise random number generator.
    seed(None)

    # Collect available language pairs for the current user.
    language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
    language_pairs = request.user.groups.filter(name__in=language_codes)

    hit_data = []
    total = [0, 0, 0]
    for language_pair in language_pairs:
        hit = _compute_next_task_for_user(request.user, language_pair)
        user_status = HIT.compute_status_for_user(request.user, language_pair)
        for i in range(3):
            total[i] = total[i] + user_status[i]

        if hit:
            # Convert status seconds back into datetime.time instances.
            for i in range(2):
                user_status[i + 1] = seconds_to_timedelta(
                    int(user_status[i + 1]))

            hit_data.append((hit.get_language_pair_display(),
                             hit.get_absolute_url(), hit.hit_id, user_status))

    # Convert total seconds back into datetime.timedelta instances.
    total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))

    # Remove microseconds to get a nicer timedelta rendering in templates.
    total[1] = total[1] - timedelta(microseconds=total[1].microseconds)

    total[2] = seconds_to_timedelta(int(total[2]))

    group = None
    for _group in request.user.groups.all():
        if _group.name == 'WMT14' \
          or _group.name.startswith('eng2') \
          or _group.name.endswith('2eng'):
            continue

        group = _group
        break

    if group is not None:
        group_name = group.name
        group_status = HIT.compute_status_for_group(group)
        for i in range(2):
            group_status[i + 1] = seconds_to_timedelta(int(group_status[i +
                                                                        1]))

    else:
        group_status = None
        group_name = None

    LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
        request.user.username or "Anonymous",
        u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))

    # Compute admin URL for super users.
    admin_url = None
    if request.user.is_superuser:
        admin_url = reverse('admin:index')

    dictionary = {
        'active_page': "OVERVIEW",
        'commit_tag': COMMIT_TAG,
        'hit_data': hit_data,
        'total': total,
        'group_name': group_name,
        'group_status': group_status,
        'admin_url': admin_url,
        'title': 'WMT14 Dashboard',
    }

    return render(request, 'wmt14/overview.html', dictionary)
Esempio n. 7
0
def overview(request):
    """
    Renders the evaluation tasks overview.
    """
    LOGGER.info('Rendering WMT13 HIT overview for user "{0}".'.format(request.user.username or "Anonymous"))

    # Re-initialise random number generator.
    seed(None)

    # Collect available language pairs for the current user.
    language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
    language_pairs = request.user.groups.filter(name__in=language_codes)

    hit_data = []
    total = [0, 0, 0]
    for language_pair in language_pairs:
        hit = _compute_next_task_for_user(request.user, language_pair)
        status = HIT.compute_status_for_user(request.user, language_pair)
        for i in range(3):
            total[i] = total[i] + status[i]

        if hit:
            # Convert status seconds back into datetime.time instances.
            for i in range(2):
                status[i + 1] = seconds_to_timedelta(int(status[i + 1]))

            hit_data.append((hit.get_language_pair_display(), hit.get_absolute_url(), hit.block_id, status))

    # Convert total seconds back into datetime.time instances.
    for i in range(2):
        total[i + 1] = seconds_to_timedelta(int(total[i + 1]))

    group = None
    for _group in request.user.groups.all():
        if _group.name == "WMT13" or _group.name.startswith("eng2") or _group.name.endswith("2eng"):
            continue

        group = _group
        break

    if group is not None:
        group_name = group.name
        group_status = HIT.compute_status_for_group(group)
        for i in range(2):
            group_status[i + 1] = seconds_to_timedelta(int(group_status[i + 1]))

    else:
        group_status = None
        group_name = None

    LOGGER.debug(
        u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
            request.user.username or "Anonymous", u"\n".join([u"{0}\t{1}\t{2}\t{3}".format(*x) for x in hit_data])
        )
    )

    dictionary = {
        "active_page": "OVERVIEW",
        "commit_tag": COMMIT_TAG,
        "hit_data": hit_data,
        "total": total,
        "group_name": group_name,
        "group_status": group_status,
        "title": "WMT13 Dashboard",
    }

    return render(request, "wmt13/overview.html", dictionary)
Esempio n. 8
0
def overview(request):
    """
    Renders the evaluation tasks overview.
    """
    LOGGER.info('Rendering WMT13 HIT overview for user "{0}".'.format(
      request.user.username or "Anonymous"))
    
    # Re-initialise random number generator.
    seed(None)
    
    # Collect available language pairs for the current user.
    language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
    language_pairs = request.user.groups.filter(name__in=language_codes)
    
    hit_data = []
    total = [0, 0, 0]
    for language_pair in language_pairs:
        hit = _compute_next_task_for_user(request.user, language_pair)
        user_status = HIT.compute_status_for_user(request.user, language_pair)
        for i in range(3):
            total[i] = total[i] + user_status[i]
        
        if hit:
            # Convert status seconds back into datetime.time instances.
            for i in range(2):
                user_status[i+1] = seconds_to_timedelta(int(user_status[i+1]))
            
            hit_data.append(
              (hit.get_language_pair_display(), hit.get_absolute_url(),
               hit.block_id, user_status)
            )
    
    # Convert total seconds back into datetime.timedelta instances.
    total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))
    
    # Remove microseconds to get a nicer timedelta rendering in templates.
    total[1] = total[1] - timedelta(microseconds=total[1].microseconds)
    
    total[2] = seconds_to_timedelta(int(total[2]))
    
    group = None
    for _group in request.user.groups.all():
        if _group.name == 'WMT13' \
          or _group.name.startswith('eng2') \
          or _group.name.endswith('2eng'):
            continue
        
        group = _group
        break
    
    if group is not None:
        group_name = group.name
        group_status = HIT.compute_status_for_group(group)
        for i in range(2):
            group_status[i+1] = seconds_to_timedelta(int(group_status[i+1]))
    
    else:
        group_status = None
        group_name = None
    
    LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
      request.user.username or "Anonymous",
      u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))
    
    dictionary = {
      'active_page': "OVERVIEW",
      'commit_tag': COMMIT_TAG,
      'hit_data': hit_data,
      'total': total,
      'group_name': group_name,
      'group_status': group_status,
      'title': 'WMT13 Dashboard',
    }
    
    return render(request, 'wmt13/overview.html', dictionary)
Esempio n. 9
0
def _compute_global_stats():
    """
    Computes some global statistics for the WMT15 evaluation campaign.
    """
    global_stats = []
    
    wmt15_group = Group.objects.filter(name='WMT15')
    wmt15_users = []
    if wmt15_group.exists():
        wmt15_users = wmt15_group[0].user_set.all()
      
    # Check how many HITs have been completed.  We now consider a HIT to be
    # completed once it has been annotated by one or more annotators.
    #
    # Before we required `hit.users.count() >= 3` for greater overlap.
    hits_completed = HIT.objects.filter(mturk_only=False, completed=True).count()
    
    # Check any remaining active HITs which are not yet marked complete.
    for hit in HIT.objects.filter(active=True, mturk_only=False, completed=False):
        if hit.users.count() >= 1:
            hits_completed = hits_completed + 1
            hit.completed = True
            hit.save()
    
    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()
    
    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.filter(
      item__hit__completed=True, item__hit__mturk_only=False)
    
    from math import factorial
    system_comparisons = 0
    for result in ranking_results:
        result.reload_dynamic_fields()
        combinations = factorial(result.systems)/(factorial(result.systems-2) * 2) if result.systems > 2 else 0
        system_comparisons = system_comparisons + combinations
    
    # Aggregate information about participating groups.
    groups = set()
    for user in wmt15_users:
        for group in user.groups.all():
            if group.name == 'WMT15' or group.name.startswith('eng2') \
              or group.name.endswith('2eng'):
                continue
            
            groups.add(group)
    
    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list('duration', flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)
    avg_user_time = total_time / float(3 * hits_completed or 1)
    
    global_stats.append(('Users', len(wmt15_users)))
    global_stats.append(('Groups', len(groups)))
    global_stats.append(('HITs completed', '{0:,}'.format(hits_completed)))
    global_stats.append(('HITs remaining', '{0:,}'.format(hits_remaining)))
    global_stats.append(('Ranking results', '{0:,}'.format(ranking_results.count())))
    global_stats.append(('System comparisons', '{0:,}'.format(system_comparisons)))
    global_stats.append(('Average duration (per HIT)', seconds_to_timedelta(avg_time)))
    global_stats.append(('Average duration (per task)', seconds_to_timedelta(avg_user_time)))
    global_stats.append(('Total duration', seconds_to_timedelta(total_time)))
    
    return global_stats
Esempio n. 10
0
def _compute_global_stats():
    """
    Computes some global statistics for the WMT16 evaluation campaign.
    """
    global_stats = []
    
    wmt16_group = Group.objects.filter(name='WMT16')
    wmt16_users = _get_active_users_for_group(wmt16_group)
      
    # Check how many HITs have been completed.  We now consider a HIT to be
    # completed once it has been annotated by one or more annotators.
    #
    # Before we required `hit.users.count() >= 3` for greater overlap.
    hits_completed = HIT.objects.filter(mturk_only=False, completed=True).count()
    
    # Check any remaining active HITs which are not yet marked complete.
    for hit in HIT.objects.filter(active=True, mturk_only=False, completed=False):
        if hit.users.count() >= 1:
            hits_completed = hits_completed + 1
            hit.completed = True
            hit.save()
    
    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()
    
    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.filter(
      item__hit__completed=True, item__hit__mturk_only=False)
    
    from math import factorial
    system_comparisons = 0
    for result in ranking_results:
        result.reload_dynamic_fields()
        # TODO: this implicitly counts A=B comparisons for multi systems.
        # Basically, inflating the number of pairwise comparisons... Fix!
        combinations = factorial(result.systems)/(factorial(result.systems-2) * 2) if result.systems > 2 else 0
        system_comparisons = system_comparisons + combinations
    
    # Aggregate information about participating groups.
    groups = set()
    for user in wmt16_users:
        for group in _identify_groups_for_user(user):
            groups.add(group)
    
    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list('duration', flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)
    avg_user_time = total_time / float(3 * hits_completed or 1)
    
    global_stats.append(('Users', len(wmt16_users)))
    global_stats.append(('Groups', len(groups)))
    global_stats.append(('HITs completed', '{0:,}'.format(hits_completed)))
    global_stats.append(('HITs remaining', '{0:,}'.format(hits_remaining)))
    global_stats.append(('Ranking results', '{0:,}'.format(ranking_results.count())))
    global_stats.append(('System comparisons', '{0:,}'.format(system_comparisons)))
    global_stats.append(('Average duration (per HIT)', seconds_to_timedelta(avg_time)))
    global_stats.append(('Average duration (per task)', seconds_to_timedelta(avg_user_time)))
    global_stats.append(('Total duration', seconds_to_timedelta(total_time)))
    
    # Create new status data snapshot
    TimedKeyValueData.update_status_if_changed('users', str(len(wmt16_users)))
    TimedKeyValueData.update_status_if_changed('groups', str(len(groups)))
    TimedKeyValueData.update_status_if_changed('hits_completed', str(hits_completed))
    TimedKeyValueData.update_status_if_changed('hits_remaining', str(hits_remaining))
    TimedKeyValueData.update_status_if_changed('ranking_results', str(ranking_results.count()))
    TimedKeyValueData.update_status_if_changed('system_comparisons', str(system_comparisons))
    TimedKeyValueData.update_status_if_changed('duration_per_hit', str(seconds_to_timedelta(avg_time)))
    TimedKeyValueData.update_status_if_changed('duration_per_task', str(seconds_to_timedelta(avg_user_time)))
    TimedKeyValueData.update_status_if_changed('duration_total', str(seconds_to_timedelta(total_time)))
    
    return global_stats
Esempio n. 11
0
def overview(request):
    """
    Renders the evaluation tasks overview.
    """
    LOGGER.info('Rendering WMT16 HIT overview for user "{0}".'.format(
      request.user.username or "Anonymous"))
    
    # Re-initialise random number generator.
    seed(None)
    
    # Collect available language pairs for the current user.
    language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
    language_pairs = request.user.groups.filter(name__in=language_codes)
    
    # Collect available annotation projects for the current user.
    annotation_projects = request.user.project_set.all()
    
    hit_data = []
    total = [0, 0, 0]

    for language_pair in language_pairs:
        for annotation_project in annotation_projects:
            hit = _compute_next_task_for_user(request.user, annotation_project, language_pair)
            user_status = HIT.compute_status_for_user(request.user, annotation_project, language_pair)
            for i in range(3):
                total[i] = total[i] + user_status[i]
        
            if hit:
                # Convert status seconds back into datetime.time instances.
                for i in range(2):
                    user_status[i+1] = seconds_to_timedelta(int(user_status[i+1]))
            
                hit_data.append(
                  (hit.get_language_pair_display(), hit.get_absolute_url(),
                   hit.hit_id, user_status, annotation_project)
                )
    
    # Convert total seconds back into datetime.timedelta instances.
    total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))
    
    # Remove microseconds to get a nicer timedelta rendering in templates.
    total[1] = total[1] - timedelta(microseconds=total[1].microseconds)
    
    total[2] = seconds_to_timedelta(int(total[2]))
    
    groups = _identify_groups_for_user(request.user)
    group = None
    if len(groups) > 1:
        LOGGER.debug(u'User "{0}" assigned to multiple annotation groups: {1}'.format(
          request.user.username or u'Anonymous',
          u', '.join([x.name for x in groups]))
        )
        group = groups[0]
    
    if group is not None:
        group_name = group.name
        group_status = HIT.compute_status_for_group(group)
        for i in range(2):
            group_status[i+1] = seconds_to_timedelta(int(group_status[i+1]))
    
    else:
        group_status = None
        group_name = None
    
    LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
      request.user.username or "Anonymous",
      u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))

    # Compute admin URL for super users.
    admin_url = None
    if request.user.is_superuser:
        admin_url = reverse('admin:index')
    
    dictionary = {
      'active_page': "OVERVIEW",
      'hit_data': hit_data,
      'total': total,
      'group_name': group_name,
      'group_status': group_status,
      'admin_url': admin_url,
      'title': 'WMT16 Dashboard',
      'annotation_groups': [x.name for x in groups],
    }
    dictionary.update(BASE_CONTEXT)
    
    LOGGER.info(dictionary.values())
    
    return render(request, 'wmt16/overview.html', dictionary)
Esempio n. 12
0
def _compute_global_stats():
    """
    Computes some global statistics for the WMT16 evaluation campaign.
    """
    global_stats = []

    wmt16_group = Group.objects.filter(name='WMT16')
    wmt16_users = _get_active_users_for_group(wmt16_group)

    # Check how many HITs have been completed.  We now consider a HIT to be
    # completed once it has been annotated by one or more annotators.
    #
    # Before we required `hit.users.count() >= 3` for greater overlap.
    hits_completed = HIT.objects.filter(mturk_only=False,
                                        completed=True).count()

    # Check any remaining active HITs which are not yet marked complete.
    for hit in HIT.objects.filter(active=True,
                                  mturk_only=False,
                                  completed=False):
        if hit.users.count() >= 1:
            hits_completed = hits_completed + 1
            hit.completed = True
            hit.save()

    # Compute remaining HITs for all language pairs.
    hits_remaining = HIT.compute_remaining_hits()

    # Compute number of results contributed so far.
    ranking_results = RankingResult.objects.filter(item__hit__completed=True,
                                                   item__hit__mturk_only=False)

    from math import factorial
    system_comparisons = 0
    for result in ranking_results:
        result.reload_dynamic_fields()
        # TODO: this implicitly counts A=B comparisons for multi systems.
        # Basically, inflating the number of pairwise comparisons... Fix!
        combinations = factorial(result.systems) / (
            factorial(result.systems - 2) * 2) if result.systems > 2 else 0
        system_comparisons = system_comparisons + combinations

    # Aggregate information about participating groups.
    groups = set()
    for user in wmt16_users:
        for group in _identify_groups_for_user(user):
            groups.add(group)

    # Compute average/total duration over all results.
    durations = RankingResult.objects.all().values_list('duration', flat=True)
    total_time = sum([datetime_to_seconds(x) for x in durations])
    avg_time = total_time / float(hits_completed or 1)
    avg_user_time = total_time / float(3 * hits_completed or 1)

    global_stats.append(('Users', len(wmt16_users)))
    global_stats.append(('Groups', len(groups)))
    global_stats.append(('HITs completed', '{0:,}'.format(hits_completed)))
    global_stats.append(('HITs remaining', '{0:,}'.format(hits_remaining)))
    global_stats.append(
        ('Ranking results', '{0:,}'.format(ranking_results.count())))
    global_stats.append(
        ('System comparisons', '{0:,}'.format(system_comparisons)))
    global_stats.append(
        ('Average duration (per HIT)', seconds_to_timedelta(avg_time)))
    global_stats.append(
        ('Average duration (per task)', seconds_to_timedelta(avg_user_time)))
    global_stats.append(('Total duration', seconds_to_timedelta(total_time)))

    # Create new status data snapshot
    TimedKeyValueData.update_status_if_changed('users', str(len(wmt16_users)))
    TimedKeyValueData.update_status_if_changed('groups', str(len(groups)))
    TimedKeyValueData.update_status_if_changed('hits_completed',
                                               str(hits_completed))
    TimedKeyValueData.update_status_if_changed('hits_remaining',
                                               str(hits_remaining))
    TimedKeyValueData.update_status_if_changed('ranking_results',
                                               str(ranking_results.count()))
    TimedKeyValueData.update_status_if_changed('system_comparisons',
                                               str(system_comparisons))
    TimedKeyValueData.update_status_if_changed(
        'duration_per_hit', str(seconds_to_timedelta(avg_time)))
    TimedKeyValueData.update_status_if_changed(
        'duration_per_task', str(seconds_to_timedelta(avg_user_time)))
    TimedKeyValueData.update_status_if_changed(
        'duration_total', str(seconds_to_timedelta(total_time)))

    return global_stats
Esempio n. 13
0
def overview(request):
    """
    Renders the evaluation tasks overview.
    """
    LOGGER.info('Rendering WMT16 HIT overview for user "{0}".'.format(
        request.user.username or "Anonymous"))

    # Re-initialise random number generator.
    seed(None)

    # Collect available language pairs for the current user.
    language_codes = set([x[0] for x in LANGUAGE_PAIR_CHOICES])
    language_pairs = request.user.groups.filter(name__in=language_codes)

    # Collect available annotation projects for the current user.
    annotation_projects = request.user.project_set.all()

    hit_data = []
    total = [0, 0, 0]

    for language_pair in language_pairs:
        for annotation_project in annotation_projects:
            hit = _compute_next_task_for_user(request.user, annotation_project,
                                              language_pair)
            user_status = HIT.compute_status_for_user(request.user,
                                                      annotation_project,
                                                      language_pair)
            for i in range(3):
                total[i] = total[i] + user_status[i]

            if hit:
                # Convert status seconds back into datetime.time instances.
                for i in range(2):
                    user_status[i + 1] = seconds_to_timedelta(
                        int(user_status[i + 1]))

                hit_data.append(
                    (hit.get_language_pair_display(), hit.get_absolute_url(),
                     hit.hit_id, user_status, annotation_project))

    # Convert total seconds back into datetime.timedelta instances.
    total[1] = seconds_to_timedelta(int(total[2]) / float(int(total[0]) or 1))

    # Remove microseconds to get a nicer timedelta rendering in templates.
    total[1] = total[1] - timedelta(microseconds=total[1].microseconds)

    total[2] = seconds_to_timedelta(int(total[2]))

    groups = _identify_groups_for_user(request.user)
    group = None
    if len(groups) > 1:
        LOGGER.debug(
            u'User "{0}" assigned to multiple annotation groups: {1}'.format(
                request.user.username or u'Anonymous',
                u', '.join([x.name for x in groups])))
        group = groups[0]

    if group is not None:
        group_name = group.name
        group_status = HIT.compute_status_for_group(group)
        for i in range(2):
            group_status[i + 1] = seconds_to_timedelta(int(group_status[i +
                                                                        1]))

    else:
        group_status = None
        group_name = None

    LOGGER.debug(u'\n\nHIT data for user "{0}":\n\n{1}\n'.format(
        request.user.username or "Anonymous",
        u'\n'.join([u'{0}\t{1}\t{2}\t{3}'.format(*x) for x in hit_data])))

    # Compute admin URL for super users.
    admin_url = None
    if request.user.is_superuser:
        admin_url = reverse('admin:index')

    dictionary = {
        'active_page': "OVERVIEW",
        'hit_data': hit_data,
        'total': total,
        'group_name': group_name,
        'group_status': group_status,
        'admin_url': admin_url,
        'title': 'WMT16 Dashboard',
        'annotation_groups': [x.name for x in groups],
    }
    dictionary.update(BASE_CONTEXT)

    LOGGER.info(dictionary.values())

    return render(request, 'wmt16/overview.html', dictionary)