def test_challenge_card_status( client, phase1_submission_limit, phase1_submissions_open, phase1_submissions_close, phase2_submission_limit, phase2_submissions_open, phase2_submissions_close, expected_status, phase_in_status, ): ch = ChallengeFactory(hidden=False) phase1 = ch.phase_set.first() phase2 = PhaseFactory(challenge=ch) u = UserFactory() phase1.submission_limit = phase1_submission_limit phase1.submissions_open_at = phase1_submissions_open phase1.submissions_close_at = phase1_submissions_close phase2.submission_limit = phase2_submission_limit phase2.submissions_open_at = phase2_submissions_open phase2.submissions_close_at = phase2_submissions_close phase1.save() phase2.save() response = get_view_for_user( client=client, viewname="challenges:list", user=u ) if phase_in_status: title = ch.phase_set.order_by("created").all()[phase_in_status].title assert f"{expected_status} for {title}" in response.rendered_content else: assert expected_status in response.rendered_content
def test_null_results(): phase = PhaseFactory() results = [{"a": 0.6}, {"a": None}] queryset = [ EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS) for _ in range(len(results)) ] for e, r in zip(queryset, results): e.outputs.add( ComponentInterfaceValue.objects.create( interface=ComponentInterface.objects.get( slug="metrics-json-file"), value=r, )) phase.score_jsonpath = "a" phase.result_display_choice = Phase.ALL phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [1, 0] assert_ranks(queryset, expected_ranks)
def test_open_for_submission( submission_limit, submissions_open, submissions_close, open_for_submissions, expected_status, ): phase = PhaseFactory() phase.submission_limit = submission_limit phase.submissions_open_at = submissions_open phase.submissions_close_at = submissions_close phase.save() assert phase.open_for_submissions == open_for_submissions assert expected_status in phase.submission_status_string
def test_public_private_default(): p = PhaseFactory() r1 = EvaluationFactory(submission__phase=p) assert r1.published is True p.auto_publish_new_results = False p.save() r2 = EvaluationFactory(submission__phase=p) assert r2.published is False # The public/private status should only update on first save r1.save() assert r1.published is True
def test_null_results(): phase = PhaseFactory() results = [{"a": 0.6}, {"a": None}] queryset = [ EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS) for _ in range(len(results)) ] for e, r in zip(queryset, results): e.create_result(result=r) phase.score_jsonpath = "a" phase.result_display_choice = Phase.ALL phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [1, 0] assert_ranks(queryset, expected_ranks)
def test_calculate_ranks(django_assert_max_num_queries): phase = PhaseFactory() results = [ # Warning: Do not change this values without updating the # expected_ranks below. { "a": 0.0, "b": 0.0 }, { "a": 0.5, "b": 0.2 }, { "a": 1.0, "b": 0.3 }, { "a": 0.7, "b": 0.4 }, { "a": 0.5, "b": 0.5 }, # Following two are invalid as they are incomplete { "a": 1.0 }, { "b": 0.3 }, # Add a valid, but unpublished result { "a": 0.1, "b": 0.1 }, ] queryset = [ EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS) for _ in range(len(results)) ] for e, r in zip(queryset, results): e.create_result(result=r) # Unpublish the result queryset[-1].published = False queryset[-1].save() expected = { Phase.DESCENDING: { Phase.ABSOLUTE: { Phase.DESCENDING: { "ranks": [5, 3, 1, 2, 3, 0, 0, 0], "rank_scores": [5, 3, 1, 2, 3, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [5, 3, 1, 2, 3, 0, 0, 0], "rank_scores": [5, 3, 1, 2, 3, 0, 0, 0], }, }, Phase.MEDIAN: { Phase.DESCENDING: { "ranks": [5, 4, 1, 1, 1, 0, 0, 0], "rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [3, 2, 1, 3, 5, 0, 0, 0], "rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0], }, }, Phase.MEAN: { Phase.DESCENDING: { "ranks": [5, 4, 1, 1, 1, 0, 0, 0], "rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [3, 2, 1, 3, 5, 0, 0, 0], "rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0], }, }, }, Phase.ASCENDING: { Phase.ABSOLUTE: { Phase.DESCENDING: { "ranks": [1, 2, 5, 4, 2, 0, 0, 0], "rank_scores": [1, 2, 5, 4, 2, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [1, 2, 5, 4, 2, 0, 0, 0], "rank_scores": [1, 2, 5, 4, 2, 0, 0, 0], }, }, Phase.MEDIAN: { Phase.DESCENDING: { "ranks": [2, 2, 5, 2, 1, 0, 0, 0], "rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [1, 2, 4, 4, 3, 0, 0, 0], "rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0], }, }, Phase.MEAN: { Phase.DESCENDING: { "ranks": [2, 2, 5, 2, 1, 0, 0, 0], "rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0], }, Phase.ASCENDING: { "ranks": [1, 2, 4, 4, 3, 0, 0, 0], "rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0], }, }, }, } for score_method in (Phase.ABSOLUTE, Phase.MEDIAN, Phase.MEAN): for a_order in (Phase.DESCENDING, Phase.ASCENDING): for b_order in (Phase.DESCENDING, Phase.ASCENDING): phase.score_jsonpath = "a" phase.scoring_method_choice = score_method phase.score_default_sort = a_order phase.extra_results_columns = [{ "path": "b", "title": "b", "order": b_order }] phase.save() with django_assert_max_num_queries(7): calculate_ranks(phase_pk=phase.pk) assert_ranks( queryset, expected[a_order][score_method][b_order]["ranks"], expected[a_order][score_method][b_order]["rank_scores"], )
def test_results_display(): phase = PhaseFactory() user1 = UserFactory() user2 = UserFactory() metrics = "metrics" creator = "creator" results = [ { metrics: { "b": 0.3 }, creator: user1 }, # Invalid result { metrics: { "a": 0.6 }, creator: user1 }, { metrics: { "a": 0.4 }, creator: user1 }, { metrics: { "a": 0.2 }, creator: user1 }, { metrics: { "a": 0.1 }, creator: user2 }, { metrics: { "a": 0.5 }, creator: user2 }, { metrics: { "a": 0.3 }, creator: user2 }, ] queryset = [ EvaluationFactory( submission__phase=phase, submission__creator=r[creator], status=Evaluation.SUCCESS, ) for r in results ] for e, r in zip(queryset, results): e.create_result(result=r[metrics]) phase.score_jsonpath = "a" phase.result_display_choice = Phase.ALL phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [0, 1, 3, 5, 6, 2, 4] assert_ranks(queryset, expected_ranks) phase.result_display_choice = Phase.MOST_RECENT phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [0, 0, 0, 2, 0, 0, 1] assert_ranks(queryset, expected_ranks) phase.result_display_choice = Phase.BEST phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [0, 1, 0, 0, 0, 2, 0] assert_ranks(queryset, expected_ranks) # now test reverse order phase.score_default_sort = phase.ASCENDING phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [0, 0, 0, 2, 1, 0, 0] assert_ranks(queryset, expected_ranks) phase.result_display_choice = Phase.MOST_RECENT phase.save() calculate_ranks(phase_pk=phase.pk) expected_ranks = [0, 0, 0, 1, 0, 0, 2] assert_ranks(queryset, expected_ranks)