def _score_question(submission, submission_report,
question, problem_instance):
points = question.points
question_report = QuestionReport(
submission_report=submission_report,
question=question,
question_max_score=points,
score=IntegerScore(0),
)
award_points = False
ignore_question = True
if question.is_text_input:
text_answers = submission.quizsubmissiontextanswer_set.filter(question=question)
if text_answers.exists():
text_answer = text_answers.get().text_answer
correct_answers = question.quizanswer_set.filter(is_correct=True)
award_points = any(_match_text_input(question, text_answer, answer.answer, problem_instance)
for answer in correct_answers)
ignore_question = False
else:
submitted_answers = submission.quizsubmissionanswer_set.filter(answer__question=question)
if submitted_answers.exists():
award_points = all(_is_answer_correct(answer) for answer in submitted_answers)
ignore_question = False
if not ignore_question:
if award_points:
question_report.score = IntegerScore(points)
question_report.status = 'OK'
question_report.save()
return (points, ignore_question) if award_points else (0, ignore_question)
def _score_question(submission, submission_report, question):
points = question.points
question_report = QuestionReport(
submission_report=submission_report,
question=question,
question_max_score=points,
score=IntegerScore(0),
)
award_points = False
if question.is_text_input:
text_answer = submission.quizsubmissiontextanswer_set\
.get(question=question).text_answer
correct_answers = question.quizanswer_set.filter(is_correct=True)
award_points = any(
_match_text_input(question, text_answer, answer.answer)
for answer in correct_answers)
else:
submitted_answers = submission.quizsubmissionanswer_set\
.filter(answer__question=question)
award_points = all(
_is_answer_correct(answer) for answer in submitted_answers)
if award_points:
question_report.score = IntegerScore(points)
question_report.status = 'OK'
question_report.save()
return points
question_report.save()
return 0
def threshold_linear_test_scorer(test, result):
"""Full score if took less than half of limit and then decreasing to 1"""
limit = test.get('exec_time_limit', 0)
used = result.get('time_used', 0)
status = result['result_code']
percentage = result.get('result_percentage', 100)
max_score = int(ceil(percentage * test['max_score'] / 100.))
test_max_score = IntegerScore(test['max_score'])
if status != 'OK':
return IntegerScore(0), test_max_score, status
if not limit:
return IntegerScore(max_score), test_max_score, status
if used > limit:
score = 0
status = 'TLE'
elif max_score == 0:
score = 0
elif used <= limit / 2.:
score = max_score
else:
score = 1 + int((max_score - 1) * ((limit - used) / (limit / 2.)))
return IntegerScore(score), test_max_score, status
def _create_score_report(max_score, score, submission_report):
ScoreReport.objects.create(
submission_report=submission_report,
score=IntegerScore(score),
status='OK',
max_score=IntegerScore(max_score),
)
def test_pa_score(self):
score = [PAScore(IntegerScore(x)) for x in range(0, 11)]
self.assertLess(score[0], score[5])
self.assertLess(score[5], score[10])
self.assertLess(score[5] + score[5], score[10])
self.assertLess(score[5] + score[5], score[2] + score[2] + score[6])
self.assertLess(score[10], score[2] + score[4] + score[5])
self.assertLess(score[2] + score[2] + score[6],
score[1] + score[3] + score[6])
dist1 = ScoreDistribution([0] * 8 + [2, 4])
dist2 = ScoreDistribution([0] * 8 + [1, 6])
score1 = PAScore(IntegerScore(8), dist1)
score2 = PAScore(IntegerScore(8), dist2)
self.assertLess(score2, score1)
score3 = score[10] + score[10] + score[10] + score[4] + score[2] + \
score1 + score2
self.assertEqual(score3, (3 * 10 + 4 + 2 + 2 * 8))
self.assertEqual(
repr(score3),
'PAScore(IntegerScore(52), ScoreDistribution(10: 3, 9: 0, 8: '
'0, 7: 0, 6: 0, 5: 0, 4: 1, 3: 0, 2: 4, 1: 10))')
self.assertEqual(
score3._to_repr(), '0000000000000000052;00003:00000:'
'00000:00000:00000:00000:00001:00000:00004:00010')
self.assertEqual(score3, PAScore._from_repr(score3._to_repr()))
def test_judging(self):
self.assertTrue(self.client.login(username='******'))
contest = Contest.objects.get()
url = reverse('submit', kwargs={'contest_id': contest.id})
# Show submission form
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
self.assertIn('contests/submit.html',
[getattr(t, 'name', None) for t in response.templates])
form = response.context['form']
self.assertEqual(len(form.fields['problem_instance_id'].choices), 1)
pi_id = form.fields['problem_instance_id'].choices[0][0]
# Submit
filename = get_test_filename('sum-various-results.cpp')
response = self.client.post(url, {
'problem_instance_id': pi_id,
'file': open(filename, 'rb')
})
self.assertEqual(response.status_code, 302)
self.assertEqual(Submission.objects.count(), 1)
self.assertEqual(TestReport.objects.count(), 6)
self.assertEqual(TestReport.objects.filter(status='OK').count(), 4)
self.assertEqual(TestReport.objects.filter(status='WA').count(), 1)
self.assertEqual(TestReport.objects.filter(status='RE').count(), 1)
submission = Submission.objects.get()
self.assertEqual(submission.status, 'INI_OK')
self.assertEqual(submission.score, IntegerScore(34))
urc = UserResultForContest.objects.get()
self.assertEqual(urc.score, IntegerScore(34))
class TestPAScorer(TestCase):
t_results_ok = (
({
'exec_time_limit': 100,
'max_score': 100
}, {
'result_code': 'OK',
'time_used': 0
}),
({
'exec_time_limit': 100,
'max_score': 10
}, {
'result_code': 'OK',
'time_used': 99
}),
({
'exec_time_limit': 1000,
'max_score': 0
}, {
'result_code': 'OK',
'time_used': 123
}),
)
t_expected_ok = [
(IntegerScore(1), IntegerScore(1), 'OK'),
(IntegerScore(1), IntegerScore(1), 'OK'),
(IntegerScore(0), IntegerScore(0), 'OK'),
]
t_results_wrong = [
({
'exec_time_limit': 100,
'max_score': 100
}, {
'result_code': 'WA',
'time_used': 75
}),
({
'exec_time_limit': 100,
'max_score': 0
}, {
'result_code': 'RV',
'time_used': 75
}),
]
t_expected_wrong = [
(IntegerScore(0), IntegerScore(1), 'WA'),
(IntegerScore(0), IntegerScore(0), 'RV'),
]
def test_pa_test_scorer(self):
results = map(utils.pa_test_scorer, *zip(*self.t_results_ok))
self.assertEquals(self.t_expected_ok, results)
results = map(utils.pa_test_scorer, *zip(*self.t_results_wrong))
self.assertEquals(self.t_expected_wrong, results)
def test_integer_score(self):
s1 = IntegerScore(1)
s2 = IntegerScore(2)
self.assertLess(s1, s2)
self.assertGreater(s2, s1)
self.assertEqual(s1, IntegerScore(1))
self.assertEqual((s1 + s2).value, 3)
self.assertEqual(unicode(s1), '1')
self.assertEqual(IntegerScore._from_repr(s1._to_repr()), s1)
def smart_score_copier(apps, schema_editor):
Contest = apps.get_model('contests', 'Contest')
ProblemInstance = apps.get_model('contests', 'ProblemInstance')
Submission = apps.get_model('contests', 'Submission')
SubmissionReport = apps.get_model('contests', 'SubmissionReport')
TestReport = apps.get_model('programs', 'TestReport')
db_alias = schema_editor.connection.alias
# Firstly, all max_scores will be set as equal to test_max_scores
# provided that they are not None – this is the behaviour used
# all contests except for the Algorithmic Engagements
# and the ACM type contests.
# This operates on raw, serialized data, which is a bit dirty but works.
TestReport.objects.using(db_alias).filter(test_max_score__isnull=False) \
.update(
max_score=Concat(
V('int:'),
Substr(
Concat(V('0000000000000000000'), 'test_max_score'),
Length(Concat(V('0000000000000000000'), 'test_max_score')) - 18,
19
)
)
)
# Secondly, all max_scores related to the Algorithmic Engagements
# will be set to either 1 or 0, the same way they are defined
# in pa_test_scorer from oioioi/pa/utils.py
pa_test_reports = TestReport.objects.using(db_alias).filter(
submission_report__submission__problem_instance__contest__controller_name
='oioioi.pa.controllers.PAContestController',
test_max_score__isnull=False)
pa_test_reports.update(max_score=IntegerScore(1))
pa_test_reports.filter(test_max_score=0).update(max_score=IntegerScore(0))
# In the end, all max_scores related to the ACM type contests will be left
# as none, which agrees with their behaviour defined in the ACM contest
# controller.
acm_test_reports = TestReport.objects.using(db_alias).filter(
submission_report__submission__problem_instance__contest__controller_name
='oioioi.acm.controllers.ACMContestController')
acm_test_reports.update(max_score=None)
def test_score_field(self):
contest = Contest.objects.get()
user = User.objects.get(username='******')
instance = UserResultForContest(user=user,
contest=contest,
score=IntegerScore(42))
instance.save()
del instance
instance = UserResultForContest.objects.get()
self.assertTrue(isinstance(instance.score, IntegerScore))
self.assertEqual(instance.score.value, 42)
instance.score = "int:12"
self.assertEqual(instance.score.value, 12)
with self.assertRaises(ValidationError):
instance.score = "1"
with self.assertRaises(ValidationError):
instance.score = "foo:1"
instance.score = None
instance.save()
del instance
instance = UserResultForContest.objects.get()
self.assertIsNone(instance.score)
def _check_interactive_package(self, problem):
self.assertEqual(problem.short_name, 'arc')
config = ExtraConfig.objects.get(problem=problem)
assert len(config.parsed_config['extra_compilation_args']) == 2
assert len(config.parsed_config['extra_compilation_files']) == 3
self.assertEqual(problem.name, u'arc')
tests = Test.objects.filter(problem_instance=problem.main_problem_instance)
t0 = tests.get(name='0')
self.assertEqual(t0.input_file.read(), b'3\n12\n5\n8\n3\n15\n8\n0\n')
self.assertEqual(t0.output_file.read(), b'12\n15\n8\n')
self.assertEqual(t0.kind, 'EXAMPLE')
self.assertEqual(t0.group, '0')
self.assertEqual(t0.max_score, 0)
self.assertEqual(t0.time_limit, DEFAULT_TIME_LIMIT)
self.assertEqual(t0.memory_limit, 66000)
t1a = tests.get(name='1a')
self.assertEqual(
t1a.input_file.read(), b'0\n-435634223 1 30 23 130 0 -324556462\n'
)
self.assertEqual(
t1a.output_file.read(),
b"""126\n126\n82\n85\n80\n64\n84\n5\n128\n66\n4\n79\n64\n96
22\n107\n84\n112\n92\n63\n125\n82\n1\n""",
)
self.assertEqual(t1a.kind, 'NORMAL')
self.assertEqual(t1a.group, '1')
self.assertEqual(t1a.max_score, 50)
t2a = tests.get(name='2a')
self.assertEqual(
t2a.input_file.read(), b'0\n-435634223 1 14045 547 60000 0 -324556462\n'
)
self.assertEqual(t2a.kind, 'NORMAL')
self.assertEqual(t2a.group, '2')
self.assertEqual(t2a.max_score, 50)
checker = OutputChecker.objects.get(problem=problem)
self.assertIsNotNone(checker.exe_file)
extra_files = ExtraFile.objects.filter(problem=problem)
self.assertEqual(extra_files.count(), 3)
model_solutions = ModelSolution.objects.filter(problem=problem).order_by(
'order_key'
)
solc = model_solutions.get(name='arc.c')
self.assertEqual(solc.kind, 'NORMAL')
solcpp = model_solutions.get(name='arc1.cpp')
self.assertEqual(solcpp.kind, 'NORMAL')
solpas = model_solutions.get(name='arc2.pas')
self.assertEqual(solpas.kind, 'NORMAL')
self.assertEqual(list(model_solutions), [solc, solcpp, solpas])
submissions = Submission.objects.all()
for s in submissions:
self.assertEqual(s.status, 'INI_OK')
self.assertEqual(s.score, IntegerScore(100))
def _score_question(submission, submission_report, question):
points = question.points
question_report = QuestionReport(
submission_report=submission_report,
question=question,
question_max_score=points,
score=IntegerScore(0),
)
submitted_answers = submission.quizsubmissionanswer_set\
.filter(answer__question=question)
are_all_answers_correct = all(
_is_answer_correct(answer) for answer in submitted_answers)
if are_all_answers_correct:
question_report.score = IntegerScore(points)
question_report.status = 'OK'
question_report.save()
return points
question_report.save()
return 0
def create_score(username, score):
user = User.objects.create_user(username, username + '@example.pl',
username)
result = UserResultForProblem()
result.user = user
result.problem_instance = pis
result.status = 'OK'
result.score = IntegerScore(score)
result.save()
return user
def __init__(self, points=None, distribution=None):
if points:
assert isinstance(points, IntegerScore)
self.points = points
else:
self.points = IntegerScore(0)
if distribution:
assert isinstance(distribution, ScoreDistribution)
self.distribution = distribution
else:
self.distribution = ScoreDistribution()
self.distribution.update(self.points.value)
def grade_tests(env, **kwargs):
"""Grades tests using a scoring function.
The ``env['test_scorer']``, which is used by this ``Handler``,
should be a path to a function which gets test definition (e.g. a
``env['tests'][test_name]`` dict) and test run result (e.g. a
``env['test_results'][test_name]`` dict) and returns a score
(instance of some subclass of
:class:`~oioioi.contests.scores.ScoreValue`) and a status.
Used ``environ`` keys:
* ``tests``
* ``test_results``
* ``test_scorer``
Produced ``environ`` keys:
* `score`, `max_score` and `status` keys in ``env['test_result']``
"""
fun = import_string(env.get('test_scorer') or settings.DEFAULT_TEST_SCORER)
tests = env['tests']
for test_name, test_result in six.iteritems(env['test_results']):
if tests[test_name]['to_judge']:
score, max_score, status = fun(tests[test_name], test_result)
assert isinstance(score, (type(None), ScoreValue))
assert isinstance(max_score, (type(None), ScoreValue))
test_result['score'] = score and score.serialize()
test_result['max_score'] = max_score and max_score.serialize()
test_result['status'] = status
else:
report = TestReport.objects.get(
submission_report__submission__id=env['submission_id'],
submission_report__status='ACTIVE',
test_name=test_name)
score = report.score
max_score = IntegerScore(report.test_max_score)
status = report.status
time_used = report.time_used
test_result['score'] = score and score.serialize()
test_result['max_score'] = max_score and max_score.serialize()
test_result['status'] = status
test_result['time_used'] = time_used
env['test_results'][test_name] = test_result
return env
def test_score_field(self):
instance = ScoreFieldTestModel(score=IntegerScore(42))
instance.save()
del instance
instance = ScoreFieldTestModel.objects.get()
self.assertTrue(isinstance(instance.score, IntegerScore))
self.assertEqual(instance.score.value, 42)
instance.score = "int:12"
self.assertEqual(instance.score.value, 12)
with self.assertRaises(ValidationError):
instance.score = "1"
with self.assertRaises(ValidationError):
instance.score = "foo:1"
instance.score = None
instance.save()
del instance
instance = ScoreFieldTestModel.objects.get()
self.assertIsNone(instance.score)
def discrete_test_scorer(test, result):
status = result['result_code']
score = (status == 'OK') and test['max_score'] or 0
return IntegerScore(score), status
def pa_score_factory(int_score):
return PAScore(IntegerScore(int_score))
class TestScorers(TestCase):
t_results_ok = (
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 0}),
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 50}),
({'exec_time_limit': 1000, 'max_score': 100},
{'result_code': 'OK', 'time_used': 501}),
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 75}),
({'exec_time_limit': 1000, 'max_score': 100},
{'result_code': 'OK', 'time_used': 999}),
({'max_score': 100},
{'result_code': 'OK', 'time_used': 0}),
({'max_score': 100},
{'result_code': 'OK', 'time_used': 99999}),
)
t_results_ok_perc = (
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 0, 'result_percentage': 99}),
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 75, 'result_percentage': 50}),
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 75, 'result_percentage': 0}),
)
t_results_wrong = [
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'WA', 'time_used': 75}),
({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'RV', 'time_used': 75}),
]
t_expected_wrong = [
(IntegerScore(0), 'WA'),
(IntegerScore(0), 'RV'),
]
def test_discrete_test_scorer(self):
exp_scores = [100] * len(self.t_results_ok)
exp_statuses = ['OK'] * len(self.t_results_ok)
expected = zip(exp_scores, exp_statuses)
results = map(utils.discrete_test_scorer, *zip(*self.t_results_ok))
self.assertEquals(expected, results)
results = map(utils.discrete_test_scorer, *zip(*self.t_results_wrong))
self.assertEquals(self.t_expected_wrong, results)
def test_threshold_linear_test_scorer(self):
exp_scores = [100, 100, 99, 50, 0, 100, 100]
exp_statuses = ['OK'] * len(self.t_results_ok)
expected = zip(exp_scores, exp_statuses)
results = map(utils.threshold_linear_test_scorer,
*zip(*self.t_results_ok))
self.assertEquals(expected, results)
exp_scores = [99, 25, 0]
exp_statuses = ['OK'] * len(self.t_results_ok_perc)
expected = zip(exp_scores, exp_statuses)
results = map(utils.threshold_linear_test_scorer,
*zip(*self.t_results_ok_perc))
self.assertEquals(expected, results)
malformed = ({'exec_time_limit': 100, 'max_score': 100},
{'result_code': 'OK', 'time_used': 101})
self.assertEqual(utils.threshold_linear_test_scorer(*malformed),
(0, 'TLE'))
results = map(utils.threshold_linear_test_scorer,
*zip(*self.t_results_wrong))
self.assertEquals(self.t_expected_wrong, results)
@memoized_property
def g_results_ok(self):
# Tested elsewhere
results = map(utils.threshold_linear_test_scorer,
*zip(*self.t_results_ok[:4]))
dicts = [dict(score=sc.serialize(), status=st) for sc, st in results]
return dict(zip(xrange(len(dicts)), dicts))
@memoized_property
def g_results_wrong(self):
results = map(utils.threshold_linear_test_scorer,
*zip(*self.t_results_wrong))
dicts = self.g_results_ok.values()
dicts += [dict(score=sc.serialize(), status=st) for sc, st in results]
return dict(zip(xrange(len(dicts)), dicts))
def test_min_group_scorer(self):
self.assertEqual((50, 'OK'), utils.min_group_scorer(self.g_results_ok))
self.assertEqual((0, 'WA'),
utils.min_group_scorer(self.g_results_wrong))
def test_sum_group_scorer(self):
self.assertEqual((349, 'OK'), utils.sum_group_scorer(self.g_results_ok))
self.assertEqual((349, 'WA'),
utils.sum_group_scorer(self.g_results_wrong))
def test_sum_score_aggregator(self):
self.assertEqual((349, 'OK'),
utils.sum_score_aggregator(self.g_results_ok))
self.assertEqual((349, 'WA'),
utils.sum_score_aggregator(self.g_results_wrong))
def pa_test_scorer(test, result):
status = result['result_code']
max_score = min(1, test['max_score'])
score = max_score if status == 'OK' else 0
return IntegerScore(score), IntegerScore(max_score), status
def discrete_test_scorer(test, result):
status = result['result_code']
max_score = test['max_score']
score = max_score if status == 'OK' else 0
return IntegerScore(score), IntegerScore(max_score), status
