def on_interact(self):
"""
Responsible for unlocking each test.
"""
if not self.args.unlock:
return
formatting.print_title('Unlocking tests for {}'.format(
self.assignment['name']))
print('At each "{}",'.format(UnlockConsole.PROMPT)
+ ' type in what you would expect the output to be.')
print('Type {} to quit'.format(UnlockConsole.EXIT_INPUTS[0]))
for test in self._filter_tests():
if test.num_cases == 0:
print('No tests to unlock for {}.'.format(test.name))
else:
formatting.underline('Unlocking tests for {}'.format(test.name))
print()
# TODO(albert): the unlock function returns the number
# of unlocked test cases. This can be a useful metric
# for analytics in the future.
cases_unlocked, end_session = unlock(
test, self.logger, self.assignment['name'], self.analytics)
if end_session:
break
print()
return self.analytics
def display_breakdown(scores):
"""Prints the point breakdown given a dictionary of scores.
RETURNS:
int; the total score for the assignment
"""
partner_totals = {}
formatting.underline('Point breakdown')
for (name, partner), (score, total) in scores.items():
print(name + ': ' + '{}/{}'.format(score, total))
partner_totals[partner] = partner_totals.get(partner, 0) + score
print()
if len(partner_totals) == 1:
# If only one partner.
print('Total score:')
print(partner_totals[core.Test.DEFAULT_PARTNER])
else:
for partner, total in sorted(partner_totals.items()):
if partner == core.Test.DEFAULT_PARTNER:
continue
print('Partner {} score:'.format(partner))
# Add partner-specific score with partner-agnostic score.
print(total + partner_totals.get(core.Test.DEFAULT_PARTNER, 0))
return partner_totals
def _handle_test(self, test):
"""Grades a single Test."""
formatting.underline('Scoring tests for ' + test.name)
print()
points, passed, total = score(test, self.logger, self.args.interactive,
self.args.verbose, self.args.timeout)
self.scores[(test.name, test['partner'])] = (points, test['points'])
return passed, total
def _handle_test(self, test):
"""Grades a single Test."""
formatting.underline('Running tests for ' + test.name)
print()
if test['note']:
print(test['note'])
total_passed = grade(test, self.logger, self.args.interactive,
self.args.verbose, self.args.timeout)
if test.num_locked > 0:
print('-- There are still {} locked test cases.'.format(
test.num_locked) + ' Use the -u flag to unlock them. --')
return total_passed, test.num_cases
def lock(test, hash_fn):
formatting.underline('Locking Test ' + test.name, line='-')
if test['hidden_params']:
test['hidden_params'] = {}
print('* Removed hidden params')
num_cases = 0
for suite in test['suites']:
for case in list(suite):
num_cases += 1 # 1-indexed
if case['hidden']:
suite.remove(case)
print('* Case {}: removed hidden test'.format(num_cases))
elif not case['never_lock'] and not case['locked']:
case.on_lock(hash_fn)
print('* Case {}: locked test'.format(num_cases))
elif case['never_lock']:
print('* Case {}: never lock'.format(num_cases))
elif case['locked']:
print('* Case {}: already locked'.format(num_cases))
def unlock(test, logger, hash_key, analytics=None):
"""Unlocks TestCases for a given Test.
PARAMETERS:
test -- Test; the test to unlock.
logger -- OutputLogger.
hash_key -- string; hash_key to be used to unlock.
analytics -- dict; dictionary used to store analytics for this protocol
DESCRIPTION:
This function incrementally unlocks all TestCases in a specified
Test. Students must answer in the order that TestCases are
written. Once a TestCase is unlocked, it will remain unlocked.
RETURN:
int, bool; the number of cases that are newly unlocked for this Test
after going through an unlocking session and whether the student wanted
to exit the unlocker or not.
"""
if analytics is None:
analytics = {}
console = UnlockConsole(logger, hash_key, analytics)
cases = 0
cases_unlocked = 0
analytics[test.name] = []
analytics['current'] = test.name
for suite in test['suites']:
for case in suite:
cases += 1
if not isinstance(case, UnlockTestCase) \
or not case['locked']:
continue
formatting.underline('Case {}'.format(cases), line='-')
if console.run(case): # Abort unlocking.
return cases_unlocked, True
cases_unlocked += 1
print("You are done unlocking tests for this question!")
del analytics['current']
return cases_unlocked, False
def run_suite(suite, logger, cases_tested, verbose, interactive, timeout, stop_fast=True):
"""Runs tests for a single suite.
PARAMETERS:
suite -- list; each element is a TestCase
logger -- OutputLogger.
cases_tested -- Counter; an object that keeps track of the
number of cases that have been tested so far.
verbose -- bool; True if verbose mode is toggled on
interactive -- bool; True if interactive mode is toggled on
stop_fast -- bool; True if grading should stop at the first
test case where should_grade returns False. If
False, grading will continue.
RETURNS:
(passed, errored), where
passed -- int; number of TestCases that passed
errored -- bool; True if a TestCase resulted in error.
"""
passed = 0
for case in suite:
if not isinstance(case, GradedTestCase):
# TODO(albert): should non-GradedTestCases be counted as
# passing?
continue
elif stop_fast and not case.should_grade():
logger.on()
return passed, True # students must unlock first
cases_tested.increment()
formatting.underline('Case {}'.format(cases_tested), line='-')
error = case.on_grade(logger, verbose, interactive, timeout)
if error:
return passed, True
passed += 1
logger.on()
return passed, False