def on_interact(self): """ Responsible for unlocking each test. """ if not self.args.unlock: return formatting.print_title('Unlocking tests for {}'.format( self.assignment['name'])) print('At each "{}",'.format(UnlockConsole.PROMPT) + ' type in what you would expect the output to be.') print('Type {} to quit'.format(UnlockConsole.EXIT_INPUTS[0])) for test in self._filter_tests(): if test.num_cases == 0: print('No tests to unlock for {}.'.format(test.name)) else: formatting.underline('Unlocking tests for {}'.format(test.name)) print() # TODO(albert): the unlock function returns the number # of unlocked test cases. This can be a useful metric # for analytics in the future. cases_unlocked, end_session = unlock( test, self.logger, self.assignment['name'], self.analytics) if end_session: break print() return self.analytics
def display_breakdown(scores): """Prints the point breakdown given a dictionary of scores. RETURNS: int; the total score for the assignment """ partner_totals = {} formatting.underline('Point breakdown') for (name, partner), (score, total) in scores.items(): print(name + ': ' + '{}/{}'.format(score, total)) partner_totals[partner] = partner_totals.get(partner, 0) + score print() if len(partner_totals) == 1: # If only one partner. print('Total score:') print(partner_totals[core.Test.DEFAULT_PARTNER]) else: for partner, total in sorted(partner_totals.items()): if partner == core.Test.DEFAULT_PARTNER: continue print('Partner {} score:'.format(partner)) # Add partner-specific score with partner-agnostic score. print(total + partner_totals.get(core.Test.DEFAULT_PARTNER, 0)) return partner_totals
def _handle_test(self, test): """Grades a single Test.""" formatting.underline('Scoring tests for ' + test.name) print() points, passed, total = score(test, self.logger, self.args.interactive, self.args.verbose, self.args.timeout) self.scores[(test.name, test['partner'])] = (points, test['points']) return passed, total
def _handle_test(self, test): """Grades a single Test.""" formatting.underline('Running tests for ' + test.name) print() if test['note']: print(test['note']) total_passed = grade(test, self.logger, self.args.interactive, self.args.verbose, self.args.timeout) if test.num_locked > 0: print('-- There are still {} locked test cases.'.format( test.num_locked) + ' Use the -u flag to unlock them. --') return total_passed, test.num_cases
def lock(test, hash_fn): formatting.underline('Locking Test ' + test.name, line='-') if test['hidden_params']: test['hidden_params'] = {} print('* Removed hidden params') num_cases = 0 for suite in test['suites']: for case in list(suite): num_cases += 1 # 1-indexed if case['hidden']: suite.remove(case) print('* Case {}: removed hidden test'.format(num_cases)) elif not case['never_lock'] and not case['locked']: case.on_lock(hash_fn) print('* Case {}: locked test'.format(num_cases)) elif case['never_lock']: print('* Case {}: never lock'.format(num_cases)) elif case['locked']: print('* Case {}: already locked'.format(num_cases))
def unlock(test, logger, hash_key, analytics=None): """Unlocks TestCases for a given Test. PARAMETERS: test -- Test; the test to unlock. logger -- OutputLogger. hash_key -- string; hash_key to be used to unlock. analytics -- dict; dictionary used to store analytics for this protocol DESCRIPTION: This function incrementally unlocks all TestCases in a specified Test. Students must answer in the order that TestCases are written. Once a TestCase is unlocked, it will remain unlocked. RETURN: int, bool; the number of cases that are newly unlocked for this Test after going through an unlocking session and whether the student wanted to exit the unlocker or not. """ if analytics is None: analytics = {} console = UnlockConsole(logger, hash_key, analytics) cases = 0 cases_unlocked = 0 analytics[test.name] = [] analytics['current'] = test.name for suite in test['suites']: for case in suite: cases += 1 if not isinstance(case, UnlockTestCase) \ or not case['locked']: continue formatting.underline('Case {}'.format(cases), line='-') if console.run(case): # Abort unlocking. return cases_unlocked, True cases_unlocked += 1 print("You are done unlocking tests for this question!") del analytics['current'] return cases_unlocked, False
def run_suite(suite, logger, cases_tested, verbose, interactive, timeout, stop_fast=True): """Runs tests for a single suite. PARAMETERS: suite -- list; each element is a TestCase logger -- OutputLogger. cases_tested -- Counter; an object that keeps track of the number of cases that have been tested so far. verbose -- bool; True if verbose mode is toggled on interactive -- bool; True if interactive mode is toggled on stop_fast -- bool; True if grading should stop at the first test case where should_grade returns False. If False, grading will continue. RETURNS: (passed, errored), where passed -- int; number of TestCases that passed errored -- bool; True if a TestCase resulted in error. """ passed = 0 for case in suite: if not isinstance(case, GradedTestCase): # TODO(albert): should non-GradedTestCases be counted as # passing? continue elif stop_fast and not case.should_grade(): logger.on() return passed, True # students must unlock first cases_tested.increment() formatting.underline('Case {}'.format(cases_tested), line='-') error = case.on_grade(logger, verbose, interactive, timeout) if error: return passed, True passed += 1 logger.on() return passed, False