def display_breakdown(scores):
"""Prints the point breakdown given a dictionary of scores.
RETURNS:
dict; maps partner (str) -> finalized score (float)
"""
partner_totals = {}
format.print_line('-')
print('Point breakdown')
for (name, partner), (score, total) in scores.items():
print(' {}: {}/{}'.format(name, score, total))
partner_totals[partner] = partner_totals.get(partner, 0) + score
print()
shared_points = partner_totals.get(None, 0)
if None in partner_totals:
del partner_totals[None]
finalized_scores = {}
print('Score:')
if len(partner_totals) == 0:
print(' {}: {}'.format(NO_PARTNER_NAME, shared_points))
finalized_scores[NO_PARTNER_NAME] = shared_points
else:
for partner, score in sorted(partner_totals.items()):
print(' Partner {}: {}'.format(partner, score + shared_points))
finalized_scores[partner] = score + shared_points
return finalized_scores
def run(self, messages):
"""Score tests and print results
Tests are taken from self.assignment.specified_tests. Each test belongs
to a partner. If test.partner is omitted (i.e. core.NoValue), the score
for that test is added to every partner's score.
If there are no tests, the mapping will only contain one entry, mapping
"Total" -> 0 (total score).
If there are no partners specified by the tests, the mapping will only
contain one entry, mapping "Total" (partner) -> total score (float).
This assumes there is always at least one partner.
"""
if self.args.export or not self.args.score:
return
format.print_line('~')
print('Scoring tests')
print()
raw_scores = OrderedDict()
for test in self.assignment.specified_tests:
assert isinstance(
test,
sources_models.Test), 'ScoringProtocol received invalid test'
log.info('Scoring test {}'.format(test.name))
partner = test.partner if test.partner != core.NoValue else None
raw_scores[test.name, partner] = (test.score(), test.points)
messages['scoring'] = display_breakdown(raw_scores)
print()
def grade(questions, messages, env=None, verbose=True):
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
# The environment in which to run the tests.
for test in questions:
log.info('Running tests for {}'.format(test.name))
results = test.run(env)
# if correct once, set persistent flag
if results['failed'] == 0 and results['locked'] == 0:
storage.store(test.name, 'correct', True)
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
if not verbose and (failed > 0 or locked > 0):
# Stop at the first failed test
break
format.print_progress_bar('Test summary', passed, failed, locked,
verbose=verbose)
print()
messages['grading'] = analytics
def _run_case(self, test_name, suite_number, case, case_number):
"""A wrapper for case.run().
Prints informative output and also captures output of the test case
and returns it as a log. The output is printed only if the case fails,
or if self.verbose is True.
"""
output.off() # Delay printing until case status is determined.
log_id = output.new_log()
format.print_line('-')
print('{} > Suite {} > Case {}'.format(test_name, suite_number,
case_number))
print()
success = case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
if not success or self.verbose:
print(''.join(output_log))
if not success:
short_name = self.test.get_short_name()
# TODO: Change when in notebook mode
print('Run only this test case with '
'"python3 ok -q {} --suite {} --case {}"'.format(
short_name, suite_number, case_number))
return success
def on_interact(self):
"""Run gradeable tests and print results and return analytics.
RETURNS:
dict; a mapping of test name -> JSON-serializable object. It is up to
each test to determine what kind of data it wants to return as
significant for analytics. However, all tests must include the number
passed, the number of locked tests and the number of failed tests.
"""
if self.args.score:
return
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
for test in self.assignment.specified_tests:
log.info('Running tests for {}'.format(test.name))
results = test.run()
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
format.print_progress_bar('Test summary', passed, failed, locked)
return analytics
def grade(questions, messages, env=None, verbose=True):
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
# The environment in which to run the tests.
for test in questions:
log.info('Running tests for {}'.format(test.name))
results = test.run(env)
# if correct once, set persistent flag
if results['failed'] == 0:
storage.store(test.name, 'correct', True)
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
current_directory = os.getcwd()
if not os.path.exists(current_directory + "/submissions"):
def run(self):
"""Runs the suites associated with this doctest.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
output.off()
log_id = output.new_log()
format.print_line('-')
print('Doctests for {}'.format(self.name))
print()
success = self.case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
if not success or self.verbose:
print(''.join(output_log))
if not success and self.interactive:
self.console.interact()
if success:
return {'passed': 1, 'failed': 0, 'locked': 0}
else:
return {'passed': 0, 'failed': 1, 'locked': 0}
def unlock(self, interact):
total_cases = len([case for suite in self.suites
for case in suite.cases])
for suite_num, suite in enumerate(self.suites):
for case_num, case in enumerate(suite.cases):
case_id = '{} > Suite {} > Case {}'.format(
self.name, suite_num + 1, case_num + 1)
format.print_line('-')
print(case_id)
print('(cases remaining: {})'.format(total_cases))
print()
total_cases -= 1
if case.locked != True:
print('-- Already unlocked --')
print()
continue
case.unlock(self.unique_id_prefix, case_id, interact)
assert total_cases == 0, 'Number of cases is incorrect'
format.print_line('-')
print('OK! All cases for {} unlocked.'.format(self.name))
print()
def on_interact(self):
"""Run gradeable tests and print results and return analytics.
RETURNS:
dict; a mapping of test name -> JSON-serializable object. It is up to
each test to determine what kind of data it wants to return as
significant for analytics. However, all tests must include the number
passed, the number of locked tests and the number of failed tests.
"""
if self.args.score:
return
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
for test in self.assignment.specified_tests:
log.info('Running tests for {}'.format(test.name))
results = test.run()
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
format.print_progress_bar('Test summary', passed, failed, locked)
return analytics
def run(self, messages, env=None):
"""Score tests and print results. Tests are taken from
self.assignment.specified_tests. A score breakdown by question and the
total score are both printed.
ENV is used by the programatic API for Python doctests only.
"""
if not self.args.score or self.args.testing:
return
format.print_line('~')
print('Scoring tests')
print()
raw_scores = OrderedDict()
for test in self.assignment.specified_tests:
assert isinstance(test, sources_models.Test), 'ScoringProtocol received invalid test'
log.info('Scoring test {}'.format(test.name))
# A hack that allows programmatic API users to plumb a custom
# environment through to Python tests.
# Use type to ensure is an actual OkTest and not a subclass
if type(test) == ok_test_models.OkTest:
score = test.score(env=env)
else:
score = test.score()
raw_scores[test.name] = (score, test.points)
messages['scoring'] = display_breakdown(raw_scores, self.args.score_out)
print()
def run(self, messages):
"""Score tests and print results
Tests are taken from self.assignment.specified_tests. Each test belongs
to a partner. If test.partner is omitted (i.e. core.NoValue), the score
for that test is added to every partner's score.
If there are no tests, the mapping will only contain one entry, mapping
"Total" -> 0 (total score).
If there are no partners specified by the tests, the mapping will only
contain one entry, mapping "Total" (partner) -> total score (float).
This assumes there is always at least one partner.
"""
if self.args.export or not self.args.score:
return
format.print_line('~')
print('Scoring tests')
print()
raw_scores = OrderedDict()
for test in self.assignment.specified_tests:
assert isinstance(test, sources_models.Test), 'ScoringProtocol received invalid test'
log.info('Scoring test {}'.format(test.name))
partner = test.partner if test.partner != core.NoValue else None
raw_scores[test.name, partner] = (test.score(), test.points)
messages['scoring'] = display_breakdown(raw_scores)
print()
def display_breakdown(scores):
"""Prints the point breakdown given a dictionary of scores.
RETURNS:
dict; maps partner (str) -> finalized score (float)
"""
partner_totals = {}
format.print_line('-')
print('Point breakdown')
for (name, partner), (score, total) in scores.items():
print(' {}: {}/{}'.format(name, score, total))
partner_totals[partner] = partner_totals.get(partner, 0) + score
print()
shared_points = partner_totals.get(None, 0)
if None in partner_totals:
del partner_totals[None]
finalized_scores = {}
print('Score:')
if len(partner_totals) == 0:
print(' {}: {}'.format(NO_PARTNER_NAME, shared_points))
finalized_scores[NO_PARTNER_NAME] = shared_points
else:
for partner, score in sorted(partner_totals.items()):
print(' Partner {}: {}'.format(partner, score + shared_points))
finalized_scores[partner] = score + shared_points
return finalized_scores
def grade(questions, messages, env=None, verbose=True):
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
# The environment in which to run the tests.
for test in questions:
log.info('Running tests for {}'.format(test.name))
results = test.run(env)
# if correct once, set persistent flag
if results['failed'] == 0 and results['locked'] == 0:
storage.store(test.name, 'correct', True)
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
if not verbose and (failed > 0 or locked > 0):
# Stop at the first failed test
break
format.print_progress_bar('Test summary', passed, failed, locked,
verbose=verbose)
print()
messages['grading'] = analytics
def run(self):
"""Runs the suites associated with this doctest.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
output.off()
log_id = output.new_log()
format.print_line('-')
print('Doctests for {}'.format(self.name))
print()
if not self.docstring:
print('-- No doctests found for {} --'.format(self.name))
success = False
else:
success = self.case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
if not success or self.verbose:
print(''.join(output_log))
if not success and self.interactive:
self.console.interact()
if success:
return {'passed': 1, 'failed': 0, 'locked': 0}
else:
return {'passed': 0, 'failed': 1, 'locked': 0}
def run(self, messages, env=None):
"""Score tests and print results. Tests are taken from
self.assignment.specified_tests. A score breakdown by question and the
total score are both printed.
ENV is used by the programatic API for Python doctests only.
"""
if not self.args.score:
return
format.print_line('~')
print('Scoring tests')
print()
raw_scores = OrderedDict()
for test in self.assignment.specified_tests:
assert isinstance(test, sources_models.Test), 'ScoringProtocol received invalid test'
log.info('Scoring test {}'.format(test.name))
# A hack that allows programmatic API users to plumb a custom
# environment through to Python tests.
# Use type to ensure is an actual OkTest and not a subclass
if type(test) == ok_test_models.OkTest:
score = test.score(env=env)
else:
score = test.score()
raw_scores[test.name] = (score, test.points)
messages['scoring'] = display_breakdown(raw_scores, self.args.score_out)
print()
def _print_header(self):
if getattr(self.cmd_args, 'autobackup_actual_run_sync', False):
return
format.print_line('=')
print('Assignment: {}'.format(self.name))
print('OK, version {}'.format(client.__version__))
format.print_line('=')
print()
def score(self):
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
_, failed = self._run_tests()
score = 1.0 if failed == 0 else 0.0
print('Score: {}/1'.format(score))
print()
return score
def score(self):
format.print_line('-')
print('Doctests for {}'.format(self.name))
print()
success = self.case.run()
score = 1.0 if success else 0.0
print('Score: {}/1'.format(score))
print()
return score
def score(self):
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
_, failed = self._run_tests()
score = 1.0 if failed == 0 else 0.0
print('Score: {}/1'.format(score))
print()
return score
def score(self):
format.print_line('-')
print('Doctests for {}'.format(self.name))
print()
success = self.case.run()
score = 1.0 if success else 0.0
print('Score: {}/1'.format(score))
print()
return score
def on_start(self):
"""Responsible for locking each test."""
if not self.args.lock:
return
format.print_line('~')
print('Locking tests')
print()
for test in self.assignment.test_map.values():
log.info('Locking {}'.format(test.name))
test.lock(self._hash_fn)
def _compare(self, expected, code):
try:
value, printed = self.evaluate(code)
except ConsoleException as e:
detail = "{}: {}".format(e.exception_type, str(e.exception))
actual = CodeAnswer(exception=True,
exception_type=e.exception_type,
exception_detail=detail.splitlines())
else:
if value is not None:
print(self._output_fn(value))
printed += self._output_fn(value)
output = printed.splitlines()
actual = CodeAnswer(output=output)
self.cases_total += 1
if not self.skip_locked_cases and expected.locked:
if '\n'.join(expected.output) != locking.lock(
self.hash_key, actual.dump()):
print()
print("# Error: expected and actual results do not match")
raise ConsoleException
else:
return
correct = (expected.exception == actual.exception
and expected.output_lines() == actual.output_lines())
correct_legacy_exception = (actual.exception
and [actual.exception_type
] == expected.output_lines())
if not correct and not correct_legacy_exception:
print()
print('# Error: expected')
print('\n'.join('# {}'.format(line)
for line in expected.output_lines()))
print('# but got')
print('\n'.join('# {}'.format(line)
for line in actual.output_lines()))
# Bail out on first failed test, or if we're showing all test results, bail on infinite loop timeout
if not self.show_all_cases or (actual.exception
and actual.exception_type
== exceptions.Timeout.__name__):
raise ConsoleException
elif self.CASE_PREFIX in code:
print(":(", f"Test Case {self.cases_total} failed")
format.print_line('-')
print()
elif correct:
self.cases_passed += 1
if self.CASE_PREFIX in code:
print(":D", f"Test Case {self.cases_total} passed")
format.print_line('-')
print()
def run(self, messages):
"""Responsible for locking each test."""
if not self.args.lock:
return
format.print_line('~')
print('Locking tests')
print()
for test in self.assignment.test_map.values():
log.info('Locking {}'.format(test.name))
test.lock(self._hash_fn)
def run(self, messages):
"""Run gradeable tests and print results and return analytics.
RETURNS:
dict; a mapping of test name -> JSON-serializable object. It is up to
each test to determine what kind of data it wants to return as
significant for analytics. However, all tests must include the number
passed, the number of locked tests and the number of failed tests.
"""
if self.args.score or self.args.export or self.args.unlock or self.args.restore:
return
format.print_line('~')
print('Running tests')
print()
passed = 0
failed = 0
locked = 0
analytics = {}
# check if analytics info is in messages
if 'analytics' in messages:
started = messages['analytics']['started']
else:
started = None
for test in self.assignment.specified_tests:
# run test if the question is not detected, or question detected and started
if (started is None
or test.name not in started
or started[test.name]):
log.info('Running tests for {}'.format(test.name))
results = test.run()
passed += results['passed']
failed += results['failed']
locked += results['locked']
analytics[test.name] = results
else:
print('It looks like you haven\'t started {}. Skipping the tests.'.format(test.name))
print()
if not self.args.verbose and (failed > 0 or locked > 0):
# Stop at the first failed test
break
format.print_progress_bar('Test summary', passed, failed, locked,
verbose=self.args.verbose)
print()
messages['grading'] = analytics
def lock(self, hash_fn):
format.print_line('-')
print(self.name)
for suite_num, suite in enumerate(list(self.suites)):
for case_num, case in enumerate(list(suite.cases)):
message = '* Suite {} > Case {}: '.format(suite_num, case_num)
if case.hidden:
suite.cases.remove(case)
print(message + 'removing hidden case')
elif case.locked == core.NoValue:
case.lock(hash_fn)
print(message + 'locking')
elif case.locked == False:
print(message + 'leaving unlocked')
elif case.locked == True:
print(message + 'already unlocked')
def run(self):
"""Runs the suites associated with this doctest.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
passed, failed = self._run_tests()
print('{} passed; {} failed'.format(passed, failed))
if failed == 0 and passed > 0:
print('-- OK! --')
print()
return {'passed': passed, 'failed': failed, 'locked': 0}
def run(self):
"""Runs the suites associated with this doctest.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
format.print_line("-")
print("Scheme tests in {}".format(self.file))
print()
passed, failed = self._run_tests()
print("{} passed; {} failed".format(passed, failed))
if failed == 0 and passed > 0:
print("-- OK! --")
print()
return {"passed": passed, "failed": failed, "locked": 0}
def run(self):
"""Runs the suites associated with this doctest.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
passed, failed = self._run_tests()
print('{} passed; {} failed'.format(passed, failed))
if failed == 0 and passed > 0:
print('-- OK! --')
print()
return {'passed': passed, 'failed': failed, 'locked': 0}
def display_breakdown(scores, outfile):
"""Writes the point breakdown to outfile given a dictionary of scores.
RETURNS:
dict; 'Total' -> finalized score (float)
"""
total = 0
format.print_line('-')
print('Point breakdown', file=outfile)
for name, (score, max_score) in scores.items():
print(' {}: {}/{}'.format(name, score, max_score), file=outfile)
total += score
print(file=outfile)
print('Score:', file=outfile)
print(' Total: {}'.format(total), file=outfile)
return {'Total': total}
def run(self, messages):
"""Responsible for unlocking each test.
The unlocking process can be aborted by raising a KeyboardInterrupt or
an EOFError.
RETURNS:
dict; mapping of test name (str) -> JSON-serializable object. It is up
to each test to determine what information is significant for analytics.
"""
if not self.args.unlock:
return
format.print_line('~')
print('Unlocking tests')
print()
print('At each "{}", type what you would expect the output to be.'.format(
self.PROMPT))
print('Type {} to quit'.format(self.EXIT_INPUTS[0]))
print()
for test in self.assignment.specified_tests:
log.info('Unlocking test {}'.format(test.name))
self.current_test = test.name
# Reset guidance explanation probability for every question
self.guidance_util.prompt_probability = guidance.DEFAULT_PROMPT_PROBABILITY
try:
test.unlock(self.interact)
except (KeyboardInterrupt, EOFError):
try:
# TODO(albert): When you use Ctrl+C in Windows, it
# throws two exceptions, so you need to catch both
# of them. Find a cleaner fix for this.
print()
print('-- Exiting unlocker --')
except (KeyboardInterrupt, EOFError):
pass
print()
break
messages['unlock'] = self.analytics
def run(self, messages):
"""Responsible for unlocking each test.
The unlocking process can be aborted by raising a KeyboardInterrupt or
an EOFError.
RETURNS:
dict; mapping of test name (str) -> JSON-serializable object. It is up
to each test to determine what information is significant for analytics.
"""
if not self.args.unlock:
return
format.print_line('~')
print('Unlocking tests')
print()
print('At each "{}", type what you would expect the output to be.'.format(
self.PROMPT))
print('Type {} to quit'.format(self.EXIT_INPUTS[0]))
print()
for test in self.assignment.specified_tests:
log.info('Unlocking test {}'.format(test.name))
self.current_test = test.name
# Reset guidance explanation probability for every question
self.guidance_util.prompt_probability = guidance.DEFAULT_PROMPT_PROBABILITY
try:
test.unlock(self.interact)
except (KeyboardInterrupt, EOFError):
try:
# TODO(albert): When you use Ctrl+C in Windows, it
# throws two exceptions, so you need to catch both
# of them. Find a cleaner fix for this.
print()
print('-- Exiting unlocker --')
except (KeyboardInterrupt, EOFError):
pass
print()
break
messages['unlock'] = self.analytics
def display_breakdown(scores, outfile=None):
"""Writes the point breakdown to `outfile` given a dictionary of scores.
`outfile` should be a string. If `outfile` is None, write to stdout.
RETURNS:
dict; 'Total' -> finalized score (float)
"""
total = 0
outfile = open(outfile, 'w') if outfile else sys.stdout
format.print_line('-')
print('Point breakdown', file=outfile)
for name, (score, max_score) in scores.items():
print(' {}: {}/{}'.format(name, score, max_score), file=outfile)
total += score
print(file=outfile)
print('Score:', file=outfile)
print(' Total: {}'.format(total), file=outfile)
return {'Total': total}
def display_breakdown(scores, outfile=None):
"""Writes the point breakdown to `outfile` given a dictionary of scores.
`outfile` should be a string. If `outfile` is None, write to stdout.
RETURNS:
dict; 'Total' -> finalized score (float)
"""
total = 0
outfile = open(outfile, 'w') if outfile else sys.stdout
format.print_line('-')
print('Point breakdown', file=outfile)
for name, (score, max_score) in scores.items():
print(' {}: {}/{}'.format(name, score, max_score), file=outfile)
total += score
print(file=outfile)
print('Score:', file=outfile)
print(' Total: {}'.format(total), file=outfile)
return {'Total': total}
def unlock(self, interact):
total_cases = len([case for suite in self.suites
for case in suite.cases])
for suite_num, suite in enumerate(self.suites):
for case_num, case in enumerate(suite.cases):
case_id = '{} > Suite {} > Case {}'.format(
self.name, suite_num + 1, case_num + 1)
format.print_line('-')
print(case_id)
print('(cases remaining: {})'.format(total_cases))
print()
total_cases -= 1
if case.locked != True:
print('-- Already unlocked --')
print()
continue
case.unlock(self.unique_id_prefix, case_id, interact)
def run(self, env):
"""Runs the suites associated with this doctest.
NOTE: env is intended only for use with the programmatic API to support
Python OK tests. It is not used here.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
output.off()
log_id = output.new_log()
format.print_line('-')
print('Doctests for {}'.format(self.name))
print()
if not self.docstring:
print('-- No doctests found for {} --'.format(self.name))
if self.ignore_empty:
success = True
else:
success = False
else:
success = self.case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
if not success or self.verbose:
print(''.join(output_log))
if not success and self.interactive:
self.console.interact()
if success:
return {'passed': 1, 'failed': 0, 'locked': 0}
else:
return {'passed': 0, 'failed': 1, 'locked': 0}
def run(self, env):
"""Runs the suites associated with this doctest.
NOTE: env is intended only for use with the programmatic API to support
Python OK tests. It is not used here.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
passed, failed = self._run_tests()
print('{} passed; {} failed'.format(passed, failed))
if failed == 0 and passed > 0:
print('-- OK! --')
print()
return {'passed': passed, 'failed': failed, 'locked': 0}
def lock(self, hash_fn):
format.print_line('-')
print(self.name)
for suite_num, suite in enumerate(list(self.suites)):
for case_num, case in enumerate(list(suite.cases)):
message = '* Suite {} > Case {}: '.format(suite_num, case_num)
if case.hidden:
suite.cases.remove(case)
print(message + 'removing hidden case')
elif case.locked == core.NoValue:
case.lock(hash_fn)
print(message + 'locking')
elif case.locked == False:
print(message + 'leaving unlocked')
elif case.locked == True:
print(message + 'already unlocked')
if not suite.cases:
self.suites.remove(suite)
print('* Suite {}: removing empty suite'.format(suite_num))
print()
def run(self, env):
"""Runs the suites associated with this doctest.
NOTE: env is intended only for use with the programmatic API to support
Python OK tests. It is not used here.
RETURNS:
bool; True if the doctest completely passes, False otherwise.
"""
format.print_line('-')
print('Scheme tests in {}'.format(self.file))
print()
passed, failed = self._run_tests()
print('{} passed; {} failed'.format(passed, failed))
if failed == 0 and passed > 0:
print('-- OK! --')
print()
return {'passed': passed, 'failed': failed, 'locked': 0}
class Suite(core.Serializable):
type = core.String()
scored = core.Boolean(default=True)
cases = core.List()
def __init__(self, verbose, interactive, timeout=None, **fields):
super().__init__(**fields)
self.verbose = verbose
self.interactive = interactive
self.timeout = timeout
def run(self, test_name, suite_number):
"""Subclasses should override this method to run tests.
PARAMETERS:
test_name -- str; name of the parent test.
suite_number -- int; suite number, assumed to be 1-indexed.
RETURNS:
dict; results of the following form:
{
'passed': int,
'failed': int,
'locked': int,
}
"""
raise NotImplementedError
def _run_case(self, test_name, suite_number, case, case_number):
"""A wrapper for case.run().
Prints informative output and also captures output of the test case
and returns it as a log. The output is suppressed -- it is up to the
calling function to decide whether or not to print the log.
"""
output.off() # Delay printing until case status is determined.
log_id = output.new_log()
format.print_line('-')
print('{} > Suite {} > Case {}'.format(test_name, suite_number,
case_number))
print()
success = case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
return success, output_log
def on_interact(self):
"""Responsible for unlocking each test.
The unlocking process can be aborted by raising a KeyboardInterrupt or
an EOFError.
RETURNS:
dict; mapping of test name (str) -> JSON-serializable object. It is up
to each test to determine what information is significant for analytics.
"""
if not self.args.unlock:
return
format.print_line('~')
print('Unlocking tests')
print()
print('At each "{}", type what you would expect the output to be.'.format(
self.PROMPT))
print('Type {} to quit'.format(self.EXIT_INPUTS[0]))
print()
analytics = {}
for test in self.assignment.specified_tests:
log.info('Unlocking test {}'.format(test.name))
try:
analytics[test.name] = test.unlock(self.interact)
except (KeyboardInterrupt, EOFError):
try:
# TODO(albert): When you use Ctrl+C in Windows, it
# throws two exceptions, so you need to catch both
# of them. Find a cleaner fix for this.
print()
print('-- Exiting unlocker --')
except (KeyboardInterrupt, EOFError):
pass
print()
break
return analytics
def _run_case(self, test_name, suite_number, case, case_number):
"""A wrapper for case.run().
Prints informative output and also captures output of the test case
and returns it as a log. The output is suppressed -- it is up to the
calling function to decide whether or not to print the log.
"""
output.off() # Delay printing until case status is determined.
log_id = output.new_log()
format.print_line('-')
print('{} > Suite {} > Case {}'.format(test_name, suite_number,
case_number))
print()
success = case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
return success, output_log
def _print_header(self):
format.print_line('=')
print('Assignment: {}'.format(self.name))
print('OK, version {}'.format(client.__version__))
format.print_line('=')
print()
class OkTest(models.Test):
suites = core.List()
description = core.String(optional=True)
def __init__(self, file, suite_map, assign_name, verbose, interactive,
timeout=None, **fields):
super().__init__(**fields)
self.file = file
self.suite_map = suite_map
self.verbose = verbose
self.interactive = interactive
self.timeout = timeout
self.assignment_name = assign_name
def post_instantiation(self):
for i, suite in enumerate(self.suites):
if not isinstance(suite, dict):
raise ex.SerializeException('Test cases must be dictionaries')
elif 'type' not in suite:
raise ex.SerializeException('Suites must have field "type"')
elif suite['type'] not in self.suite_map:
raise ex.SerializeException('Invalid suite type: '
'{}'.format(suite['type']))
self.suites[i] = self.suite_map[suite['type']](
self.verbose, self.interactive, self.timeout, **suite)
def run(self):
"""Runs the suites associated with this OK test.
RETURNS:
dict; the results for this test, in the form
{
'passed': int,
'failed': int,
'locked': int,
}
"""
passed, failed, locked = 0, 0, 0
for i, suite in enumerate(self.suites):
results = suite.run(self.name, i + 1)
passed += results['passed']
failed += results['failed']
locked += results['locked']
if not self.verbose and (failed > 0 or locked > 0):
# Stop at the first failed test
break
if locked > 0:
print()
print('There are still locked tests! '
'Use the -u option to unlock them')
if type(self.description) == str and self.description:
print()
print(self.description)
print()
return {
'passed': passed,
'failed': failed,
'locked': locked,
}
def score(self):
"""Runs test cases and computes the score for this particular test.
Scores are determined by aggregating results from suite.run() for each
suite. A suite is considered passed only if it results in no locked
nor failed results.
The points available for this test are distributed evenly across
scoreable (i.e. unlocked and 'scored' = True) suites.
"""
passed, total = 0, 0
for i, suite in enumerate(self.suites):
if not suite.scored:
continue
total += 1
results = suite.run(self.name, i + 1)
if results['locked'] == 0 and results['failed'] == 0:
passed += 1
if total > 0:
score = passed * self.points / total
else:
score = 0.0
format.print_progress_bar(self.name, passed, total - passed, 0)
print()
return score
def unlock(self, interact):
total_cases = len([case for suite in self.suites
for case in suite.cases])
for suite_num, suite in enumerate(self.suites):
for case_num, case in enumerate(suite.cases):
case_id = '{} > Suite {} > Case {}'.format(
self.name, suite_num + 1, case_num + 1)
format.print_line('-')
print(case_id)
print('(cases remaining: {})'.format(total_cases))
print()
total_cases -= 1
if case.locked != True:
print('-- Already unlocked --')
print()
continue
case.unlock(self.unique_id_prefix, case_id, interact)
assert total_cases == 0, 'Number of cases is incorrect'
format.print_line('-')
print('OK! All cases for {} unlocked.'.format(self.name))
print()
def _print_header(self):
format.print_line('=')
print('Assignment: {}'.format(self.name))
print('OK, version {}'.format(client.__version__))
format.print_line('=')
print()
class Suite(core.Serializable):
type = core.String()
scored = core.Boolean(default=True)
cases = core.List()
def __init__(self, test, verbose, interactive, timeout=None, **fields):
super().__init__(**fields)
self.test = test
self.verbose = verbose
self.interactive = interactive
self.timeout = timeout
self.run_only = []
def run(self, test_name, suite_number, env=None):
"""Subclasses should override this method to run tests.
PARAMETERS:
test_name -- str; name of the parent test.
suite_number -- int; suite number, assumed to be 1-indexed.
env -- dict; used by programmatic API to provide a
custom environment to run tests with.
RETURNS:
dict; results of the following form:
{
'passed': int,
'failed': int,
'locked': int,
}
"""
raise NotImplementedError
def enumerate_cases(self):
enumerated = enumerate(self.cases)
if self.run_only:
return [x for x in enumerated if x[0] + 1 in self.run_only]
return enumerated
def _run_case(self, test_name, suite_number, case, case_number):
"""A wrapper for case.run().
Prints informative output and also captures output of the test case
and returns it as a log. The output is printed only if the case fails,
or if self.verbose is True.
"""
output.off() # Delay printing until case status is determined.
log_id = output.new_log()
format.print_line('-')
print('{} > Suite {} > Case {}'.format(test_name, suite_number,
case_number))
print()
success = case.run()
if success:
print('-- OK! --')
output.on()
output_log = output.get_log(log_id)
output.remove_log(log_id)
if not success or self.verbose:
print(''.join(output_log))
if not success:
short_name = self.test.get_short_name()
# TODO: Change when in notebook mode
print('Run only this test case with '
'"python3 ok -q {} --suite {} --case {}"'.format(
short_name, suite_number, case_number))
return success
