def testFailOnNonexistentConfig(self):
self.mock_is_file.return_value = False
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment('does_not_exist.ok')
self.assertEquals(
'Could not find config file: does_not_exist.ok',
str(cm.exception))
def testFailOnCorruptedConfig(self):
corrupted_config = '{not valid json}'
mock_open = mock.mock_open(read_data=corrupted_config)
with mock.patch('builtins.open', mock_open):
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment('corrupted.ok')
self.assertEquals(
'corrupted.ok is a malformed .ok configuration file. '
'Please re-download corrupted.ok.', str(cm.exception))
def testFailOnMultipleFoundConfigs(self):
self.mock_glob.return_value = ['1.ok', '2.ok']
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment()
self.assertEquals(
'\n'.join([
'Multiple .ok files found:', ' 1.ok 2.ok',
"Please specify a particular assignment's config file with",
' python3 ok --config <config file>'
]), str(cm.exception))
def testFailOnCorruptedConfig(self):
corrupted_config = '{not valid json}'
mock_open = mock.mock_open(read_data=corrupted_config)
with mock.patch('builtins.open', mock_open):
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment('corrupted.ok')
self.assertEquals(
'corrupted.ok is a malformed .ok configuration file. '
'Please re-download corrupted.ok.',
str(cm.exception))
def testFailOnMultipleFoundConfigs(self):
self.mock_glob.return_value = ['1.ok', '2.ok']
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment()
self.assertEquals(
'\n'.join([
'Multiple .ok files found:',
' 1.ok 2.ok',
"Please specify a particular assignment's config file with",
' python3 ok --config <config file>'
]),
str(cm.exception))
def main():
"""Run the LockingProtocol."""
args = parse_input()
args.lock = True
args.question = []
args.all = False
args.timeout = 0
args.verbose = False
args.interactive = False
args.ignore_empty = False
args.parsons = False
try:
assign = assignment.load_assignment(args.config, args)
msgs = messages.Messages()
lock.protocol(args, assign).run(msgs)
except (ex.LoadingException, ex.SerializeException) as e:
log.warning('Assignment could not instantiate', exc_info=True)
print_error('Error: ' + str(e).strip())
exit(1)
except (KeyboardInterrupt, EOFError):
log.info('Quitting...')
else:
assign.dump_tests()
def analytics_event():
"""
{
problem_name: string,
event: 'start' | 'stop'
}
Triggered when user starts interacting with the problem and when they stop (e.g. switch tabs).
This data can be used to get compute analytics about time spent on parsons.
"""
e, problem_name = request.json['event'], request.json['problem_name']
msgs = messages.Messages()
args = cache['args']
args.question = [problem_name]
with DisableStdout():
assign = load_assignment(args.config, args)
if e == 'start':
msgs['action'] = 'start'
elif e == 'stop':
msgs['action'] = 'stop'
msgs['problem'] = problem_name
analytics_protocol = assign.protocol_map['analytics']
backup_protocol = assign.protocol_map['backup']
with DisableStdout():
analytics_protocol.run(msgs)
backup_protocol.run(msgs)
msgs['timestamp'] = str(datetime.now())
return jsonify({})
def load_assignment_if_possible(args):
"""
A syntax error in a source file leads to ok not being able to load an assignment.
For parsons files, we can get around this by replacing a parsons program with dummy
code. This function will do that if necessary and return the assignment, or raise
the relevant LoadingException if a different error occurs (such as a syntax error
in the main python file).
"""
# remove syntax errors so assignment can load
num_retries = MAX_NUM_RETRIES
reloaded = []
assign = None
while num_retries > 0:
try:
assign = load_assignment(args.config, args)
break
except ex.LoadingException as e:
# TODO: LoadingException unique with syntax error (vs missing .ok, for example)
if PARSONS_FOLDER_NAME not in str(e):
raise
fname = str(e).split(" ")[-1]
rel_path = fname.split("/")[1]
problem_name = rel_path[:-3]
reloaded.append(problem_name)
# replaces syntax-error code with error-free dummy code
write_parsons_prob_locally(
fname, "def dummy():\n print('Syntax Error')\n", None,
False)
num_retries -= 1
return assign
def grade_and_backup(problem_name):
args = cache['args']
args.question = [problem_name]
msgs = messages.Messages()
old_stdout = sys.stdout
sys.stdout = out = open(PARSONS_OUTFILE, 'w')
assign = load_assignment(args.config, args)
for name, proto in assign.protocol_map.items():
log.info('Execute {}.run()'.format(name))
proto.run(msgs)
out.close()
sys.stdout = old_stdout
msgs['timestamp'] = str(datetime.now())
feedback = {}
feedback['passed'] = assign.specified_tests[0].console.cases_passed
feedback['failed'] = assign.specified_tests[
0].console.cases_total - feedback['passed']
# get output from doctests
with open(PARSONS_OUTFILE, "r", encoding="utf8") as f:
all_lines = f.readlines()
# still need to fix ok-client show all cases to not print extra ------
# feedback['doctest_logs'] = "".join(all_lines[3:-10])
log_lines = all_lines[9:-10]
if is_syntax_error(feedback):
log_lines = get_useful_syntax_error_logs(log_lines, problem_name)
feedback['doctest_logs'] = "".join(log_lines)
# passed == 0 and failed == 0 can be a result of SyntaxError so we check passed >= 1 instead
store_correctness(problem_name, feedback['passed'] >= 1
and feedback['failed'] == 0)
return feedback
def testSpecificConfig(self):
mock_open = mock.mock_open(read_data=self.VALID_CONFIG)
with mock.patch('builtins.open', mock_open):
with mock.patch('client.api.assignment.Assignment') as mock_assign:
assign = assignment.load_assignment('some_config.ok')
# Verify config file was actually opened.
mock_open.assert_called_with('some_config.ok', 'r')
mock_open().read.assert_called_once_with()
def testSpecificConfig(self):
mock_open = mock.mock_open(read_data=self.VALID_CONFIG)
with mock.patch('builtins.open', mock_open):
with mock.patch('client.api.assignment.Assignment') as mock_assign:
assign = assignment.load_assignment('some_config.ok')
# Verify config file was actually opened.
mock_open.assert_called_with('some_config.ok', 'r')
mock_open().read.assert_called_once_with()
def testSearchForConfigInFilesystem(self):
self.mock_glob.return_value = ['some_config.ok']
mock_open = mock.mock_open(read_data=self.VALID_CONFIG)
with mock.patch('builtins.open', mock_open):
with mock.patch('client.api.assignment.Assignment') as mock_assign:
assign = assignment.load_assignment()
# Verify config file was actually opened.
mock_open.assert_called_with('some_config.ok', 'r')
mock_open().read.assert_called_once_with()
def testSearchForConfigInFilesystem(self):
self.mock_glob.return_value = ['some_config.ok']
mock_open = mock.mock_open(read_data=self.VALID_CONFIG)
with mock.patch('builtins.open', mock_open):
with mock.patch('client.api.assignment.Assignment') as mock_assign:
assign = assignment.load_assignment()
# Verify config file was actually opened.
mock_open.assert_called_with('some_config.ok', 'r')
mock_open().read.assert_called_once_with()
def run_tests():
reload_tests()
# noinspection PyUnresolvedReferences
from client.api import assignment
import logging
LOGGING_FORMAT = '%(levelname)s | %(filename)s:%(lineno)d | %(message)s'
logging.basicConfig(format=LOGGING_FORMAT)
log = logging.getLogger('client') # Get top-level logger
# noinspection PyUnresolvedReferences
from client.cli.ok import parse_input
# noinspection PyUnresolvedReferences
from client.sources.ok_test.scheme import SchemeSuite
# noinspection PyUnresolvedReferences
from client.sources.doctest.models import Doctest
# noinspection PyUnresolvedReferences
from client.sources.scheme_test.models import SchemeTest
log.setLevel(logging.ERROR)
args = parse_input(["--all", "--verbose"])
assign = assignment.load_assignment(None, args)
try:
result = []
for test in assign.specified_tests:
if isinstance(test, (Doctest, SchemeTest)):
# doctests are python
continue
suites = []
for suite in test.suites:
if not isinstance(suite, SchemeSuite):
# python ok test
continue
suites.append(
[process_case(case).dictionary for case in suite.cases])
if not suites:
continue
result.append({
"problem": test.name.replace("-", " ").title(),
"suites": suites,
"passed": all(x['passed'] for t in suites for x in t)
})
return result
except TerminatedError:
return [{
'problem': "Tests Terminated by User",
'suites': [],
'passed': False
}]
def makeAssignmentWithTestCode(self, test_code):
self.mock_glob.return_value = ['q1.py']
self.mock_load_module.return_value.test = {
'name': 'Homework 1',
'points': 1,
'suites': [{
'type': 'doctest',
'cases': [{
'code': test_code
}]
}]
}
with mock.patch('builtins.open',
mock.mock_open(read_data=self.CONFIG)):
return assignment.load_assignment('some_config.ok')
def makeAssignmentWithTestCode(self, test_code):
self.mock_glob.return_value = ['q1.py']
self.mock_load_module.return_value.test = {
'name': 'Homework 1',
'points': 1,
'suites': [
{
'type': 'doctest',
'cases': [
{ 'code': test_code }
]
}
]
}
with mock.patch('builtins.open', mock.mock_open(read_data=self.CONFIG)):
return assignment.load_assignment('some_config.ok')
def __init__(self,
filepath=None,
cmd_args=None,
debug=False,
mode='jupyter'):
ok_logger = logging.getLogger('client') # Get top-level ok logger
ok_logger.setLevel(logging.DEBUG if debug else logging.ERROR)
self.assignment = load_assignment(filepath, cmd_args)
# Attempt a login with enviornment based tokens
login_with_env(self.assignment)
if mode not in ["jupyter", "jupyterlab"]:
raise ValueError(
"Bad mode argument: should be either \'jupyter\' or \'jupyterlab\'"
)
self.mode = mode
def main():
"""Run GradingProtocol and ScoringProtocol."""
args = ok.parse_input()
log.setLevel(logging.DEBUG if args.debug else logging.ERROR)
log.debug(args)
try:
assign = assignment.load_assignment(args.config, args)
msgs = messages.Messages()
grading.protocol(args, assign).run(msgs)
scoring.protocol(args, assign).run(msgs)
except (ex.LoadingException, ex.SerializeException) as e:
log.warning('Assignment could not instantiate', exc_info=True)
print('Error: ' + str(e).strip())
exit(1)
except (KeyboardInterrupt, EOFError):
log.info('Quitting...')
def main():
"""Run GradingProtocol and ScoringProtocol."""
args = ok.parse_input()
log.setLevel(logging.DEBUG if args.debug else logging.ERROR)
log.debug(args)
try:
assign = assignment.load_assignment(args.config, args)
msgs = messages.Messages()
grading.protocol(args, assign).run(msgs)
scoring.protocol(args, assign).run(msgs)
except (ex.LoadingException, ex.SerializeException) as e:
log.warning('Assignment could not instantiate', exc_info=True)
print('Error: ' + str(e).strip())
exit(1)
except (KeyboardInterrupt, EOFError):
log.info('Quitting...')
def grade(src_names, test_names):
# create config and test files required by ok
create_ok_files(src_names, test_names)
args = assignment.Settings(
verbose=
True, # prints out ALL tests but only sends JSON results up to first fail
config=CONFIG_NAME # config file to use
)
r, w = os.pipe()
# parent
if os.fork() > 0:
# wait for child to finish
childProcExitInfo = os.wait()
# read grading results from child
os.close(w)
r = os.fdopen(r)
data = r.read()
r.close()
return data
# child
os.close(r)
# fork because this ignores subsequent submissions
assign = assignment.load_assignment(args.config, args)
os.dup2(w, 1) # stdout goes to w
msgs = messages.Messages()
for name, proto in assign.protocol_map.items():
if name == 'grading': # only run grading protocol
proto.run(msgs)
os._exit(1)
def main():
"""Run the LockingProtocol."""
args = parse_input()
args.lock = True
args.question = []
args.all = False
args.timeout = 0
args.verbose = False
args.interactive = False
try:
assign = assignment.load_assignment(args.config, args)
msgs = messages.Messages()
lock.protocol(args, assign).run(msgs)
except (ex.LoadingException, ex.SerializeException) as e:
log.warning('Assignment could not instantiate', exc_info=True)
print('Error: ' + str(e).strip())
exit(1)
except (KeyboardInterrupt, EOFError):
log.info('Quitting...')
else:
assign.dump_tests()
def __init__(self, filepath=None, cmd_args=None, debug=False):
ok_logger = logging.getLogger('client') # Get top-level ok logger
ok_logger.setLevel(logging.DEBUG if debug else logging.ERROR)
self.assignment = load_assignment(filepath, cmd_args)
# Attempt a login with enviornment based tokens
login_with_env(self.assignment)
def main():
"""Run all relevant aspects of ok.py."""
args = parse_input()
log.setLevel(logging.DEBUG if args.debug else logging.ERROR)
log.debug(args)
# Checking user's Python bit version
bit_v = (8 * struct.calcsize("P"))
log.debug("Python {} ({}bit)".format(sys.version, bit_v))
if args.version:
print("okpy=={}".format(client.__version__))
exit(0)
elif args.update:
print("Current version: {}".format(client.__version__))
did_update = software_update.check_version(
args.server, client.__version__, client.FILE_NAME, timeout=10)
exit(not did_update) # exit with error if ok failed to update
assign = None
try:
if args.get_token:
access_token = auth.authenticate(args, force=True)
print("Token: {}".format(access_token))
exit(not access_token) # exit with error if no access_token
# Instantiating assignment
assign = assignment.load_assignment(args.config, args)
if args.authenticate:
# Authenticate and check for success
if not assign.authenticate(force=True):
exit(1)
if args.tests:
print('Available tests:')
for name in assign.test_map:
print(' ' + name)
exit(0)
msgs = messages.Messages()
for name, proto in assign.protocol_map.items():
log.info('Execute {}.run()'.format(name))
proto.run(msgs)
msgs['timestamp'] = str(datetime.now())
except ex.LoadingException as e:
log.warning('Assignment could not load', exc_info=True)
print('Error loading assignment: ' + str(e))
except ex.AuthenticationException as e:
log.warning('Authentication exception occurred', exc_info=True)
print('Authentication error: {0}'.format(e))
except ex.OkException as e:
log.warning('General OK exception occurred', exc_info=True)
print('Error: ' + str(e))
except KeyboardInterrupt:
log.info('KeyboardInterrupt received.')
finally:
if not args.no_update and not args.local:
try:
software_update.check_version(args.server, client.__version__,
client.FILE_NAME)
except KeyboardInterrupt:
pass
if assign:
assign.dump_tests()
def main():
"""Run all relevant aspects of ok.py."""
args = parse_input()
log.setLevel(logging.DEBUG if args.debug else logging.ERROR)
# Checking user's Python bit version
bit_v = (8 * struct.calcsize("P"))
log.debug("Python bit version: {}".format(bit_v))
log.debug(args)
if args.version:
print("okpy=={}".format(client.__version__))
exit(0)
elif args.update:
print("Current version: {}".format(client.__version__))
did_update = software_update.check_version(
args.server, client.__version__, client.FILE_NAME, timeout=10)
exit(not did_update) # exit with error if ok failed to update
assign = None
try:
if args.authenticate:
auth.authenticate(True)
# Instantiating assignment
assign = assignment.load_assignment(args.config, args)
if args.tests:
print('Available tests:')
for name in assign.test_map:
print(' ' + name)
exit(0)
msgs = messages.Messages()
for name, proto in assign.protocol_map.items():
log.info('Execute {}.run()'.format(name))
proto.run(msgs)
msgs['timestamp'] = str(datetime.now())
except ex.LoadingException as e:
log.warning('Assignment could not load', exc_info=True)
print('Error loading assignment: ' + str(e))
except ex.AuthenticationException as e:
log.warning('Authentication exception occurred', exc_info=True)
print('Authentication error: {0}'.format(e))
except ex.OkException as e:
log.warning('General OK exception occurred', exc_info=True)
print('Error: ' + str(e))
except KeyboardInterrupt:
log.info('KeyboardInterrupt received.')
finally:
if not args.no_update:
try:
software_update.check_version(args.server, client.__version__,
client.FILE_NAME)
except KeyboardInterrupt:
pass
if assign:
assign.dump_tests()
def testFailOnNonexistentConfig(self):
self.mock_is_file.return_value = False
with self.assertRaises(ex.LoadingException) as cm:
assignment.load_assignment('does_not_exist.ok')
self.assertEquals('Could not find config file: does_not_exist.ok',
str(cm.exception))
def main():
"""Run all relevant aspects of ok.py."""
args = parse_input()
log.setLevel(logging.DEBUG if args.debug else logging.ERROR)
log.debug(args)
# Checking user's Python bit version
bit_v = (8 * struct.calcsize("P"))
log.debug("Python {} ({}bit)".format(sys.version, bit_v))
if args.version:
print("okpy=={}".format(client.__version__))
exit(0)
elif args.update:
print("Current version: {}".format(client.__version__))
did_update = software_update.check_version(args.server,
client.__version__,
client.FILE_NAME,
timeout=10)
exit(not did_update) # exit with error if ok failed to update
assign = None
try:
if args.get_token:
if args.nointeract:
print_error(
"Cannot pass in --get-token and --nointeract, the only way to get a token is by interaction"
)
exit(1)
access_token = auth.authenticate(args, force=True)
print("Token: {}".format(access_token))
exit(not access_token) # exit with error if no access_token
# Instantiating assignment
assign = assignment.load_assignment(args.config, args)
if assign.decryption_keypage:
# do not allow running locally if decryption keypage is provided
args.local = False
if args.autobackup_actual_run_sync:
assign.autobackup(run_sync=True)
# do not dump tests back out, this overwrites any changes that may have been made
assign = None
exit(0)
if args.generate_encryption_key:
assign.generate_encryption_key(args.generate_encryption_key)
exit(0)
if args.encrypt:
assign.encrypt(args.encrypt, args.encrypt_padding)
# do not dump tests back out, this overwrites any changes that may have been made
assign = None
exit(0)
if args.decrypt is not None:
raise ex.ForceDecryptionException(args.decrypt)
if args.tests:
print('Available tests:')
for name in assign.test_map:
print(' ' + name)
exit(0)
if args.autobackup:
assign.autobackup(run_sync=False)
exit(0)
force_authenticate = args.authenticate
retry = True
while retry:
retry = False
if force_authenticate:
if args.nointeract:
print_error(
"Cannot pass in --authenticate and --nointeract")
exit(1)
# Authenticate and check for success
if not assign.authenticate(force=True):
exit(1)
try:
msgs = messages.Messages()
for name, proto in assign.protocol_map.items():
log.info('Execute {}.run()'.format(name))
proto.run(msgs)
msgs['timestamp'] = str(datetime.now())
except ex.AuthenticationException as e:
if not force_authenticate:
force_authenticate = True
retry = True
elif not args.no_browser:
args.no_browser = True
retry = True
if retry:
msg = "without a browser" if args.no_browser else "with a browser"
log.warning(
'Authentication exception occurred; will retry {0}'.
format(msg),
exc_info=True)
print_error(
'Authentication error; will try to re-authenticate {0}...'
.format(msg))
else:
raise # outer handler will be called
except ex.ForceDecryptionException as e:
assign.decrypt(e.keys)
# begin an autobackup
assign.autobackup(run_sync=False)
# do not dump tests back out, this could overwrite any changes that may have been made
assign = None
exit(0)
except ex.LoadingException as e:
log.warning('Assignment could not load', exc_info=True)
print_error('Error loading assignment: ' + str(e))
except ex.AuthenticationException as e:
log.warning('Authentication exception occurred', exc_info=True)
print_error('Authentication error: {0}'.format(e))
except ex.EarlyExit as e:
log.warning('OK exited early (non-error)')
print_error(str(e))
except ex.OkException as e:
log.warning('General OK exception occurred', exc_info=True)
print_error('Error: ' + str(e))
except KeyboardInterrupt:
log.info('KeyboardInterrupt received.')
finally:
if not args.no_update and not args.local:
try:
software_update.check_version(args.server, client.__version__,
client.FILE_NAME)
except KeyboardInterrupt:
pass
if assign:
assign.dump_tests()
def __init__(self, filepath=None, cmd_args=None, debug=False):
ok_logger = logging.getLogger('client') # Get top-level ok logger
ok_logger.setLevel(logging.DEBUG if debug else logging.ERROR)
self.assignment = load_assignment(filepath, cmd_args)
