def _load_test(file, module, name, assign):
namespace = module
for attr in name.split('.'):
if not hasattr(namespace, attr):
raise ex.LoadingException('Module {} has no attribute {}'.format(
module.__name__, name))
namespace = getattr(namespace, attr)
func = namespace
if not callable(func):
raise ex.LoadingException(
'Attribute {} is not a function'.format(name))
docstring = func.__doc__ if func.__doc__ else ''
try:
return models.Doctest(file,
assign.cmd_args.verbose,
assign.cmd_args.interactive,
assign.cmd_args.timeout,
name=name,
points=1,
docstring=docstring)
except ex.SerializeException:
raise ex.LoadingException('Unable to load doctest for {} '
'from {}'.format(name, file))
def load(file, _, assign):
"""Loads Scheme tests from a specified filepath.
PARAMETERS:
file -- str; a filepath to a Scheme file.
RETURNS:
Test
"""
if not os.path.isfile(file) or not file.endswith('.scm'):
raise ex.LoadingException(
'Cannot run Scheme tests from {}'.format(file))
with open(file, 'r') as f:
file_contents = f.read()
try:
return {
file:
models.SchemeTest(file,
file_contents,
assign.cmd_args.timeout,
name=file,
points=1)
}
except ex.SerializeException:
raise ex.LoadingException('Unable to load Scheme test '
'from {}'.format(file))
def load(file, parameter, assign):
"""Loads an OK-style test from a specified filepath.
PARAMETERS:
file -- str; a filepath to a Python module containing OK-style
tests.
RETURNS:
Test
"""
filename, ext = os.path.splitext(file)
if not os.path.isfile(file) or ext != '.py':
log.info('Cannot import {} as an OK test'.format(file))
raise ex.LoadingException(
'Cannot import {} as an OK test'.format(file))
try:
test = importing.load_module(file).test
test = copy.deepcopy(test)
except Exception as e:
raise ex.LoadingException('Error importing file {}: {}'.format(
file, str(e)))
name = os.path.basename(filename)
try:
return {
name:
models.OkTest(file, SUITES, assign.endpoint, assign,
assign.cmd_args.verbose, assign.cmd_args.interactive,
assign.cmd_args.timeout, **test)
}
except ex.SerializeException:
raise ex.LoadingException('Cannot load OK test {}'.format(file))
def get_config(config):
if config is None:
configs = glob.glob(CONFIG_EXTENSION)
if len(configs) > 1:
raise ex.LoadingException(textwrap.dedent("""
Multiple .ok files found:
{}
Please specify a particular assignment's config file with
python3 ok --config <config file>
""".format(' '.join(configs))))
elif not configs:
raise ex.LoadingException('No .ok configuration file found')
config = configs[0]
try:
with open(config, 'r') as f:
result = json.load(f, object_pairs_hook=collections.OrderedDict)
except IOError:
raise ex.LoadingException('Error loading config: {}'.format(config))
except ValueError:
raise ex.LoadingException(textwrap.dedent("""
{0} is a malformed .ok configuration file. Please re-download {0}.
""".format(config)))
else:
log.info('Loaded config from {}'.format(config))
return result
def _load_tests(self):
"""Loads all tests specified by test_map."""
log.info('Loading tests')
for file_pattern, source in self.tests.items():
# Separate filepath and parameter
if ':' in file_pattern:
file_pattern, parameter = file_pattern.split(':', 1)
else:
parameter = ''
for file in sorted(glob.glob(file_pattern)):
try:
module = importlib.import_module(self._TESTS_PACKAGE +
'.' + source)
except ImportError:
raise ex.LoadingException(
'Invalid test source: {}'.format(source))
test_name = file
if parameter:
test_name += ':' + parameter
self.test_map.update(module.load(file, parameter, self))
log.info('Loaded {}'.format(test_name))
if not self.test_map:
raise ex.LoadingException('No tests loaded')
def _get_config(config):
if config is None:
configs = glob.glob(CONFIG_EXTENSION)
if len(configs) > 1:
raise ex.LoadingException('\n'.join([
'Multiple .ok files found:', ' ' + ' '.join(configs),
"Please specify a particular assignment's config file with",
' python3 ok --config <config file>'
]))
elif not configs:
raise ex.LoadingException('No .ok configuration file found')
config = configs[0]
elif not os.path.isfile(config):
raise ex.LoadingException(
'Could not find config file: {}'.format(config))
try:
with open(config, 'r') as f:
result = json.load(f, object_pairs_hook=collections.OrderedDict)
except IOError:
raise ex.LoadingException('Error loading config: {}'.format(config))
except ValueError:
raise ex.LoadingException('{0} is a malformed .ok configuration file. '
'Please re-download {0}.'.format(config))
else:
log.info('Loaded config from {}'.format(config))
return result
def _resolve_specified_tests(self, questions, all_tests=False):
"""For each of the questions specified on the command line,
find the test corresponding that question.
Questions are preserved in the order that they are specified on the
command line. If no questions are specified, use the entire set of
tests.
"""
if not questions and not all_tests \
and self.default_tests != core.NoValue \
and len(self.default_tests) > 0:
log.info('Using default tests (no questions specified): '
'{}'.format(self.default_tests))
bad_tests = sorted(test for test in self.default_tests if test not in self.test_map)
if bad_tests:
error_message = ("Required question(s) missing: {}. "
"This often is the result of accidentally deleting the question's doctests or the entire function.")
raise ex.LoadingException(error_message.format(", ".join(bad_tests)))
return [self.test_map[test] for test in self.default_tests]
elif not questions:
log.info('Using all tests (no questions specified and no default tests)')
return list(self.test_map.values())
elif not self.test_map:
log.info('No tests loaded')
return []
specified_tests = []
for question in questions:
if question not in self.test_map:
raise ex.InvalidTestInQuestionListException(list(self.test_map), question)
log.info('Adding {} to specified tests'.format(question))
if question not in specified_tests:
specified_tests.append(self.test_map[question])
return specified_tests
def load_assignment(filepath=None, cmd_args=None):
config = _get_config(filepath)
if not isinstance(config, dict):
raise ex.LoadingException('Config should be a dictionary')
if cmd_args is None:
cmd_args = _MockNamespace()
return Assignment(cmd_args, **config)
def _resolve_specified_tests(self):
"""For each of the questions specified on the command line,
find the test corresponding that question.
Questions are preserved in the order that they are specified on the
command line. If no questions are specified, use the entire set of
tests.
"""
if not self.cmd_args.question and not self.cmd_args.all \
and self.default_tests != core.NoValue \
and len(self.default_tests) > 0:
log.info('Using default tests (no questions specified): '
'{}'.format(self.default_tests))
self.specified_tests = [self.test_map[test]
for test in self.default_tests]
return
elif not self.cmd_args.question:
log.info('Using all tests (no questions specified and no default tests)')
self.specified_tests = list(self.test_map.values())
return
elif not self.test_map:
log.info('No tests loaded')
return
for question in self.cmd_args.question:
if question not in self.test_map:
print('Test "{}" not found.'.format(question))
print('Did you mean one of the following? '
'(Names are case sensitive)')
for test in self.test_map:
print(' {}'.format(test))
raise ex.LoadingException('Invalid test specified: {}'.format(question))
log.info('Adding {} to specified tests'.format(question))
if question not in self.specified_tests:
self.specified_tests.append(self.test_map[question])
def _load_protocols(self):
log.info('Loading protocols')
for proto in self.protocols:
try:
module = importlib.import_module(self._PROTOCOL_PACKAGE + '.' + proto)
except ImportError:
raise ex.LoadingException('Invalid protocol: {}'.format(proto))
self.protocol_map[proto] = module.protocol(self.cmd_args, self)
log.info('Loaded protocol "{}"'.format(proto))
def _resolve_specified_tests(self):
"""For each of the questions specified on the command line,
find the best test corresponding that question.
The best match is found by finding the test filepath that has the
smallest edit distance with the specified question.
Questions are preserved in the order that they are specified on the
command line. If no questions are specified, use the entire set of
tests.
"""
if not self.cmd_args.question:
log.info('Using all tests (no questions specified)')
self.specified_tests = list(self.test_map.values())
return
elif not self.test_map:
log.info('No tests loaded')
return
for question in self.cmd_args.question:
matches = []
for test in self.test_map:
if _has_subsequence(test.lower(), question.lower()):
matches.append(test)
if len(matches) > 1:
print('Did you mean one of the following?')
for test in matches:
print(' {}'.format(test))
raise ex.LoadingException(
'Ambiguous test specified: {}'.format(question))
elif not matches:
print('Did you mean one of the following?')
for test in self.test_map:
print(' {}'.format(test))
raise ex.LoadingException(
'Invalid test specified: {}'.format(question))
match = matches[0]
log.info('Matched {} to {}'.format(question, match))
if match not in self.specified_tests:
self.specified_tests.append(self.test_map[match])
def load(file, parameter, assign):
"""Loads an OK-style test from a specified filepath.
PARAMETERS:
file -- str; a filepath to a Python module containing OK-style
tests.
RETURNS:
Test
"""
filename, ext = os.path.splitext(file)
if not os.path.isfile(file) or ext != '.py':
log.info('Cannot import {} as an OK test'.format(file))
raise ex.LoadingException(
'Cannot import {} as an OK test'.format(file))
if os.path.exists(file):
with open(file) as f:
data = f.read()
if encryption.is_encrypted(data):
decrypted, _ = assign.attempt_decryption([])
if file not in decrypted:
name = os.path.basename(filename)
return {name: models.EncryptedOKTest(name=name, points=1)}
try:
test = importing.load_module(file).test
test = copy.deepcopy(test)
except Exception as e:
raise ex.LoadingException('Error importing file {}: {}'.format(
file, str(e)))
name = os.path.basename(filename)
try:
return {
name:
models.OkTest(file, SUITES, assign.endpoint, assign,
assign.cmd_args.verbose, assign.cmd_args.interactive,
assign.cmd_args.timeout, **test)
}
except ex.SerializeException as e:
raise ex.LoadingException('Cannot load OK test {}: {}'.format(file, e))
def _load_parsons(self):
"""Verifies that all desired parsons problems exist and that the
structure of parsons is proper.
"""
if self.parsons is core.NoValue:
return
log.info('Loading parsons problems')
for prob_group_name, v in self.parsons.items():
req_probs = v.get('required', [])
opt_probs = v.get('optional', [])
if 'required' not in v and 'optional' not in v:
error_message = 'Need a "required" and/or "optional" key in a parsons config object'
raise ex.LoadingException(error_message)
if not isinstance(req_probs, list) or not isinstance(opt_probs, list):
error_message = 'The "required" and "optional" keys, if included in a parsons config object, must be lists'
raise ex.LoadingException(error_message)
for prob in (req_probs + opt_probs):
if prob not in self.test_map:
error_message = f'Problem name "{prob}" in the parsons problem group "{prob_group_name}" is invalid'
raise ex.LoadingException(error_message)
def load(file, name, args):
"""Loads doctests from a specified filepath.
PARAMETERS:
file -- str; a filepath to a Python module containing OK-style
tests.
RETURNS:
Test
"""
if not os.path.isfile(file) or not file.endswith('.py'):
log.info('Cannot import doctests from {}'.format(file))
raise ex.LoadingException('Cannot import doctests from {}'.format(file))
try:
module = importing.load_module(file)
except Exception:
# Assume that part of the traceback includes frames from importlib.
# Begin printing the traceback after the last line involving importlib.
# TODO(albert): Try to find a cleaner way to do this. Also, might want
# to move this to a more general place.
print('Traceback (most recent call last):')
stacktrace = traceback.format_exc().split('\n')
start = 0
for i, line in enumerate(stacktrace):
if 'importlib' in line:
start = i + 1
print('\n'.join(stacktrace[start:]))
raise ex.LoadingException('Error importing file {}'.format(file))
if not hasattr(module, name):
raise ex.LoadingException('Module {} has no function {}'.format(
module.__name__, name))
func = getattr(module, name)
if not callable(func):
raise ex.LoadingException('Attribute {} is not a function'.format(name))
return models.Doctest(file, args.verbose, args.interactive, args.timeout,
name=name, points=1, docstring=func.__doc__)
def _load_tests(self):
"""Loads all tests specified by test_map.
PARAMETERS:
test_map -- dict; file pattern -> serialize module. Every file that
that matches the UNIX style file pattern will be loaded
by the module.load method.
"""
log.info('Loading tests')
for file_pattern, source in self.tests.items():
# Separate filepath and parameter
if ':' in file_pattern:
file_pattern, parameter = file_pattern.split(':', maxsplit=1)
else:
parameter = ''
files = glob.glob(file_pattern)
if not files:
error_msg = 'No tests found for pattern: {}'.format(
file_pattern)
print(error_msg)
raise ex.LoadingException(error_msg)
for file in files:
try:
module = importlib.import_module(self._TESTS_PACKAGE +
'.' + source)
except ImportError:
raise ex.LoadingException(
'Invalid test source: {}'.format(source))
test_name = file
if parameter:
test_name += ':' + parameter
self.test_map[test_name] = module.load(file, parameter,
self.cmd_args)
log.info('Loaded {}'.format(test_name))
def load(file, name, assign):
"""Loads doctests from a specified filepath.
PARAMETERS:
file -- str; a filepath to a Python module containing OK-style
tests.
name -- str; optional parameter that specifies a particular function in
the file. If omitted, all doctests will be included.
RETURNS:
Test
"""
if not os.path.isfile(file) or not file.endswith('.py'):
raise ex.LoadingException(
'Cannot import doctests from {}'.format(file))
try:
module = importing.load_module(file)
except Exception:
# Assume that part of the traceback includes frames from importlib.
# Begin printing the traceback after the last line involving importlib.
# TODO(albert): Try to find a cleaner way to do this. Also, might want
# to move this to a more general place.
print('Traceback (most recent call last):')
stacktrace = traceback.format_exc().split('\n')
start = 0
for i, line in enumerate(stacktrace):
if 'importlib' in line:
start = i + 1
print('\n'.join(stacktrace[start:]))
raise ex.LoadingException('Error importing file {}'.format(file))
if name:
return {name: _load_test(file, module, name, assign)}
else:
return _load_tests(file, module, assign)
def load_config(filepath, args):
config = get_config(filepath)
log.info('Loaded config from {}'.format(filepath))
if not isinstance(config, dict):
raise ex.LoadingException('Config should be a dictionary')
return Assignment(args, **config)
def load_config(filepath, args):
config = get_config(filepath)
if not isinstance(config, dict):
raise ex.LoadingException('Config should be a dictionary')
return Assignment(args, **config)
