def expand_tests_from_program(question, tests: IoSpec, language=None):
"""
Uses source code from source code reference in the provided language
to expand tests.
"""
language = get_programming_language(language)
answer_key = question.answers.get(language=language)
if not answer_key.source:
raise ValueError('cannot expand from %s: no program set' % language)
source = answer_key.source
language_ref = language.ejudge_ref()
if tests.is_simple:
return tests.copy()
if tests.is_standard_test_case:
tests = tests.copy()
tests.expand_inputs()
if tests.is_simple:
return tests
return question.run_code(source, tests, language)
def expand_tests_from_program(question, tests: IoSpec, language=None):
"""
Uses source code from source code reference in the provided language
to expand tests.
"""
language = get_programming_language(language)
answer_key = question.answers.get(language=language)
if not answer_key.source:
raise ValueError('cannot expand from %s: no program set' % language)
source = answer_key.source
language_ref = language.ejudge_ref()
if tests.is_simple:
return tests.copy()
if tests.is_standard_test_case:
tests = tests.copy()
tests.expand_inputs()
if tests.is_simple:
return tests
return question.run_code(source, tests, language)
def expand_tests(question, tests: IoSpec) -> IoSpec:
"""
Expand tests and return a new expanded IoSpec object.
"""
if tests.is_simple:
return tests.copy()
# Check if result is usable after a simple expansion
tests = tests.copy()
tests.expand_inputs()
if tests.is_simple:
return tests
# Further expansion requires a reference program to automatically
# compute all the inputs and outputs
qs = question.answers_with_code()
if qs:
language = qs.first().language
else:
raise ValidationError(_(
'No program was provided to expand the given test cases.'
))
return expand_tests_from_program(question, tests, language)
def expand_tests(question, tests: IoSpec) -> IoSpec:
"""
Expand tests and return a new expanded IoSpec object.
"""
if tests.is_simple:
return tests.copy()
# Check if result is usable after a simple expansion
tests = tests.copy()
tests.expand_inputs()
if tests.is_simple:
return tests
# Further expansion requires a reference program to automatically
# compute all the inputs and outputs
qs = question.answers_with_code()
if qs:
language = qs.first().language
else:
raise ValidationError(_(
'No program was provided to expand the given test cases.'
))
return expand_tests_from_program(question, tests, language)
def grade_from_lang(lang, source, iospec, **kwargs):
"""A version of grade_from_manager() in which both the inputs and ouputs
can be converted to JSON."""
manager = manager_from_lang(lang, source)
iospec = IoSpec.from_json(iospec)
result = grade_from_manager(manager, iospec, **kwargs)
return result.to_json()
def grade_from_lang(lang, source, iospec, **kwargs):
"""A version of grade_from_manager() in which both the inputs and ouputs
can be converted to JSON."""
manager = manager_from_lang(lang, source)
iospec = IoSpec.from_json(iospec)
result = grade_from_manager(manager, iospec, **kwargs)
return result.to_json()
def post_tests(self):
try:
return self._post_tests
except AttributeError:
if self.post_tests_source:
post_tests = parse_iospec(self.post_tests_source)
else:
post_tests = IoSpec()
self._post_tests = ejudge.combine_iospec(self.pre_tests,
post_tests)
return self._post_tests
def run_from_manager(manager, inputs, *, raises, timeout):
"""Run command from given manager."""
try:
manager.build()
except BuildError as ex:
if raises:
raise
return IoSpec([ErrorTestCase.build(error_message=str(ex))])
data = []
for x in inputs:
try:
data.append(manager.run(x))
except Exception as ex:
if raises:
raise
data.append(error_test_case(ex, ex.__traceback__))
result = IoSpec(data)
result.set_meta('lang', manager.name)
result.set_meta('buildargs', manager.buildargs)
result.set_meta('shellargs', manager.shellargs)
return result
def run_from_manager(manager, inputs, *, raises, timeout):
"""Run command from given manager."""
try:
manager.build()
except BuildError as ex:
if raises:
raise
return IoSpec([ErrorTestCase.build(error_message=str(ex))])
data = []
for x in inputs:
try:
data.append(manager.run(x))
except Exception as ex:
if raises:
raise
data.append(error_test_case(ex, ex.__traceback__))
result = IoSpec(data)
result.setmeta('lang', manager.name)
result.setmeta('buildargs', manager.buildargs)
result.setmeta('shellargs', manager.shellargs)
return result
def post_tests(self, value):
pre_tests = self.pre_tests
value = IoSpec([test for test in value if test not in pre_tests])
self._post_tests = ejudge.combine_iospec(self.pre_tests, value)
self.post_tests_source = value.source()
def run(source,
inputs,
lang=None,
*,
timeout=None,
raises=False,
path=None,
sandbox=True):
"""Run program with the given list of inputs and returns the
corresponding :cls:`iospec.IoSpec` instance.
Parameters
----------
source : str or file
The source code for the test program
inputs : sequence
A sequence of input strings. If input is a sequence of sequences,
this function will perform multiple test cases. It can also be a IoSpec
or a TestCase instance which are used to extract the necessary input
strings.
lang : str
The name for the source code language. See
:func:`ejudge.graders.io.grade` for more details.
timeout : float
A time limit for the entire run (in seconds). If this attribute is not
given, the program will run without any timeout. This can be potentially
dangerous if the input program has an infinite loop.
sandbox : bool
Controls if code is run in sandboxed mode or not. Sandbox protection
is the default behavior on supported platforms.
raises : bool
If True, raise a BuildError if the build process fails. The default
behavior is to return a IoSpec instance with a single ErrorTestCase with
a type string of 'error-build'.
path : str
The absolute file path for the input string or file object.
Returns
-------
A :cls:`iospec.IoSpec` structure. If ``inputs`` is a sequence of strings,
the resulting tree will have a single test case.
"""
manager = get_manager(lang, source, path)
manager.is_sandboxed = sandbox
# Normalize inputs
if isinstance(inputs, (IoSpec, TestCase)):
inputs = inputs.inputs()
else:
if isinstance(inputs[0], str):
inputs = [list(inputs)]
else:
inputs = [list(x) for x in inputs]
# Execute
with manager.keep_cwd():
if sandbox:
imports = manager.modules()
result = run_sandbox(
run_from_lang,
args=(manager.lang, manager.source, inputs),
kwargs={
'raises': raises,
'timeout': timeout
},
imports=imports,
)
result = IoSpec.from_json(result)
else:
result = run_from_manager(manager,
inputs,
raises=raises,
timeout=timeout)
return result
def run_worker(source, inputs, lang=None, *,
fast=False, timeout=None, raises=False, path=None, sandbox=True,
compare_streams=False, is_sandboxed=False, fake_sandbox=False,
debug=False):
# Normalize inputs
if isinstance(inputs, (IoSpec, TestCase)):
inputs = inputs.inputs()
else:
if inputs and isinstance(inputs[0], str):
inputs = [list(inputs)]
else:
inputs = [list(map(str, x)) for x in inputs]
# Validate params
if timeout is not None and timeout <= 0:
raise ValueError('timeout must be positive, got: %s' % timeout)
if sandbox and is_sandboxed:
raise ValueError('cannot set sandbox = is_sandboxed = True')
# Create build manager
build_manager = registry.build_manager_from_path(
lang, source, path,
is_sandboxed=is_sandboxed,
compare_streams=compare_streams,
)
# Run in sandboxed mode
if sandbox:
logger.debug('executing %s program inside sandbox' % lang)
imports = build_manager.get_modules()
args = (source, inputs, lang)
kwargs = {
'raises': raises,
'timeout': timeout,
'fast': fast,
'path': path,
'sandbox': False,
'compare_streams': compare_streams,
'is_sandboxed': True,
}
if fake_sandbox:
result, messages = run_worker(*args, **kwargs)
else:
try:
with capture_print() as data:
result, messages = run_sandbox(
run_worker,
args=args,
kwargs=kwargs,
imports=imports,
print_messages=True,
)
except Exception:
print(data.read(), file=sys.stderr)
raise
for (level, message) in messages:
getattr(logger, level)(message)
return IoSpec.from_json(result), []
# Prepare build manager
try:
build_manager.build()
except BuildError as ex:
if raises:
raise
result = IoSpec([ErrorTestCase.build(error_message=str(ex))])
if is_sandboxed:
return result.to_json(), []
else:
return result, []
# Run all examples with the execution manager
data = []
language = build_manager.language
for input_strings in inputs:
ctrl = registry.execution_manager(language, build_manager,
input_strings)
result = ctrl.run(timeout)
assert isinstance(result, TestCase)
data.append(result)
if fast and result.is_error_test_case:
break
build_manager.log('info', 'executed all %s testcases in %s sec' %
(len(inputs), build_manager.execution_duration))
# Prepare resulting iospec object
result = IoSpec(data)
result.set_meta('lang', build_manager.language)
if is_sandboxed:
return result.to_json(), build_manager.messages
else:
return result, []
def run(source, inputs, lang=None, *,
timeout=None, raises=False, path=None, sandbox=True):
"""Run program with the given list of inputs and returns the
corresponding :cls:`iospec.IoSpec` instance.
Parameters
----------
source : str or file
The source code for the test program
inputs : sequence
A sequence of input strings. If input is a sequence of sequences,
this function will perform multiple test cases. It can also be a IoSpec
or a TestCase instance which are used to extract the necessary input
strings.
lang : str
The name for the source code language. See
:func:`ejudge.graders.io.grade` for more details.
timeout : float
A time limit for the entire run (in seconds). If this attribute is not
given, the program will run without any timeout. This can be potentially
dangerous if the input program has an infinite loop.
sandbox : bool
Controls if code is run in sandboxed mode or not. Sandbox protection
is the default behavior on supported platforms.
raises : bool
If True, raise a BuildError if the build process fails. The default
behavior is to return a IoSpec instance with a single ErrorTestCase with
a type string of 'error-build'.
path : str
The absolute file path for the input string or file object.
Returns
-------
A :cls:`iospec.IoSpec` structure. If ``inputs`` is a sequence of strings,
the resulting tree will have a single test case.
"""
manager = get_manager(lang, source, path)
manager.is_sandboxed = sandbox
# Normalize inputs
if isinstance(inputs, (IoSpec, TestCase)):
inputs = inputs.inputs()
else:
if isinstance(inputs[0], str):
inputs = [list(inputs)]
else:
inputs = [list(x) for x in inputs]
# Execute
with manager.keep_cwd():
if sandbox:
imports = manager.modules()
result = run_sandbox(
run_from_lang,
args=(manager.lang, manager.source, inputs),
kwargs={'raises': raises, 'timeout': timeout},
imports=imports,
)
result = IoSpec.from_json(result)
else:
result = run_from_manager(
manager,
inputs,
raises=raises,
timeout=timeout
)
return result
def post_tests(self, value):
pre_tests = self.pre_tests
value = IoSpec([test for test in value if test not in pre_tests])
self._post_tests = ejudge.combine_iospec(self.pre_tests, value)
self.post_tests_source = value.source()