def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = self.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main)
# HACK for NECKLACE: one-time hack to support a task with very low memory limit
if job.memory_limit == 3 and job.language == "Java / JDK":
jvm_args = ["-Deval=true", "-Xmx4224k", "-Xss256k", "-XX:MaxMetaspaceSize=8704k"]
commands = language.get_evaluation_commands(
executable_filename, main=main, jvm_args=jvm_args)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {
self._actual_input: job.input
}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.get_root_path())
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Special handling: if there's a batchmanager, then this is really an
# interactive task to be evaluated in a single sandbox.
# Do NOT use check_manager_present() here, as it will raise an error
# for normal tasks with no batchmanager.
if Batch.MANAGER_CODENAME in job.managers:
sandbox.create_file_from_storage(Batch.MANAGER_CODENAME,
job.managers[Batch.MANAGER_CODENAME].digest, executable=True)
# If there is a batchmanagermanager, run the last command with it.
commands[-1][:0] = ["./%s" % Batch.MANAGER_CODENAME,
self.input_filename, self.output_filename]
# Actually performs the execution
# HACK for NECKLACE: one-time hack to support a task with very low memory limit
if job.memory_limit == 3 and job.language == "Java / JDK":
memory_limit = 20
elif job.memory_limit == 3 and job.language == "Python 3 / CPython":
memory_limit = 8
elif job.memory_limit == 3 and job.language == "C++11 / g++":
memory_limit = 4
elif job.memory_limit == 3 and job.language == "C11 / gcc":
memory_limit = 4
else:
memory_limit = job.memory_limit
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.effective_time_limit(),
memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
self._actual_output]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
self.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = Batch.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(executable_filename,
main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {self._actual_input: job.input}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.path)
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [
N_("Evaluation didn't produce file %s"),
self._actual_output
]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher,
job,
Batch.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job.success)
def evaluate(self, job, file_cacher):
"""See TaskType.evaluate."""
if not check_executables_number(job, 1):
return
# Prepare the execution
executable_filename = next(iterkeys(job.executables))
language = get_language(job.language)
main = self.GRADER_BASENAME \
if self._uses_grader() else executable_filename
commands = language.get_evaluation_commands(
executable_filename, main=main)
executables_to_get = {
executable_filename: job.executables[executable_filename].digest
}
files_to_get = {
self._actual_input: job.input
}
# Check which redirect we need to perform, and in case we don't
# manage the output via redirect, the submission needs to be able
# to write on it.
files_allowing_write = []
stdin_redirect = None
stdout_redirect = None
if len(self.input_filename) == 0:
stdin_redirect = self._actual_input
if len(self.output_filename) == 0:
stdout_redirect = self._actual_output
else:
files_allowing_write.append(self._actual_output)
# Create the sandbox
sandbox = create_sandbox(file_cacher, name="evaluate")
job.sandboxes.append(sandbox.get_root_path())
# Put the required files into the sandbox
for filename, digest in iteritems(executables_to_get):
sandbox.create_file_from_storage(filename, digest, executable=True)
for filename, digest in iteritems(files_to_get):
sandbox.create_file_from_storage(filename, digest)
# Actually performs the execution
box_success, evaluation_success, stats = evaluation_step(
sandbox,
commands,
job.time_limit,
job.memory_limit,
writable_files=files_allowing_write,
stdin_redirect=stdin_redirect,
stdout_redirect=stdout_redirect,
multiprocess=job.multithreaded_sandbox)
outcome = None
text = None
# Error in the sandbox: nothing to do!
if not box_success:
pass
# Contestant's error: the marks won't be good
elif not evaluation_success:
outcome = 0.0
text = human_evaluation_message(stats)
if job.get_output:
job.user_output = None
# Otherwise, advance to checking the solution
else:
# Check that the output file was created
if not sandbox.file_exists(self._actual_output):
outcome = 0.0
text = [N_("Evaluation didn't produce file %s"),
self._actual_output]
if job.get_output:
job.user_output = None
else:
# If asked so, put the output file into the storage.
if job.get_output:
job.user_output = sandbox.get_file_to_storage(
self._actual_output,
"Output file in job %s" % job.info,
trunc_len=100 * 1024)
# If just asked to execute, fill text and set dummy outcome.
if job.only_execution:
outcome = 0.0
text = [N_("Execution completed successfully")]
# Otherwise evaluate the output file.
else:
box_success, outcome, text = eval_output(
file_cacher, job,
self.CHECKER_CODENAME
if self._uses_checker() else None,
user_output_path=sandbox.relative_path(
self._actual_output),
user_output_filename=self.output_filename)
# Fill in the job with the results.
job.success = box_success
job.outcome = str(outcome) if outcome is not None else None
job.text = text
job.plus = stats
delete_sandbox(sandbox, job.success)
