def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def __init__(self, program, assignment_result):
self.assignment_result = assignment_result
self.program = program
# Extracting the list of files relevant to only this section.
if program.program_files:
prgrm_files = get_file_name_list(
name=program.program_files.file.name)
prgrm_files = " ".join(prgrm_files)
else:
prgrm_files = ""
# Adding the list of files in the assignment model solution to the above list.
self.programFiles = program.assignment.student_program_files + " " + prgrm_files
self.language = program.assignment.program_language
self.compiler_command = get_compilation_command(program)
self.execution_command = get_execution_command(program)
self.commandExecutor = CommandExecutor()
def compile_model_solution(sender, instance, **kwargs):
# If source file was not changed no need to re-compile.
if not hasattr(instance, 'source_changed'):
return
if not kwargs.get('created'):
print "Not created!"
programFilePath = instance.program_files.name
ModelSolutionPath = instance.model_solution.name
# Model Solution was not provided hence do nothing.
if os.path.isdir(ModelSolutionPath):
return
os.chdir(tmpDir)
extractFile(os.path.join(MEDIA_ROOT, ModelSolutionPath), tmpDir)
# Change directory if solution tar contains a directory.
if os.path.isdir(os.listdir('./')[0]):
os.chdir(os.listdir('./')[0])
currentDir = os.getcwd()
extractFile(os.path.join(MEDIA_ROOT, programFilePath), currentDir)
# Setting up compilation and creating CommandExecutor
executor = CommandExecutor(timeout=5)
command = get_compilation_command(instance)
''' To Remove: COMPILER_FLAGS
exeName = "exefile_" + str(instance.id)
compilerFlags = instance.compiler_flags if instance.compiler_flags else ""
command = instance.compiler_name + " " + compilerFlags\
+ " " + instance.file_names_to_compile + " " + " -o " + exeName
'''
compileResult = executor.executeCommand(command=command)
# Update output files on success
if compileResult.returnCode == 0:
instance.UpdateOutput()
cleanUp()
def compile_model_solution(sender, instance, **kwargs):
# If source file was not changed no need to re-compile.
if not hasattr(instance, 'source_changed'):
return
if not kwargs.get('created'):
print "Not created!"
programFilePath = instance.program_files.name
ModelSolutionPath = instance.model_solution.name
# Model Solution was not provided hence do nothing.
if os.path.isdir(ModelSolutionPath):
return
os.chdir(tmpDir)
extractFile(os.path.join(MEDIA_ROOT, ModelSolutionPath), tmpDir)
# Change directory if solution tar contains a directory.
if os.path.isdir(os.listdir('./')[0]):
os.chdir(os.listdir('./')[0])
currentDir = os.getcwd()
extractFile(os.path.join(MEDIA_ROOT, programFilePath), currentDir)
# Setting up compilation and creating CommandExecutor
executor = CommandExecutor(timeout=5)
command = get_compilation_command(instance)
''' To Remove: COMPILER_FLAGS
exeName = "exefile_" + str(instance.id)
compilerFlags = instance.compiler_flags if instance.compiler_flags else ""
command = instance.compiler_name + " " + compilerFlags\
+ " " + instance.file_names_to_compile + " " + " -o " + exeName
'''
compileResult = executor.executeCommand(command=command)
# Update output files on success
if compileResult.returnCode == 0:
instance.UpdateOutput()
cleanUp()
def __init__(self, program, assignment_result):
self.assignment_result = assignment_result
self.program = program
# Extracting the list of files relevant to only this section.
if program.program_files:
prgrm_files = get_file_name_list(name=program.program_files.file.name)
prgrm_files = " ".join(prgrm_files)
else:
prgrm_files = ""
# Adding the list of files in the assignment model solution to the above list.
self.programFiles = program.assignment.student_program_files + " " + prgrm_files
self.language = program.assignment.program_language
self.compiler_command = get_compilation_command(program)
self.execution_command = get_execution_command(program)
self.commandExecutor = CommandExecutor()
class CustomTestcase(object):
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def get_resource_limit(self):
# Return default values because there is no setting for custom testcase.
return {
'cpu_time': 10,
'clock_time': 10,
'memory': 32768,
'stack_size': 8192,
'child_processes': 0,
'open_files': 512,
'file_size': 1024,
}
# Store the input file that student uploaded
def store_input_file(self, inputFile, dest):
file_path = os.path.join(dest, inputFile.name.split('/')[-1])
destination = open(file_path, 'wb+')
# Work-around to support file and InMemoeryUploadedFile objects
if hasattr(inputFile, 'chunks'):
all_file = inputFile.chunks()
else:
all_file = inputFile
# Write everything to the input file represented by destination object. Then return the full path of file name.
for chunk in all_file:
destination.write(chunk)
destination.close()
return file_path
# Function to setup the environment (input file, solution files and submission files) before running the code.
def setup(self):
# This is the directory where the student program is copied to.
self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir)
os.chdir(self.temp_dir)
# Copy student solution files to tmp directory.
extract_or_copy(src=self.submission.filePath.file.name,
dest=self.temp_dir)
# Another temp directory for running solution program. This is the directory where the solution program is copied to.
self.solution_temp_dir = tempfile.mkdtemp(prefix="solution",
dir=self.eval_dir)
directories = [a for a in os.listdir('./') if os.path.isdir(a)]
if directories:
os.chdir(directories[0])
currentDir = os.getcwd()
# Extract the program file and the input file to the student directory, and the input file to the program directory as well.
extract_or_copy(src=self.program.program_files.file.name,
dest=currentDir)
self.submittedFiles = get_file_name_list(
name=self.submission.filePath.file.name)
input_file = self.store_input_file(self.inputFile, currentDir)
shutil.copy(src=input_file, dst=self.solution_temp_dir)
# Function to compile the student program.
def compile(self):
self.exe_file = str(self.program.id)
compiler_command = " ".join(pickle.loads(
self.program.compiler_command)) + " -o " + self.exe_file
self.compileResult = self.commandExecutor.executeCommand(
command=compiler_command)
# Function to run the student and model solution on the input file.
def run(self):
# Run student exe file.
_, self.actual_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.actual_stdout, 0777)
runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name
self.actualOutput = self.commandExecutor.safe_execute(
command=runStudentProgram, limits=self.get_resource_limit())
# Run solution exe file.
shutil.copy(src=self.program.exe_file_name.file.name,
dst=self.solution_temp_dir)
os.chdir(self.solution_temp_dir)
_, self.expected_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.expected_stdout, 0777)
exe = self.program.exe_file_name.name.split('/')[-1]
runSolutionProgram = "./" + exe + " < " + self.inputFile.name
self.expectedOutput = self.commandExecutor.safe_execute(
command=runSolutionProgram, limits=self.get_resource_limit())
self.passed = self.testPassed()
# This function is called after run(), and checks the actual and the expected output.
def testPassed(self):
# At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout.
# compare them and display result on browser.
if not (self.expectedOutput.returnCode == 0
and self.actualOutput.returnCode == 0):
return False
exp_op = iter(self.expectedOutput.get_stdout())
act_op = iter(self.actualOutput.get_stdout())
if len(self.expectedOutput.get_stdout()) > len(
self.actualOutput.get_stdout()):
return False
for line1, line2 in zip(exp_op, act_op):
if line1.strip() != line2.strip():
return False
for line2 in act_op:
if line2.strip() != "":
return False
return True
# This is the function called to get the result once the input file is uploaded.
def getResult(self):
# Setup the environment.
self.setup()
# Compile the program.
self.compile()
# If compilation is successful then run the program.
if self.compileResult.returnCode == 0:
self.run()
# Clean up the temporary directories.
self.cleanUp()
return self
# Function to clean up the temporary directories.
def cleanUp(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
class TestCase(object):
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = [
'cpu_time', 'clock_time', 'memory', 'stack_size',
'child_processes', 'open_files', 'file_size'
]
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {}
# Function to make a tarfile of all output files and return the name of this file.
def make_tar(self):
# Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id"
stdoutFileName = "standardOutput_" + str(self.testcase.id)
shutil.move(self.std_out, stdoutFileName)
outputfiles = [stdoutFileName]
# Set the name of the tarfile which is going to contain the output files.
tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2"
outputFilesTar = tarfile.open(name=tarname, mode="w:bz2")
# Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution
# wrote to other than the std output) to the tarfile.
for afile in outputfiles + self.output_files:
if os.path.isfile(afile):
outputFilesTar.add(afile)
outputFilesTar.close()
return tarname
# Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section
# level itself.
def run(self, exeCommand):
# Create the output file to capture the std output in the current directory.
_, self.std_out = tempfile.mkstemp()
os.chmod(self.std_out, 0777)
command = exeCommand
# Add the command line arguments if any.
if self.testcase.command_line_args:
command = command + self.testcase.command_line_args
# Then redirect the std input to the input file.
if os.path.isfile(self.stdInFile):
command = command + " < " + self.stdInFile
# Execute the resulting command with the appropriate resource limits and the output file created above.
self.testResults = self.commandExecutor.safe_execute(
command=command,
limits=self.get_resource_limit(),
outfile=self.std_out)
# Check if the testcase has passed.
self.passed = self.test_passed()
# Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase.
attrs = {
'error_messages': "".join(self.testResults.stderr),
'return_code': int(self.testResults.returnCode),
'test_passed': self.passed
}
# This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above.
filter_attrs = {
'test_case': self.testcase,
'program_result': self.program_result
}
# If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes.
# Remove the older output file.
# Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary.
try:
obj = TestcaseResult.objects.filter(**filter_attrs)
obj.update(**attrs)
test_result = obj[0:1].get()
default_storage.delete(
test_result.output_files.path) # Remove older file.
except TestcaseResult.DoesNotExist:
attrs.update(filter_attrs)
test_result = TestcaseResult.objects.create(**attrs)
# Make a tar file of the std output file and the other files which have been written to by the student submission.
# Update the test result (updated or newly created) with this tar file as the output file.
tar_name = self.make_tar()
test_result.output_files.save(tar_name, File(open(tar_name)))
return self
# Function to check if the student submission has passed the testcase.
def test_passed(self):
# Create a temporary directory and copy the expected output of the testcase to this file.
temp_dir = tempfile.mkdtemp(prefix="grader_op_files")
extract_or_copy(src=self.testcase.output_files.file.name,
dest=temp_dir)
std_output_file = str(self.testcase.std_out_file_name)
# Other output files are stored in this variable.
self.output_files = os.listdir(temp_dir)
self.output_files.remove(std_output_file)
# Now check for the status of the testcase. Call compare_output().
is_passed = False
print "Testcase return code is - ", self.testResults.returnCode
if self.testResults.returnCode == 0: # Exited normally.
is_passed = self.compare_output(temp_dir, std_output_file)
# Remove the temporary directory and return the status of the testcase.
shutil.rmtree(temp_dir, ignore_errors=True)
return is_passed
# Function to compare output of the standard output file and the actual output.
# Takes as input the actual output file and expected output in that order.
def compare_output(self, temp_dir, std_output_file):
if not self.compare_file(os.path.join(temp_dir, std_output_file),
self.std_out):
return False
for afile in self.output_files: # compare all O/P files.
if not self.compare_file(afile, afile.split('/')[-1]):
return False
return True
# Function to compare 2 files.
# Takes as input the expected output and actual output file in that order.
def compare_file(self, expectedop, actualop):
# If the actual output is not valid return false.
if not os.path.isfile(actualop):
return False
# Open both the files and see if they have the same length. If true proceed else return false.
file1 = open(expectedop)
file2 = open(actualop)
if (len(list(file1)) > len(list(file2))):
return False
file1 = open(expectedop)
file2 = open(actualop)
# Compare the 2 files line by line.
for line1, line2 in zip(file1, file2):
if line1.strip() != line2.strip():
return False
for line2 in file2:
if line2.strip() != "":
return False
return True
class CreateOutput(object):
def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
Solve this by actually retrieving the exact SafeExec object.
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
# Setup the output creator
def setup(self):
# create temporary directory.
self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
os.chdir(self.temp_dir)
currentDir = os.getcwd()
# Copy program files and input files to the directory
if self.program.program_files :
extract_or_copy(src=self.program.program_files.file.name, dest=currentDir)
if self.program.assignment.model_solution:
extract_or_copy(src=self.program.assignment.model_solution.file.name, dest=currentDir)
# write input file to temporary file.
self.temp_input_d = tempfile.mkdtemp(prefix="input")
if self.input_files:
self.input = os.path.join(
self.temp_input_d,
os.path.basename(self.testcase.input_files.file.name)
)
f = open(self.input, 'w')
for a_line in self.input_files.file:
f.write(a_line)
f.close()
# input file has been written now extract.
extract_or_copy(src=self.input, dest=currentDir)
else:
self.input = ''
# Run the program solution files and
def run(self):
# Compile the solution. If there are errors then write it to the error file and return
if compile_required(self.program.language):
compiler_command = get_compilation_command(self.program)
print "Compiling model solution. Compilation Command - " + compiler_command
self.command_output = self.commandExecutor.safe_execute(
command=compiler_command,
limits=self.get_resource_limit()
)
if self.command_output.getReturnCode():
# There was some error. Write it in database.
print "Compilation of model solution failed."
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
return
# No errors and thus can continue. Run the solution
execution_command = get_execution_command(self.program)
print "Creating Output. Running following execution command - " + execution_command
#if self.testcase.command_line_args:
# execution_command = execution_command + self.testcase.command_line_args
if self.testcase.std_in_file_name:
execution_command = execution_command + " < " + self.testcase.std_in_file_name
self.command_output = self.commandExecutor.safe_execute(
command=execution_command,
limits=self.get_resource_limit()
)
self.success = bool(self.command_output.getReturnCode())
self.command_output.printResult()
# Delete input files from current directory. And the program files
dir_content = os.listdir('./')
for a_file in dir_content:
if self.is_program_file(a_file):
os.remove(a_file)
if self.input:
for a_file in get_file_name_list(name=self.input): # delete extracted files
if os.path.isfile(a_file):
os.remove(a_file)
# Write standard output to a file and save it in database.
directory_content = os.listdir('./')
_, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./")
out_file = open(self.std_out_file, 'w')
for a_line in ["\n".join(self.command_output.get_stdout())]:
out_file.write(a_line)
out_file.close()
# There was some error. Write it in database.
if self.command_output.getReturnCode():
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
# If directory has any content left then make a tar, else set the outfile to the output file.
if directory_content: # make a tar
self.out_file = self.make_tar(directory_content + [self.std_out_file], self.std_out_file)
else: # there are no other files standard output only.
self.out_file = self.std_out_file
def get_stdout_file_name(self):
return os.path.basename(self.std_out_file)
def get_stderr_file_name(self):
return os.path.basename(self.error_file)
def get_stderr_file_path(self):
return self.error_file
def make_tar(self, files, tarname):
tarname = tarname + ".tar.bz2"
output_files_tar = tarfile.open(name=tarname, mode="w:bz2")
# Make tar file from all output files.
for afile in files:
if os.path.isfile(afile):
output_files_tar.add(os.path.basename(afile))
output_files_tar.close()
return tarname
def get_output(self):
prev_dir = os.getcwd()
try:
self.setup()
self.run()
finally:
os.chdir(prev_dir)
return self
def is_program_file(self, fname):
filename = fname.split("/")[-1]
if self.program.program_files and filename in get_file_name_list(self.program.program_files.file.name):
return True
elif self.program.assignment.model_solution and filename in get_file_name_list(self.program.assignment.model_solution.file.name):
return True
elif is_intermediate_files(filename,self.program,self.program.language):
return True
else:
return False
def clean_up(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
shutil.rmtree(self.temp_input_d, ignore_errors=True)
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
class CustomTestcase(object):
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def get_resource_limit(self):
# Return default values because there is no setting for custom testcase.
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
# Store the input file that student uploaded
def store_input_file(self, inputFile, dest):
file_path = os.path.join(dest, inputFile.name.split('/')[-1])
destination = open(file_path, 'wb+')
# Work-around to support file and InMemoeryUploadedFile objects
if hasattr(inputFile, 'chunks'):
all_file = inputFile.chunks()
else:
all_file = inputFile
# Write everything to the input file represented by destination object. Then return the full path of file name.
for chunk in all_file:
destination.write(chunk)
destination.close()
return file_path
# Function to setup the environment (input file, solution files and submission files) before running the code.
def setup(self):
# This is the directory where the student program is copied to.
self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir)
os.chdir(self.temp_dir)
# Copy student solution files to tmp directory.
extract_or_copy(src=self.submission.filePath.file.name, dest=self.temp_dir)
# Another temp directory for running solution program. This is the directory where the solution program is copied to.
self.solution_temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
directories = [a for a in os.listdir('./') if os.path.isdir(a)]
if directories:
os.chdir(directories[0])
currentDir = os.getcwd()
# Extract the program file and the input file to the student directory, and the input file to the program directory as well.
extract_or_copy(src=self.program.program_files.file.name, dest=currentDir)
self.submittedFiles = get_file_name_list(name=self.submission.filePath.file.name)
input_file = self.store_input_file(self.inputFile, currentDir)
shutil.copy(src=input_file, dst=self.solution_temp_dir)
# Function to compile the student program.
def compile(self):
self.exe_file = str(self.program.id)
compiler_command = " ".join(pickle.loads(self.program.compiler_command)) + " -o " + self.exe_file
self.compileResult = self.commandExecutor.executeCommand(command=compiler_command)
# Function to run the student and model solution on the input file.
def run(self):
# Run student exe file.
_, self.actual_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.actual_stdout, 0777)
runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name
self.actualOutput = self.commandExecutor.safe_execute(
command=runStudentProgram,
limits=self.get_resource_limit()
)
# Run solution exe file.
shutil.copy(src=self.program.exe_file_name.file.name, dst=self.solution_temp_dir)
os.chdir(self.solution_temp_dir)
_, self.expected_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.expected_stdout, 0777)
exe = self.program.exe_file_name.name.split('/')[-1]
runSolutionProgram = "./" + exe + " < " + self.inputFile.name
self.expectedOutput = self.commandExecutor.safe_execute(
command=runSolutionProgram,
limits=self.get_resource_limit()
)
self.passed = self.testPassed()
# This function is called after run(), and checks the actual and the expected output.
def testPassed(self):
# At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout.
# compare them and display result on browser.
if not (self.expectedOutput.returnCode == 0 and self.actualOutput.returnCode == 0):
return False
exp_op = iter(self.expectedOutput.get_stdout())
act_op = iter(self.actualOutput.get_stdout())
if len(self.expectedOutput.get_stdout()) > len(self.actualOutput.get_stdout()):
return False
for line1, line2 in zip(exp_op, act_op):
if line1.strip() != line2.strip():
return False
for line2 in act_op:
if line2.strip() != "":
return False
return True
# This is the function called to get the result once the input file is uploaded.
def getResult(self):
# Setup the environment.
self.setup()
# Compile the program.
self.compile()
# If compilation is successful then run the program.
if self.compileResult.returnCode == 0:
self.run()
# Clean up the temporary directories.
self.cleanUp()
return self
# Function to clean up the temporary directories.
def cleanUp(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
class CreateOutput(object):
def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
return {
'cpu_time': 10,
'clock_time': 10,
'memory': 32768,
'stack_size': 8192,
'child_processes': 0,
'open_files': 512,
'file_size': 1024,
}
'''
Solve this by actually retrieving the exact SafeExec object.
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
# Setup the output creator
def setup(self):
# create temporary directory.
self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
os.chdir(self.temp_dir)
currentDir = os.getcwd()
# Copy program files and input files to the directory
if self.program.program_files:
extract_or_copy(src=self.program.program_files.file.name,
dest=currentDir)
if self.program.assignment.model_solution:
extract_or_copy(
src=self.program.assignment.model_solution.file.name,
dest=currentDir)
# write input file to temporary file.
self.temp_input_d = tempfile.mkdtemp(prefix="input")
if self.input_files:
self.input = os.path.join(
self.temp_input_d,
os.path.basename(self.testcase.input_files.file.name))
f = open(self.input, 'w')
for a_line in self.input_files.file:
f.write(a_line)
f.close()
# input file has been written now extract.
extract_or_copy(src=self.input, dest=currentDir)
else:
self.input = ''
# Run the program solution files and
def run(self):
# Compile the solution. If there are errors then write it to the error file and return
if compile_required(self.program.language):
compiler_command = get_compilation_command(self.program)
print "Compiling model solution. Compilation Command - " + compiler_command
self.command_output = self.commandExecutor.safe_execute(
command=compiler_command, limits=self.get_resource_limit())
if self.command_output.getReturnCode():
# There was some error. Write it in database.
print "Compilation of model solution failed."
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error",
dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
return
# No errors and thus can continue. Run the solution
execution_command = get_execution_command(self.program)
print "Creating Output. Running following execution command - " + execution_command
#if self.testcase.command_line_args:
# execution_command = execution_command + self.testcase.command_line_args
if self.testcase.std_in_file_name:
execution_command = execution_command + " < " + self.testcase.std_in_file_name
self.command_output = self.commandExecutor.safe_execute(
command=execution_command, limits=self.get_resource_limit())
self.success = bool(self.command_output.getReturnCode())
self.command_output.printResult()
# Delete input files from current directory. And the program files
dir_content = os.listdir('./')
for a_file in dir_content:
if self.is_program_file(a_file):
os.remove(a_file)
if self.input:
for a_file in get_file_name_list(
name=self.input): # delete extracted files
if os.path.isfile(a_file):
os.remove(a_file)
# Write standard output to a file and save it in database.
directory_content = os.listdir('./')
_, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./")
out_file = open(self.std_out_file, 'w')
for a_line in ["\n".join(self.command_output.get_stdout())]:
out_file.write(a_line)
out_file.close()
# There was some error. Write it in database.
if self.command_output.getReturnCode():
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error",
dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
# If directory has any content left then make a tar, else set the outfile to the output file.
if directory_content: # make a tar
self.out_file = self.make_tar(
directory_content + [self.std_out_file], self.std_out_file)
else: # there are no other files standard output only.
self.out_file = self.std_out_file
def get_stdout_file_name(self):
return os.path.basename(self.std_out_file)
def get_stderr_file_name(self):
return os.path.basename(self.error_file)
def get_stderr_file_path(self):
return self.error_file
def make_tar(self, files, tarname):
tarname = tarname + ".tar.bz2"
output_files_tar = tarfile.open(name=tarname, mode="w:bz2")
# Make tar file from all output files.
for afile in files:
if os.path.isfile(afile):
output_files_tar.add(os.path.basename(afile))
output_files_tar.close()
return tarname
def get_output(self):
prev_dir = os.getcwd()
try:
self.setup()
self.run()
finally:
os.chdir(prev_dir)
return self
def is_program_file(self, fname):
filename = fname.split("/")[-1]
if self.program.program_files and filename in get_file_name_list(
self.program.program_files.file.name):
return True
elif self.program.assignment.model_solution and filename in get_file_name_list(
self.program.assignment.model_solution.file.name):
return True
elif is_intermediate_files(filename, self.program,
self.program.language):
return True
else:
return False
def clean_up(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
shutil.rmtree(self.temp_input_d, ignore_errors=True)
class Program(object):
def __init__(self, program, assignment_result):
self.assignment_result = assignment_result
self.program = program
# Extracting the list of files relevant to only this section.
if program.program_files:
prgrm_files = get_file_name_list(
name=program.program_files.file.name)
prgrm_files = " ".join(prgrm_files)
else:
prgrm_files = ""
# Adding the list of files in the assignment model solution to the above list.
self.programFiles = program.assignment.student_program_files + " " + prgrm_files
self.language = program.assignment.program_language
self.compiler_command = get_compilation_command(program)
self.execution_command = get_execution_command(program)
self.commandExecutor = CommandExecutor()
# Function to check if any files are missing.
def fileExist(self):
self.missingFiles = []
# Checking if each file in the self.programFiles variable is valid. If not add to the missing files array.
for aFile in self.programFiles.split():
if not os.path.isfile(aFile):
self.missingFiles.append(aFile)
# If there are any missing files, then either retrieve existing program result object or create one with the assignment result, section
# and the missing files. If the object was not created, then update the missing files attribute of the program result.
# Save the new or existing program result object after this, and return if there were any files missing.
if self.missingFiles:
self.programResult, created = ProgramResults.objects.get_or_create(
assignment_result=self.assignment_result,
program=self.program,
defaults={'missing_file_names': "\n".join(self.missingFiles)})
if not created:
self.programResult.missing_file_names = "\n".join(
self.missingFiles)
self.programResult.save()
return bool(self.missingFiles)
# Function to handle compilation of the section.
def compile(self):
# If any files are missing then return failure of compilation. Appropriate program result is created.
if self.fileExist():
return False
# If the language of the section/assignment needs compilation, then go ahead and compile. Set program result attributes accordingly.
# If compilation is not needed then proceed.
if compile_required(self.language):
compilation_command = self.compiler_command
self.compileResult = self.commandExecutor.executeCommand(
command=compilation_command)
attrs = {
'missing_file_names': "\n".join(self.missingFiles),
'compiler_errors': "".join(self.compileResult.stderr),
'compiler_output': "\n".join(self.compileResult.get_stdout()),
'compiler_return_code': int(self.compileResult.returnCode)
}
else:
attrs = {
'missing_file_names': "\n".join(self.missingFiles),
'compiler_errors': "",
'compiler_output': "",
'compiler_return_code': 0
}
filter_attrs = {
'assignment_result': self.assignment_result,
'program': self.program
}
# Create or update program result object with the result of compilation process for the section. The attributes of the program result object
# are the same as the result of the compilation process.
try: # create_or_update equivalent.
obj = ProgramResults.objects.filter(**filter_attrs)
obj.update(**attrs)
self.programResult = obj[0:1].get()
except ProgramResults.DoesNotExist:
attrs.update(filter_attrs)
self.programResult = ProgramResults.objects.create(**attrs)
# If compilation is successful or if compilation is not needed then return true. Else return false.
if compile_required(
self.language) == False or self.compileResult.returnCode == 0:
return True
else:
return False
# Function to run the student solution on all the testcases of the section.
def run(self):
# Filter out the testcases of the section.
testcases = Testcase.objects.filter(program=self.program)
# Run the execution command on each of the testcases. Compilation (if needed) is already taken care of.
for test in testcases:
testcase = TestCase(test, self.programResult)
testcase.run(self.execution_command)
def compile_solution(sender, instance, **kwargs):
# If the program language is an interpreted language, then we dont do anything
if not compile_required(instance.assignment.program_language) and not getattr(instance, 'execute_now', False):
instance.set_sane()
instance.delete_error_message()
return
if not compile_required(instance.assignment.program_language) :
old_pwd = os.getcwd()
instance.set_sane()
instance.UpdateOutput()
instance.delete_error_message()
os.chdir(old_pwd)
return
# instance.id would indicate that record was not created.
if not getattr(instance, 'compile_now', False) or instance.id is None:
return
# If solution is given then set the path, else do nothing
programFilePath = instance.program_files.name
if hasattr(instance.assignment.model_solution, 'file'):
ModelSolutionPath = instance.assignment.model_solution.file.name
else:
instance.set_sane()
instance.delete_error_message()
return
# Model Solution was not provided hence do nothing.
if not os.path.isfile(ModelSolutionPath):
return
# Copying module solution to a temp directory for compilation
tmp_dir = tempfile.mkdtemp(prefix='grader')
old_pwd = os.getcwd()
os.chdir(tmp_dir)
currentDir = os.getcwd()
try:
# Copy model solution to temp directory.
with Archive(fileobj=instance.assignment.model_solution.file) as archive:
if archive.is_archive():
archive.extract(dest=currentDir)
else:
shutil.copy(src=os.path.join(MEDIA_ROOT, ModelSolutionPath), dst=currentDir)
# Change directory if solution tar contains a directory.
directories = [a for a in os.listdir('./') if os.path.isdir(a)]
if directories:
os.chdir(directories[0])
currentDir = os.getcwd()
# Copying the program files to the current directory
if instance.program_files:
with Archive(name=instance.program_files.file.name) as archive:
if archive.is_archive():
archive.extract(dest=currentDir)
else:
shutil.copy(src=os.path.join(MEDIA_ROOT, programFilePath), dst=currentDir)
# Setting up compilation process and calling the CommandExecutor with appropriate command
executor = CommandExecutor(timeout=100)
compiler_command = get_compilation_command(instance)
compileResult = executor.executeCommand(command=compiler_command)
# Compilation successful => Set the program to sane and delete errors, else report the errors.
if compileResult.returnCode == 0: # save exe file on success.
instance.is_sane = True
instance.UpdateOutput()
instance.delete_error_message()
else:
message = "Compilation failed.\nReturn code = {0}.\nError message={1}".format(compileResult.returnCode, "".join(compileResult.stderr))
print "Instructor solution not sane - " + message
instance.save_error(message)
shutil.rmtree(tmp_dir)
finally:
os.chdir(old_pwd)
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
def __init__(self, submission):
self.commandExecutor = CommandExecutor()
self.submission = submission
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
class Program(object):
def __init__(self, program, assignment_result):
self.assignment_result = assignment_result
self.program = program
# Extracting the list of files relevant to only this section.
if program.program_files:
prgrm_files = get_file_name_list(name=program.program_files.file.name)
prgrm_files = " ".join(prgrm_files)
else:
prgrm_files = ""
# Adding the list of files in the assignment model solution to the above list.
self.programFiles = program.assignment.student_program_files + " " + prgrm_files
self.language = program.assignment.program_language
self.compiler_command = get_compilation_command(program)
self.execution_command = get_execution_command(program)
self.commandExecutor = CommandExecutor()
# Function to check if any files are missing.
def fileExist(self):
self.missingFiles = []
# Checking if each file in the self.programFiles variable is valid. If not add to the missing files array.
for aFile in self.programFiles.split():
if not os.path.isfile(aFile):
self.missingFiles.append(aFile)
# If there are any missing files, then either retrieve existing program result object or create one with the assignment result, section
# and the missing files. If the object was not created, then update the missing files attribute of the program result.
# Save the new or existing program result object after this, and return if there were any files missing.
if self.missingFiles:
self.programResult, created = ProgramResults.objects.get_or_create(
assignment_result = self.assignment_result,
program = self.program,
defaults = {'missing_file_names': "\n".join(self.missingFiles)}
)
if not created:
self.programResult.missing_file_names = "\n".join(self.missingFiles)
self.programResult.save()
return bool(self.missingFiles)
# Function to handle compilation of the section.
def compile(self):
# If any files are missing then return failure of compilation. Appropriate program result is created.
if self.fileExist():
return False
# If the language of the section/assignment needs compilation, then go ahead and compile. Set program result attributes accordingly.
# If compilation is not needed then proceed.
if compile_required(self.language):
compilation_command = self.compiler_command
self.compileResult = self.commandExecutor.executeCommand(command=compilation_command)
attrs = {'missing_file_names': "\n".join(self.missingFiles),
'compiler_errors': "".join(self.compileResult.stderr),
'compiler_output': "\n".join(self.compileResult.get_stdout()),
'compiler_return_code': int(self.compileResult.returnCode)}
else:
attrs = {'missing_file_names': "\n".join(self.missingFiles),
'compiler_errors': "",
'compiler_output': "",
'compiler_return_code': 0}
filter_attrs = {'assignment_result': self.assignment_result, 'program': self.program}
# Create or update program result object with the result of compilation process for the section. The attributes of the program result object
# are the same as the result of the compilation process.
try: # create_or_update equivalent.
obj = ProgramResults.objects.filter(**filter_attrs)
obj.update(**attrs)
self.programResult = obj[0:1].get()
except ProgramResults.DoesNotExist:
attrs.update(filter_attrs)
self.programResult = ProgramResults.objects.create(**attrs)
# If compilation is successful or if compilation is not needed then return true. Else return false.
if compile_required(self.language) == False or self.compileResult.returnCode == 0:
return True
else:
return False
# Function to run the student solution on all the testcases of the section.
def run(self):
# Filter out the testcases of the section.
testcases = Testcase.objects.filter(program=self.program)
# Run the execution command on each of the testcases. Compilation (if needed) is already taken care of.
for test in testcases:
testcase = TestCase(test, self.programResult)
testcase.run(self.execution_command)
class TestCase(object):
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {}
# Function to make a tarfile of all output files and return the name of this file.
def make_tar(self):
# Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id"
stdoutFileName = "standardOutput_" + str(self.testcase.id)
shutil.move(self.std_out, stdoutFileName)
outputfiles = [stdoutFileName]
# Set the name of the tarfile which is going to contain the output files.
tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2"
outputFilesTar = tarfile.open(name=tarname, mode="w:bz2")
# Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution
# wrote to other than the std output) to the tarfile.
for afile in outputfiles + self.output_files:
if os.path.isfile(afile):
outputFilesTar.add(afile)
outputFilesTar.close()
return tarname
# Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section
# level itself.
def run(self, exeCommand):
# Create the output file to capture the std output in the current directory.
_, self.std_out = tempfile.mkstemp()
os.chmod(self.std_out, 0777)
command = exeCommand
# Add the command line arguments if any.
if self.testcase.command_line_args:
command = command + self.testcase.command_line_args
# Then redirect the std input to the input file.
if os.path.isfile(self.stdInFile):
command = command + " < " + self.stdInFile
# Execute the resulting command with the appropriate resource limits and the output file created above.
self.testResults = self.commandExecutor.safe_execute(
command=command,
limits=self.get_resource_limit(),
outfile=self.std_out
)
# Check if the testcase has passed.
self.passed = self.test_passed()
# Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase.
attrs = {'error_messages': "".join(self.testResults.stderr),
'return_code': int(self.testResults.returnCode),
'test_passed': self.passed}
# This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above.
filter_attrs = {'test_case': self.testcase, 'program_result': self.program_result}
# If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes.
# Remove the older output file.
# Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary.
try:
obj = TestcaseResult.objects.filter(**filter_attrs)
obj.update(**attrs)
test_result = obj[0:1].get()
default_storage.delete(test_result.output_files.path) # Remove older file.
except TestcaseResult.DoesNotExist:
attrs.update(filter_attrs)
test_result = TestcaseResult.objects.create(**attrs)
# Make a tar file of the std output file and the other files which have been written to by the student submission.
# Update the test result (updated or newly created) with this tar file as the output file.
tar_name = self.make_tar()
test_result.output_files.save(tar_name, File(open(tar_name)))
return self
# Function to check if the student submission has passed the testcase.
def test_passed(self):
# Create a temporary directory and copy the expected output of the testcase to this file.
temp_dir = tempfile.mkdtemp(prefix="grader_op_files")
extract_or_copy(src=self.testcase.output_files.file.name, dest=temp_dir)
std_output_file = str(self.testcase.std_out_file_name)
# Other output files are stored in this variable.
self.output_files = os.listdir(temp_dir)
self.output_files.remove(std_output_file)
# Now check for the status of the testcase. Call compare_output().
is_passed = False
print "Testcase return code is - ", self.testResults.returnCode
if self.testResults.returnCode == 0: # Exited normally.
is_passed = self.compare_output(temp_dir, std_output_file)
# Remove the temporary directory and return the status of the testcase.
shutil.rmtree(temp_dir, ignore_errors=True)
return is_passed
# Function to compare output of the standard output file and the actual output.
# Takes as input the actual output file and expected output in that order.
def compare_output(self, temp_dir, std_output_file):
if not self.compare_file(os.path.join(temp_dir, std_output_file), self.std_out):
return False
for afile in self.output_files: # compare all O/P files.
if not self.compare_file(afile, afile.split('/')[-1]):
return False
return True
# Function to compare 2 files.
# Takes as input the expected output and actual output file in that order.
def compare_file(self, expectedop, actualop):
# If the actual output is not valid return false.
if not os.path.isfile(actualop):
return False
# Open both the files and see if they have the same length. If true proceed else return false.
file1 = open(expectedop)
file2 = open(actualop)
if(len(list(file1)) > len(list(file2))):
return False
file1 = open(expectedop)
file2 = open(actualop)
# Compare the 2 files line by line.
for line1, line2 in zip(file1, file2):
if line1.strip() != line2.strip():
return False
for line2 in file2:
if line2.strip() != "":
return False
return True
