class CreateOutput(object):
def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
Solve this by actually retrieving the exact SafeExec object.
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
# Setup the output creator
def setup(self):
# create temporary directory.
self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
os.chdir(self.temp_dir)
currentDir = os.getcwd()
# Copy program files and input files to the directory
if self.program.program_files :
extract_or_copy(src=self.program.program_files.file.name, dest=currentDir)
if self.program.assignment.model_solution:
extract_or_copy(src=self.program.assignment.model_solution.file.name, dest=currentDir)
# write input file to temporary file.
self.temp_input_d = tempfile.mkdtemp(prefix="input")
if self.input_files:
self.input = os.path.join(
self.temp_input_d,
os.path.basename(self.testcase.input_files.file.name)
)
f = open(self.input, 'w')
for a_line in self.input_files.file:
f.write(a_line)
f.close()
# input file has been written now extract.
extract_or_copy(src=self.input, dest=currentDir)
else:
self.input = ''
# Run the program solution files and
def run(self):
# Compile the solution. If there are errors then write it to the error file and return
if compile_required(self.program.language):
compiler_command = get_compilation_command(self.program)
print "Compiling model solution. Compilation Command - " + compiler_command
self.command_output = self.commandExecutor.safe_execute(
command=compiler_command,
limits=self.get_resource_limit()
)
if self.command_output.getReturnCode():
# There was some error. Write it in database.
print "Compilation of model solution failed."
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
return
# No errors and thus can continue. Run the solution
execution_command = get_execution_command(self.program)
print "Creating Output. Running following execution command - " + execution_command
#if self.testcase.command_line_args:
# execution_command = execution_command + self.testcase.command_line_args
if self.testcase.std_in_file_name:
execution_command = execution_command + " < " + self.testcase.std_in_file_name
self.command_output = self.commandExecutor.safe_execute(
command=execution_command,
limits=self.get_resource_limit()
)
self.success = bool(self.command_output.getReturnCode())
self.command_output.printResult()
# Delete input files from current directory. And the program files
dir_content = os.listdir('./')
for a_file in dir_content:
if self.is_program_file(a_file):
os.remove(a_file)
if self.input:
for a_file in get_file_name_list(name=self.input): # delete extracted files
if os.path.isfile(a_file):
os.remove(a_file)
# Write standard output to a file and save it in database.
directory_content = os.listdir('./')
_, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./")
out_file = open(self.std_out_file, 'w')
for a_line in ["\n".join(self.command_output.get_stdout())]:
out_file.write(a_line)
out_file.close()
# There was some error. Write it in database.
if self.command_output.getReturnCode():
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
# If directory has any content left then make a tar, else set the outfile to the output file.
if directory_content: # make a tar
self.out_file = self.make_tar(directory_content + [self.std_out_file], self.std_out_file)
else: # there are no other files standard output only.
self.out_file = self.std_out_file
def get_stdout_file_name(self):
return os.path.basename(self.std_out_file)
def get_stderr_file_name(self):
return os.path.basename(self.error_file)
def get_stderr_file_path(self):
return self.error_file
def make_tar(self, files, tarname):
tarname = tarname + ".tar.bz2"
output_files_tar = tarfile.open(name=tarname, mode="w:bz2")
# Make tar file from all output files.
for afile in files:
if os.path.isfile(afile):
output_files_tar.add(os.path.basename(afile))
output_files_tar.close()
return tarname
def get_output(self):
prev_dir = os.getcwd()
try:
self.setup()
self.run()
finally:
os.chdir(prev_dir)
return self
def is_program_file(self, fname):
filename = fname.split("/")[-1]
if self.program.program_files and filename in get_file_name_list(self.program.program_files.file.name):
return True
elif self.program.assignment.model_solution and filename in get_file_name_list(self.program.assignment.model_solution.file.name):
return True
elif is_intermediate_files(filename,self.program,self.program.language):
return True
else:
return False
def clean_up(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
shutil.rmtree(self.temp_input_d, ignore_errors=True)
class CustomTestcase(object):
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def get_resource_limit(self):
# Return default values because there is no setting for custom testcase.
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
# Store the input file that student uploaded
def store_input_file(self, inputFile, dest):
file_path = os.path.join(dest, inputFile.name.split('/')[-1])
destination = open(file_path, 'wb+')
# Work-around to support file and InMemoeryUploadedFile objects
if hasattr(inputFile, 'chunks'):
all_file = inputFile.chunks()
else:
all_file = inputFile
# Write everything to the input file represented by destination object. Then return the full path of file name.
for chunk in all_file:
destination.write(chunk)
destination.close()
return file_path
# Function to setup the environment (input file, solution files and submission files) before running the code.
def setup(self):
# This is the directory where the student program is copied to.
self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir)
os.chdir(self.temp_dir)
# Copy student solution files to tmp directory.
extract_or_copy(src=self.submission.filePath.file.name, dest=self.temp_dir)
# Another temp directory for running solution program. This is the directory where the solution program is copied to.
self.solution_temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
directories = [a for a in os.listdir('./') if os.path.isdir(a)]
if directories:
os.chdir(directories[0])
currentDir = os.getcwd()
# Extract the program file and the input file to the student directory, and the input file to the program directory as well.
extract_or_copy(src=self.program.program_files.file.name, dest=currentDir)
self.submittedFiles = get_file_name_list(name=self.submission.filePath.file.name)
input_file = self.store_input_file(self.inputFile, currentDir)
shutil.copy(src=input_file, dst=self.solution_temp_dir)
# Function to compile the student program.
def compile(self):
self.exe_file = str(self.program.id)
compiler_command = " ".join(pickle.loads(self.program.compiler_command)) + " -o " + self.exe_file
self.compileResult = self.commandExecutor.executeCommand(command=compiler_command)
# Function to run the student and model solution on the input file.
def run(self):
# Run student exe file.
_, self.actual_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.actual_stdout, 0777)
runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name
self.actualOutput = self.commandExecutor.safe_execute(
command=runStudentProgram,
limits=self.get_resource_limit()
)
# Run solution exe file.
shutil.copy(src=self.program.exe_file_name.file.name, dst=self.solution_temp_dir)
os.chdir(self.solution_temp_dir)
_, self.expected_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.expected_stdout, 0777)
exe = self.program.exe_file_name.name.split('/')[-1]
runSolutionProgram = "./" + exe + " < " + self.inputFile.name
self.expectedOutput = self.commandExecutor.safe_execute(
command=runSolutionProgram,
limits=self.get_resource_limit()
)
self.passed = self.testPassed()
# This function is called after run(), and checks the actual and the expected output.
def testPassed(self):
# At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout.
# compare them and display result on browser.
if not (self.expectedOutput.returnCode == 0 and self.actualOutput.returnCode == 0):
return False
exp_op = iter(self.expectedOutput.get_stdout())
act_op = iter(self.actualOutput.get_stdout())
if len(self.expectedOutput.get_stdout()) > len(self.actualOutput.get_stdout()):
return False
for line1, line2 in zip(exp_op, act_op):
if line1.strip() != line2.strip():
return False
for line2 in act_op:
if line2.strip() != "":
return False
return True
# This is the function called to get the result once the input file is uploaded.
def getResult(self):
# Setup the environment.
self.setup()
# Compile the program.
self.compile()
# If compilation is successful then run the program.
if self.compileResult.returnCode == 0:
self.run()
# Clean up the temporary directories.
self.cleanUp()
return self
# Function to clean up the temporary directories.
def cleanUp(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
class CreateOutput(object):
def __init__(self, instance):
self.program = instance.program
self.testcase = instance
self.input_files = instance.input_files
self.commandExecutor = CommandExecutor()
self.failed = False
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
return {
'cpu_time': 10,
'clock_time': 10,
'memory': 32768,
'stack_size': 8192,
'child_processes': 0,
'open_files': 512,
'file_size': 1024,
}
'''
Solve this by actually retrieving the exact SafeExec object.
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {'cpu_time':10, 'clock_time':10,
'memory': 32768, 'stack_size': 8192, 'child_processes': 0,
'open_files': 512, 'file_size': 1024,
}
'''
# Setup the output creator
def setup(self):
# create temporary directory.
self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir)
os.chdir(self.temp_dir)
currentDir = os.getcwd()
# Copy program files and input files to the directory
if self.program.program_files:
extract_or_copy(src=self.program.program_files.file.name,
dest=currentDir)
if self.program.assignment.model_solution:
extract_or_copy(
src=self.program.assignment.model_solution.file.name,
dest=currentDir)
# write input file to temporary file.
self.temp_input_d = tempfile.mkdtemp(prefix="input")
if self.input_files:
self.input = os.path.join(
self.temp_input_d,
os.path.basename(self.testcase.input_files.file.name))
f = open(self.input, 'w')
for a_line in self.input_files.file:
f.write(a_line)
f.close()
# input file has been written now extract.
extract_or_copy(src=self.input, dest=currentDir)
else:
self.input = ''
# Run the program solution files and
def run(self):
# Compile the solution. If there are errors then write it to the error file and return
if compile_required(self.program.language):
compiler_command = get_compilation_command(self.program)
print "Compiling model solution. Compilation Command - " + compiler_command
self.command_output = self.commandExecutor.safe_execute(
command=compiler_command, limits=self.get_resource_limit())
if self.command_output.getReturnCode():
# There was some error. Write it in database.
print "Compilation of model solution failed."
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error",
dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
return
# No errors and thus can continue. Run the solution
execution_command = get_execution_command(self.program)
print "Creating Output. Running following execution command - " + execution_command
#if self.testcase.command_line_args:
# execution_command = execution_command + self.testcase.command_line_args
if self.testcase.std_in_file_name:
execution_command = execution_command + " < " + self.testcase.std_in_file_name
self.command_output = self.commandExecutor.safe_execute(
command=execution_command, limits=self.get_resource_limit())
self.success = bool(self.command_output.getReturnCode())
self.command_output.printResult()
# Delete input files from current directory. And the program files
dir_content = os.listdir('./')
for a_file in dir_content:
if self.is_program_file(a_file):
os.remove(a_file)
if self.input:
for a_file in get_file_name_list(
name=self.input): # delete extracted files
if os.path.isfile(a_file):
os.remove(a_file)
# Write standard output to a file and save it in database.
directory_content = os.listdir('./')
_, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./")
out_file = open(self.std_out_file, 'w')
for a_line in ["\n".join(self.command_output.get_stdout())]:
out_file.write(a_line)
out_file.close()
# There was some error. Write it in database.
if self.command_output.getReturnCode():
self.failed = True
_, self.error_file = tempfile.mkstemp(prefix="error",
dir=self.temp_input_d)
f = open(self.error_file, 'w')
for a_line in self.command_output.get_stderr():
f.write(a_line)
f.close()
# If directory has any content left then make a tar, else set the outfile to the output file.
if directory_content: # make a tar
self.out_file = self.make_tar(
directory_content + [self.std_out_file], self.std_out_file)
else: # there are no other files standard output only.
self.out_file = self.std_out_file
def get_stdout_file_name(self):
return os.path.basename(self.std_out_file)
def get_stderr_file_name(self):
return os.path.basename(self.error_file)
def get_stderr_file_path(self):
return self.error_file
def make_tar(self, files, tarname):
tarname = tarname + ".tar.bz2"
output_files_tar = tarfile.open(name=tarname, mode="w:bz2")
# Make tar file from all output files.
for afile in files:
if os.path.isfile(afile):
output_files_tar.add(os.path.basename(afile))
output_files_tar.close()
return tarname
def get_output(self):
prev_dir = os.getcwd()
try:
self.setup()
self.run()
finally:
os.chdir(prev_dir)
return self
def is_program_file(self, fname):
filename = fname.split("/")[-1]
if self.program.program_files and filename in get_file_name_list(
self.program.program_files.file.name):
return True
elif self.program.assignment.model_solution and filename in get_file_name_list(
self.program.assignment.model_solution.file.name):
return True
elif is_intermediate_files(filename, self.program,
self.program.language):
return True
else:
return False
def clean_up(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
shutil.rmtree(self.temp_input_d, ignore_errors=True)
class TestCase(object):
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size']
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {}
# Function to make a tarfile of all output files and return the name of this file.
def make_tar(self):
# Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id"
stdoutFileName = "standardOutput_" + str(self.testcase.id)
shutil.move(self.std_out, stdoutFileName)
outputfiles = [stdoutFileName]
# Set the name of the tarfile which is going to contain the output files.
tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2"
outputFilesTar = tarfile.open(name=tarname, mode="w:bz2")
# Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution
# wrote to other than the std output) to the tarfile.
for afile in outputfiles + self.output_files:
if os.path.isfile(afile):
outputFilesTar.add(afile)
outputFilesTar.close()
return tarname
# Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section
# level itself.
def run(self, exeCommand):
# Create the output file to capture the std output in the current directory.
_, self.std_out = tempfile.mkstemp()
os.chmod(self.std_out, 0777)
command = exeCommand
# Add the command line arguments if any.
if self.testcase.command_line_args:
command = command + self.testcase.command_line_args
# Then redirect the std input to the input file.
if os.path.isfile(self.stdInFile):
command = command + " < " + self.stdInFile
# Execute the resulting command with the appropriate resource limits and the output file created above.
self.testResults = self.commandExecutor.safe_execute(
command=command,
limits=self.get_resource_limit(),
outfile=self.std_out
)
# Check if the testcase has passed.
self.passed = self.test_passed()
# Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase.
attrs = {'error_messages': "".join(self.testResults.stderr),
'return_code': int(self.testResults.returnCode),
'test_passed': self.passed}
# This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above.
filter_attrs = {'test_case': self.testcase, 'program_result': self.program_result}
# If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes.
# Remove the older output file.
# Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary.
try:
obj = TestcaseResult.objects.filter(**filter_attrs)
obj.update(**attrs)
test_result = obj[0:1].get()
default_storage.delete(test_result.output_files.path) # Remove older file.
except TestcaseResult.DoesNotExist:
attrs.update(filter_attrs)
test_result = TestcaseResult.objects.create(**attrs)
# Make a tar file of the std output file and the other files which have been written to by the student submission.
# Update the test result (updated or newly created) with this tar file as the output file.
tar_name = self.make_tar()
test_result.output_files.save(tar_name, File(open(tar_name)))
return self
# Function to check if the student submission has passed the testcase.
def test_passed(self):
# Create a temporary directory and copy the expected output of the testcase to this file.
temp_dir = tempfile.mkdtemp(prefix="grader_op_files")
extract_or_copy(src=self.testcase.output_files.file.name, dest=temp_dir)
std_output_file = str(self.testcase.std_out_file_name)
# Other output files are stored in this variable.
self.output_files = os.listdir(temp_dir)
self.output_files.remove(std_output_file)
# Now check for the status of the testcase. Call compare_output().
is_passed = False
print "Testcase return code is - ", self.testResults.returnCode
if self.testResults.returnCode == 0: # Exited normally.
is_passed = self.compare_output(temp_dir, std_output_file)
# Remove the temporary directory and return the status of the testcase.
shutil.rmtree(temp_dir, ignore_errors=True)
return is_passed
# Function to compare output of the standard output file and the actual output.
# Takes as input the actual output file and expected output in that order.
def compare_output(self, temp_dir, std_output_file):
if not self.compare_file(os.path.join(temp_dir, std_output_file), self.std_out):
return False
for afile in self.output_files: # compare all O/P files.
if not self.compare_file(afile, afile.split('/')[-1]):
return False
return True
# Function to compare 2 files.
# Takes as input the expected output and actual output file in that order.
def compare_file(self, expectedop, actualop):
# If the actual output is not valid return false.
if not os.path.isfile(actualop):
return False
# Open both the files and see if they have the same length. If true proceed else return false.
file1 = open(expectedop)
file2 = open(actualop)
if(len(list(file1)) > len(list(file2))):
return False
file1 = open(expectedop)
file2 = open(actualop)
# Compare the 2 files line by line.
for line1, line2 in zip(file1, file2):
if line1.strip() != line2.strip():
return False
for line2 in file2:
if line2.strip() != "":
return False
return True
class TestCase(object):
def __init__(self, testcase, program_result):
self.testcase = testcase
self.program_result = program_result
self.name = testcase.name
self.stdInFile = str(testcase.std_in_file_name)
self.commandExecutor = CommandExecutor()
# Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features
# is also created (with default values if the instructor does not change this).
# This function returns the dictionary mapping the attributes to their respective limits for this testcase.
def get_resource_limit(self):
try:
resource = {}
safeexec_obj = SafeExec.objects.get(testcase=self.testcase)
attrs = [
'cpu_time', 'clock_time', 'memory', 'stack_size',
'child_processes', 'open_files', 'file_size'
]
for attr in attrs:
val = getattr(safeexec_obj, attr)
if val is not None:
resource[attr] = val
return resource
except SafeExec.DoesNotExist:
return {}
# Function to make a tarfile of all output files and return the name of this file.
def make_tar(self):
# Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id"
stdoutFileName = "standardOutput_" + str(self.testcase.id)
shutil.move(self.std_out, stdoutFileName)
outputfiles = [stdoutFileName]
# Set the name of the tarfile which is going to contain the output files.
tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2"
outputFilesTar = tarfile.open(name=tarname, mode="w:bz2")
# Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution
# wrote to other than the std output) to the tarfile.
for afile in outputfiles + self.output_files:
if os.path.isfile(afile):
outputFilesTar.add(afile)
outputFilesTar.close()
return tarname
# Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section
# level itself.
def run(self, exeCommand):
# Create the output file to capture the std output in the current directory.
_, self.std_out = tempfile.mkstemp()
os.chmod(self.std_out, 0777)
command = exeCommand
# Add the command line arguments if any.
if self.testcase.command_line_args:
command = command + self.testcase.command_line_args
# Then redirect the std input to the input file.
if os.path.isfile(self.stdInFile):
command = command + " < " + self.stdInFile
# Execute the resulting command with the appropriate resource limits and the output file created above.
self.testResults = self.commandExecutor.safe_execute(
command=command,
limits=self.get_resource_limit(),
outfile=self.std_out)
# Check if the testcase has passed.
self.passed = self.test_passed()
# Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase.
attrs = {
'error_messages': "".join(self.testResults.stderr),
'return_code': int(self.testResults.returnCode),
'test_passed': self.passed
}
# This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above.
filter_attrs = {
'test_case': self.testcase,
'program_result': self.program_result
}
# If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes.
# Remove the older output file.
# Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary.
try:
obj = TestcaseResult.objects.filter(**filter_attrs)
obj.update(**attrs)
test_result = obj[0:1].get()
default_storage.delete(
test_result.output_files.path) # Remove older file.
except TestcaseResult.DoesNotExist:
attrs.update(filter_attrs)
test_result = TestcaseResult.objects.create(**attrs)
# Make a tar file of the std output file and the other files which have been written to by the student submission.
# Update the test result (updated or newly created) with this tar file as the output file.
tar_name = self.make_tar()
test_result.output_files.save(tar_name, File(open(tar_name)))
return self
# Function to check if the student submission has passed the testcase.
def test_passed(self):
# Create a temporary directory and copy the expected output of the testcase to this file.
temp_dir = tempfile.mkdtemp(prefix="grader_op_files")
extract_or_copy(src=self.testcase.output_files.file.name,
dest=temp_dir)
std_output_file = str(self.testcase.std_out_file_name)
# Other output files are stored in this variable.
self.output_files = os.listdir(temp_dir)
self.output_files.remove(std_output_file)
# Now check for the status of the testcase. Call compare_output().
is_passed = False
print "Testcase return code is - ", self.testResults.returnCode
if self.testResults.returnCode == 0: # Exited normally.
is_passed = self.compare_output(temp_dir, std_output_file)
# Remove the temporary directory and return the status of the testcase.
shutil.rmtree(temp_dir, ignore_errors=True)
return is_passed
# Function to compare output of the standard output file and the actual output.
# Takes as input the actual output file and expected output in that order.
def compare_output(self, temp_dir, std_output_file):
if not self.compare_file(os.path.join(temp_dir, std_output_file),
self.std_out):
return False
for afile in self.output_files: # compare all O/P files.
if not self.compare_file(afile, afile.split('/')[-1]):
return False
return True
# Function to compare 2 files.
# Takes as input the expected output and actual output file in that order.
def compare_file(self, expectedop, actualop):
# If the actual output is not valid return false.
if not os.path.isfile(actualop):
return False
# Open both the files and see if they have the same length. If true proceed else return false.
file1 = open(expectedop)
file2 = open(actualop)
if (len(list(file1)) > len(list(file2))):
return False
file1 = open(expectedop)
file2 = open(actualop)
# Compare the 2 files line by line.
for line1, line2 in zip(file1, file2):
if line1.strip() != line2.strip():
return False
for line2 in file2:
if line2.strip() != "":
return False
return True
class CustomTestcase(object):
def __init__(self, inputFile, program, submission):
# Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands.
self.program = program
self.submission = submission
self.inputFile = inputFile
self.commandExecutor = CommandExecutor()
self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec')
def get_resource_limit(self):
# Return default values because there is no setting for custom testcase.
return {
'cpu_time': 10,
'clock_time': 10,
'memory': 32768,
'stack_size': 8192,
'child_processes': 0,
'open_files': 512,
'file_size': 1024,
}
# Store the input file that student uploaded
def store_input_file(self, inputFile, dest):
file_path = os.path.join(dest, inputFile.name.split('/')[-1])
destination = open(file_path, 'wb+')
# Work-around to support file and InMemoeryUploadedFile objects
if hasattr(inputFile, 'chunks'):
all_file = inputFile.chunks()
else:
all_file = inputFile
# Write everything to the input file represented by destination object. Then return the full path of file name.
for chunk in all_file:
destination.write(chunk)
destination.close()
return file_path
# Function to setup the environment (input file, solution files and submission files) before running the code.
def setup(self):
# This is the directory where the student program is copied to.
self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir)
os.chdir(self.temp_dir)
# Copy student solution files to tmp directory.
extract_or_copy(src=self.submission.filePath.file.name,
dest=self.temp_dir)
# Another temp directory for running solution program. This is the directory where the solution program is copied to.
self.solution_temp_dir = tempfile.mkdtemp(prefix="solution",
dir=self.eval_dir)
directories = [a for a in os.listdir('./') if os.path.isdir(a)]
if directories:
os.chdir(directories[0])
currentDir = os.getcwd()
# Extract the program file and the input file to the student directory, and the input file to the program directory as well.
extract_or_copy(src=self.program.program_files.file.name,
dest=currentDir)
self.submittedFiles = get_file_name_list(
name=self.submission.filePath.file.name)
input_file = self.store_input_file(self.inputFile, currentDir)
shutil.copy(src=input_file, dst=self.solution_temp_dir)
# Function to compile the student program.
def compile(self):
self.exe_file = str(self.program.id)
compiler_command = " ".join(pickle.loads(
self.program.compiler_command)) + " -o " + self.exe_file
self.compileResult = self.commandExecutor.executeCommand(
command=compiler_command)
# Function to run the student and model solution on the input file.
def run(self):
# Run student exe file.
_, self.actual_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.actual_stdout, 0777)
runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name
self.actualOutput = self.commandExecutor.safe_execute(
command=runStudentProgram, limits=self.get_resource_limit())
# Run solution exe file.
shutil.copy(src=self.program.exe_file_name.file.name,
dst=self.solution_temp_dir)
os.chdir(self.solution_temp_dir)
_, self.expected_stdout = tempfile.mkstemp(dir='./')
os.chmod(self.expected_stdout, 0777)
exe = self.program.exe_file_name.name.split('/')[-1]
runSolutionProgram = "./" + exe + " < " + self.inputFile.name
self.expectedOutput = self.commandExecutor.safe_execute(
command=runSolutionProgram, limits=self.get_resource_limit())
self.passed = self.testPassed()
# This function is called after run(), and checks the actual and the expected output.
def testPassed(self):
# At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout.
# compare them and display result on browser.
if not (self.expectedOutput.returnCode == 0
and self.actualOutput.returnCode == 0):
return False
exp_op = iter(self.expectedOutput.get_stdout())
act_op = iter(self.actualOutput.get_stdout())
if len(self.expectedOutput.get_stdout()) > len(
self.actualOutput.get_stdout()):
return False
for line1, line2 in zip(exp_op, act_op):
if line1.strip() != line2.strip():
return False
for line2 in act_op:
if line2.strip() != "":
return False
return True
# This is the function called to get the result once the input file is uploaded.
def getResult(self):
# Setup the environment.
self.setup()
# Compile the program.
self.compile()
# If compilation is successful then run the program.
if self.compileResult.returnCode == 0:
self.run()
# Clean up the temporary directories.
self.cleanUp()
return self
# Function to clean up the temporary directories.
def cleanUp(self):
shutil.rmtree(self.temp_dir, ignore_errors=True)
