class CreateOutput(object): def __init__(self, instance): self.program = instance.program self.testcase = instance self.input_files = instance.input_files self.commandExecutor = CommandExecutor() self.failed = False self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec') # Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features # is also created (with default values if the instructor does not change this). # This function returns the dictionary mapping the attributes to their respective limits for this testcase. def get_resource_limit(self): return {'cpu_time':10, 'clock_time':10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } ''' Solve this by actually retrieving the exact SafeExec object. try: resource = {} safeexec_obj = SafeExec.objects.get(testcase=self.testcase) attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size'] for attr in attrs: val = getattr(safeexec_obj, attr) if val is not None: resource[attr] = val return resource except SafeExec.DoesNotExist: return {'cpu_time':10, 'clock_time':10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } ''' # Setup the output creator def setup(self): # create temporary directory. self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir) os.chdir(self.temp_dir) currentDir = os.getcwd() # Copy program files and input files to the directory if self.program.program_files : extract_or_copy(src=self.program.program_files.file.name, dest=currentDir) if self.program.assignment.model_solution: extract_or_copy(src=self.program.assignment.model_solution.file.name, dest=currentDir) # write input file to temporary file. self.temp_input_d = tempfile.mkdtemp(prefix="input") if self.input_files: self.input = os.path.join( self.temp_input_d, os.path.basename(self.testcase.input_files.file.name) ) f = open(self.input, 'w') for a_line in self.input_files.file: f.write(a_line) f.close() # input file has been written now extract. extract_or_copy(src=self.input, dest=currentDir) else: self.input = '' # Run the program solution files and def run(self): # Compile the solution. If there are errors then write it to the error file and return if compile_required(self.program.language): compiler_command = get_compilation_command(self.program) print "Compiling model solution. Compilation Command - " + compiler_command self.command_output = self.commandExecutor.safe_execute( command=compiler_command, limits=self.get_resource_limit() ) if self.command_output.getReturnCode(): # There was some error. Write it in database. print "Compilation of model solution failed." self.failed = True _, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d) f = open(self.error_file, 'w') for a_line in self.command_output.get_stderr(): f.write(a_line) f.close() return # No errors and thus can continue. Run the solution execution_command = get_execution_command(self.program) print "Creating Output. Running following execution command - " + execution_command #if self.testcase.command_line_args: # execution_command = execution_command + self.testcase.command_line_args if self.testcase.std_in_file_name: execution_command = execution_command + " < " + self.testcase.std_in_file_name self.command_output = self.commandExecutor.safe_execute( command=execution_command, limits=self.get_resource_limit() ) self.success = bool(self.command_output.getReturnCode()) self.command_output.printResult() # Delete input files from current directory. And the program files dir_content = os.listdir('./') for a_file in dir_content: if self.is_program_file(a_file): os.remove(a_file) if self.input: for a_file in get_file_name_list(name=self.input): # delete extracted files if os.path.isfile(a_file): os.remove(a_file) # Write standard output to a file and save it in database. directory_content = os.listdir('./') _, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./") out_file = open(self.std_out_file, 'w') for a_line in ["\n".join(self.command_output.get_stdout())]: out_file.write(a_line) out_file.close() # There was some error. Write it in database. if self.command_output.getReturnCode(): self.failed = True _, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d) f = open(self.error_file, 'w') for a_line in self.command_output.get_stderr(): f.write(a_line) f.close() # If directory has any content left then make a tar, else set the outfile to the output file. if directory_content: # make a tar self.out_file = self.make_tar(directory_content + [self.std_out_file], self.std_out_file) else: # there are no other files standard output only. self.out_file = self.std_out_file def get_stdout_file_name(self): return os.path.basename(self.std_out_file) def get_stderr_file_name(self): return os.path.basename(self.error_file) def get_stderr_file_path(self): return self.error_file def make_tar(self, files, tarname): tarname = tarname + ".tar.bz2" output_files_tar = tarfile.open(name=tarname, mode="w:bz2") # Make tar file from all output files. for afile in files: if os.path.isfile(afile): output_files_tar.add(os.path.basename(afile)) output_files_tar.close() return tarname def get_output(self): prev_dir = os.getcwd() try: self.setup() self.run() finally: os.chdir(prev_dir) return self def is_program_file(self, fname): filename = fname.split("/")[-1] if self.program.program_files and filename in get_file_name_list(self.program.program_files.file.name): return True elif self.program.assignment.model_solution and filename in get_file_name_list(self.program.assignment.model_solution.file.name): return True elif is_intermediate_files(filename,self.program,self.program.language): return True else: return False def clean_up(self): shutil.rmtree(self.temp_dir, ignore_errors=True) shutil.rmtree(self.temp_input_d, ignore_errors=True)
class CustomTestcase(object): def __init__(self, inputFile, program, submission): # Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands. self.program = program self.submission = submission self.inputFile = inputFile self.commandExecutor = CommandExecutor() self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec') def get_resource_limit(self): # Return default values because there is no setting for custom testcase. return {'cpu_time':10, 'clock_time':10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } # Store the input file that student uploaded def store_input_file(self, inputFile, dest): file_path = os.path.join(dest, inputFile.name.split('/')[-1]) destination = open(file_path, 'wb+') # Work-around to support file and InMemoeryUploadedFile objects if hasattr(inputFile, 'chunks'): all_file = inputFile.chunks() else: all_file = inputFile # Write everything to the input file represented by destination object. Then return the full path of file name. for chunk in all_file: destination.write(chunk) destination.close() return file_path # Function to setup the environment (input file, solution files and submission files) before running the code. def setup(self): # This is the directory where the student program is copied to. self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir) os.chdir(self.temp_dir) # Copy student solution files to tmp directory. extract_or_copy(src=self.submission.filePath.file.name, dest=self.temp_dir) # Another temp directory for running solution program. This is the directory where the solution program is copied to. self.solution_temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir) directories = [a for a in os.listdir('./') if os.path.isdir(a)] if directories: os.chdir(directories[0]) currentDir = os.getcwd() # Extract the program file and the input file to the student directory, and the input file to the program directory as well. extract_or_copy(src=self.program.program_files.file.name, dest=currentDir) self.submittedFiles = get_file_name_list(name=self.submission.filePath.file.name) input_file = self.store_input_file(self.inputFile, currentDir) shutil.copy(src=input_file, dst=self.solution_temp_dir) # Function to compile the student program. def compile(self): self.exe_file = str(self.program.id) compiler_command = " ".join(pickle.loads(self.program.compiler_command)) + " -o " + self.exe_file self.compileResult = self.commandExecutor.executeCommand(command=compiler_command) # Function to run the student and model solution on the input file. def run(self): # Run student exe file. _, self.actual_stdout = tempfile.mkstemp(dir='./') os.chmod(self.actual_stdout, 0777) runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name self.actualOutput = self.commandExecutor.safe_execute( command=runStudentProgram, limits=self.get_resource_limit() ) # Run solution exe file. shutil.copy(src=self.program.exe_file_name.file.name, dst=self.solution_temp_dir) os.chdir(self.solution_temp_dir) _, self.expected_stdout = tempfile.mkstemp(dir='./') os.chmod(self.expected_stdout, 0777) exe = self.program.exe_file_name.name.split('/')[-1] runSolutionProgram = "./" + exe + " < " + self.inputFile.name self.expectedOutput = self.commandExecutor.safe_execute( command=runSolutionProgram, limits=self.get_resource_limit() ) self.passed = self.testPassed() # This function is called after run(), and checks the actual and the expected output. def testPassed(self): # At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout. # compare them and display result on browser. if not (self.expectedOutput.returnCode == 0 and self.actualOutput.returnCode == 0): return False exp_op = iter(self.expectedOutput.get_stdout()) act_op = iter(self.actualOutput.get_stdout()) if len(self.expectedOutput.get_stdout()) > len(self.actualOutput.get_stdout()): return False for line1, line2 in zip(exp_op, act_op): if line1.strip() != line2.strip(): return False for line2 in act_op: if line2.strip() != "": return False return True # This is the function called to get the result once the input file is uploaded. def getResult(self): # Setup the environment. self.setup() # Compile the program. self.compile() # If compilation is successful then run the program. if self.compileResult.returnCode == 0: self.run() # Clean up the temporary directories. self.cleanUp() return self # Function to clean up the temporary directories. def cleanUp(self): shutil.rmtree(self.temp_dir, ignore_errors=True)
class CreateOutput(object): def __init__(self, instance): self.program = instance.program self.testcase = instance self.input_files = instance.input_files self.commandExecutor = CommandExecutor() self.failed = False self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec') # Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features # is also created (with default values if the instructor does not change this). # This function returns the dictionary mapping the attributes to their respective limits for this testcase. def get_resource_limit(self): return { 'cpu_time': 10, 'clock_time': 10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } ''' Solve this by actually retrieving the exact SafeExec object. try: resource = {} safeexec_obj = SafeExec.objects.get(testcase=self.testcase) attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size'] for attr in attrs: val = getattr(safeexec_obj, attr) if val is not None: resource[attr] = val return resource except SafeExec.DoesNotExist: return {'cpu_time':10, 'clock_time':10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } ''' # Setup the output creator def setup(self): # create temporary directory. self.temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir) os.chdir(self.temp_dir) currentDir = os.getcwd() # Copy program files and input files to the directory if self.program.program_files: extract_or_copy(src=self.program.program_files.file.name, dest=currentDir) if self.program.assignment.model_solution: extract_or_copy( src=self.program.assignment.model_solution.file.name, dest=currentDir) # write input file to temporary file. self.temp_input_d = tempfile.mkdtemp(prefix="input") if self.input_files: self.input = os.path.join( self.temp_input_d, os.path.basename(self.testcase.input_files.file.name)) f = open(self.input, 'w') for a_line in self.input_files.file: f.write(a_line) f.close() # input file has been written now extract. extract_or_copy(src=self.input, dest=currentDir) else: self.input = '' # Run the program solution files and def run(self): # Compile the solution. If there are errors then write it to the error file and return if compile_required(self.program.language): compiler_command = get_compilation_command(self.program) print "Compiling model solution. Compilation Command - " + compiler_command self.command_output = self.commandExecutor.safe_execute( command=compiler_command, limits=self.get_resource_limit()) if self.command_output.getReturnCode(): # There was some error. Write it in database. print "Compilation of model solution failed." self.failed = True _, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d) f = open(self.error_file, 'w') for a_line in self.command_output.get_stderr(): f.write(a_line) f.close() return # No errors and thus can continue. Run the solution execution_command = get_execution_command(self.program) print "Creating Output. Running following execution command - " + execution_command #if self.testcase.command_line_args: # execution_command = execution_command + self.testcase.command_line_args if self.testcase.std_in_file_name: execution_command = execution_command + " < " + self.testcase.std_in_file_name self.command_output = self.commandExecutor.safe_execute( command=execution_command, limits=self.get_resource_limit()) self.success = bool(self.command_output.getReturnCode()) self.command_output.printResult() # Delete input files from current directory. And the program files dir_content = os.listdir('./') for a_file in dir_content: if self.is_program_file(a_file): os.remove(a_file) if self.input: for a_file in get_file_name_list( name=self.input): # delete extracted files if os.path.isfile(a_file): os.remove(a_file) # Write standard output to a file and save it in database. directory_content = os.listdir('./') _, self.std_out_file = tempfile.mkstemp(prefix='output', dir="./") out_file = open(self.std_out_file, 'w') for a_line in ["\n".join(self.command_output.get_stdout())]: out_file.write(a_line) out_file.close() # There was some error. Write it in database. if self.command_output.getReturnCode(): self.failed = True _, self.error_file = tempfile.mkstemp(prefix="error", dir=self.temp_input_d) f = open(self.error_file, 'w') for a_line in self.command_output.get_stderr(): f.write(a_line) f.close() # If directory has any content left then make a tar, else set the outfile to the output file. if directory_content: # make a tar self.out_file = self.make_tar( directory_content + [self.std_out_file], self.std_out_file) else: # there are no other files standard output only. self.out_file = self.std_out_file def get_stdout_file_name(self): return os.path.basename(self.std_out_file) def get_stderr_file_name(self): return os.path.basename(self.error_file) def get_stderr_file_path(self): return self.error_file def make_tar(self, files, tarname): tarname = tarname + ".tar.bz2" output_files_tar = tarfile.open(name=tarname, mode="w:bz2") # Make tar file from all output files. for afile in files: if os.path.isfile(afile): output_files_tar.add(os.path.basename(afile)) output_files_tar.close() return tarname def get_output(self): prev_dir = os.getcwd() try: self.setup() self.run() finally: os.chdir(prev_dir) return self def is_program_file(self, fname): filename = fname.split("/")[-1] if self.program.program_files and filename in get_file_name_list( self.program.program_files.file.name): return True elif self.program.assignment.model_solution and filename in get_file_name_list( self.program.assignment.model_solution.file.name): return True elif is_intermediate_files(filename, self.program, self.program.language): return True else: return False def clean_up(self): shutil.rmtree(self.temp_dir, ignore_errors=True) shutil.rmtree(self.temp_input_d, ignore_errors=True)
class TestCase(object): def __init__(self, testcase, program_result): self.testcase = testcase self.program_result = program_result self.name = testcase.name self.stdInFile = str(testcase.std_in_file_name) self.commandExecutor = CommandExecutor() # Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features # is also created (with default values if the instructor does not change this). # This function returns the dictionary mapping the attributes to their respective limits for this testcase. def get_resource_limit(self): try: resource = {} safeexec_obj = SafeExec.objects.get(testcase=self.testcase) attrs = ['cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size'] for attr in attrs: val = getattr(safeexec_obj, attr) if val is not None: resource[attr] = val return resource except SafeExec.DoesNotExist: return {} # Function to make a tarfile of all output files and return the name of this file. def make_tar(self): # Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id" stdoutFileName = "standardOutput_" + str(self.testcase.id) shutil.move(self.std_out, stdoutFileName) outputfiles = [stdoutFileName] # Set the name of the tarfile which is going to contain the output files. tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2" outputFilesTar = tarfile.open(name=tarname, mode="w:bz2") # Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution # wrote to other than the std output) to the tarfile. for afile in outputfiles + self.output_files: if os.path.isfile(afile): outputFilesTar.add(afile) outputFilesTar.close() return tarname # Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section # level itself. def run(self, exeCommand): # Create the output file to capture the std output in the current directory. _, self.std_out = tempfile.mkstemp() os.chmod(self.std_out, 0777) command = exeCommand # Add the command line arguments if any. if self.testcase.command_line_args: command = command + self.testcase.command_line_args # Then redirect the std input to the input file. if os.path.isfile(self.stdInFile): command = command + " < " + self.stdInFile # Execute the resulting command with the appropriate resource limits and the output file created above. self.testResults = self.commandExecutor.safe_execute( command=command, limits=self.get_resource_limit(), outfile=self.std_out ) # Check if the testcase has passed. self.passed = self.test_passed() # Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase. attrs = {'error_messages': "".join(self.testResults.stderr), 'return_code': int(self.testResults.returnCode), 'test_passed': self.passed} # This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above. filter_attrs = {'test_case': self.testcase, 'program_result': self.program_result} # If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes. # Remove the older output file. # Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary. try: obj = TestcaseResult.objects.filter(**filter_attrs) obj.update(**attrs) test_result = obj[0:1].get() default_storage.delete(test_result.output_files.path) # Remove older file. except TestcaseResult.DoesNotExist: attrs.update(filter_attrs) test_result = TestcaseResult.objects.create(**attrs) # Make a tar file of the std output file and the other files which have been written to by the student submission. # Update the test result (updated or newly created) with this tar file as the output file. tar_name = self.make_tar() test_result.output_files.save(tar_name, File(open(tar_name))) return self # Function to check if the student submission has passed the testcase. def test_passed(self): # Create a temporary directory and copy the expected output of the testcase to this file. temp_dir = tempfile.mkdtemp(prefix="grader_op_files") extract_or_copy(src=self.testcase.output_files.file.name, dest=temp_dir) std_output_file = str(self.testcase.std_out_file_name) # Other output files are stored in this variable. self.output_files = os.listdir(temp_dir) self.output_files.remove(std_output_file) # Now check for the status of the testcase. Call compare_output(). is_passed = False print "Testcase return code is - ", self.testResults.returnCode if self.testResults.returnCode == 0: # Exited normally. is_passed = self.compare_output(temp_dir, std_output_file) # Remove the temporary directory and return the status of the testcase. shutil.rmtree(temp_dir, ignore_errors=True) return is_passed # Function to compare output of the standard output file and the actual output. # Takes as input the actual output file and expected output in that order. def compare_output(self, temp_dir, std_output_file): if not self.compare_file(os.path.join(temp_dir, std_output_file), self.std_out): return False for afile in self.output_files: # compare all O/P files. if not self.compare_file(afile, afile.split('/')[-1]): return False return True # Function to compare 2 files. # Takes as input the expected output and actual output file in that order. def compare_file(self, expectedop, actualop): # If the actual output is not valid return false. if not os.path.isfile(actualop): return False # Open both the files and see if they have the same length. If true proceed else return false. file1 = open(expectedop) file2 = open(actualop) if(len(list(file1)) > len(list(file2))): return False file1 = open(expectedop) file2 = open(actualop) # Compare the 2 files line by line. for line1, line2 in zip(file1, file2): if line1.strip() != line2.strip(): return False for line2 in file2: if line2.strip() != "": return False return True
class TestCase(object): def __init__(self, testcase, program_result): self.testcase = testcase self.program_result = program_result self.name = testcase.name self.stdInFile = str(testcase.std_in_file_name) self.commandExecutor = CommandExecutor() # Function to get resource limit for the testcase. During the creation of the testcase a Resource limit object with limits for various features # is also created (with default values if the instructor does not change this). # This function returns the dictionary mapping the attributes to their respective limits for this testcase. def get_resource_limit(self): try: resource = {} safeexec_obj = SafeExec.objects.get(testcase=self.testcase) attrs = [ 'cpu_time', 'clock_time', 'memory', 'stack_size', 'child_processes', 'open_files', 'file_size' ] for attr in attrs: val = getattr(safeexec_obj, attr) if val is not None: resource[attr] = val return resource except SafeExec.DoesNotExist: return {} # Function to make a tarfile of all output files and return the name of this file. def make_tar(self): # Create the file containing std output with a certain pattern in its name. "standardOutput_testcase-id" stdoutFileName = "standardOutput_" + str(self.testcase.id) shutil.move(self.std_out, stdoutFileName) outputfiles = [stdoutFileName] # Set the name of the tarfile which is going to contain the output files. tarname = "output_file_" + str(self.testcase.id) + ".tar.bz2" outputFilesTar = tarfile.open(name=tarname, mode="w:bz2") # Add the std output file and all the other files that were created by the student solution (typically the files to which the student solution # wrote to other than the std output) to the tarfile. for afile in outputfiles + self.output_files: if os.path.isfile(afile): outputFilesTar.add(afile) outputFilesTar.close() return tarname # Function which is going to run the execution command on the input file of the testcase. Note that compilation if needed is done at the section # level itself. def run(self, exeCommand): # Create the output file to capture the std output in the current directory. _, self.std_out = tempfile.mkstemp() os.chmod(self.std_out, 0777) command = exeCommand # Add the command line arguments if any. if self.testcase.command_line_args: command = command + self.testcase.command_line_args # Then redirect the std input to the input file. if os.path.isfile(self.stdInFile): command = command + " < " + self.stdInFile # Execute the resulting command with the appropriate resource limits and the output file created above. self.testResults = self.commandExecutor.safe_execute( command=command, limits=self.get_resource_limit(), outfile=self.std_out) # Check if the testcase has passed. self.passed = self.test_passed() # Incase the testcase has not passed the attrs dictionary below holds needed error messages and return code for the testcase. attrs = { 'error_messages': "".join(self.testResults.stderr), 'return_code': int(self.testResults.returnCode), 'test_passed': self.passed } # This dictionary is used to filter out any existing testcase result for the submission so as to update it with the new results in attrs above. filter_attrs = { 'test_case': self.testcase, 'program_result': self.program_result } # If any Testcase Results exist with the filter_attrs attributes, then update if with the new attributes. # Remove the older output file. # Else if none exist, then store the current result with the filter_attrs added to the attrs dictionary. try: obj = TestcaseResult.objects.filter(**filter_attrs) obj.update(**attrs) test_result = obj[0:1].get() default_storage.delete( test_result.output_files.path) # Remove older file. except TestcaseResult.DoesNotExist: attrs.update(filter_attrs) test_result = TestcaseResult.objects.create(**attrs) # Make a tar file of the std output file and the other files which have been written to by the student submission. # Update the test result (updated or newly created) with this tar file as the output file. tar_name = self.make_tar() test_result.output_files.save(tar_name, File(open(tar_name))) return self # Function to check if the student submission has passed the testcase. def test_passed(self): # Create a temporary directory and copy the expected output of the testcase to this file. temp_dir = tempfile.mkdtemp(prefix="grader_op_files") extract_or_copy(src=self.testcase.output_files.file.name, dest=temp_dir) std_output_file = str(self.testcase.std_out_file_name) # Other output files are stored in this variable. self.output_files = os.listdir(temp_dir) self.output_files.remove(std_output_file) # Now check for the status of the testcase. Call compare_output(). is_passed = False print "Testcase return code is - ", self.testResults.returnCode if self.testResults.returnCode == 0: # Exited normally. is_passed = self.compare_output(temp_dir, std_output_file) # Remove the temporary directory and return the status of the testcase. shutil.rmtree(temp_dir, ignore_errors=True) return is_passed # Function to compare output of the standard output file and the actual output. # Takes as input the actual output file and expected output in that order. def compare_output(self, temp_dir, std_output_file): if not self.compare_file(os.path.join(temp_dir, std_output_file), self.std_out): return False for afile in self.output_files: # compare all O/P files. if not self.compare_file(afile, afile.split('/')[-1]): return False return True # Function to compare 2 files. # Takes as input the expected output and actual output file in that order. def compare_file(self, expectedop, actualop): # If the actual output is not valid return false. if not os.path.isfile(actualop): return False # Open both the files and see if they have the same length. If true proceed else return false. file1 = open(expectedop) file2 = open(actualop) if (len(list(file1)) > len(list(file2))): return False file1 = open(expectedop) file2 = open(actualop) # Compare the 2 files line by line. for line1, line2 in zip(file1, file2): if line1.strip() != line2.strip(): return False for line2 in file2: if line2.strip() != "": return False return True
class CustomTestcase(object): def __init__(self, inputFile, program, submission): # Setting section of the testcase, submission to use, the input file given by the student and the command executor that runs the commands. self.program = program self.submission = submission self.inputFile = inputFile self.commandExecutor = CommandExecutor() self.eval_dir = os.path.join(PROJECT_ROOT, '../evaluate/safeexec') def get_resource_limit(self): # Return default values because there is no setting for custom testcase. return { 'cpu_time': 10, 'clock_time': 10, 'memory': 32768, 'stack_size': 8192, 'child_processes': 0, 'open_files': 512, 'file_size': 1024, } # Store the input file that student uploaded def store_input_file(self, inputFile, dest): file_path = os.path.join(dest, inputFile.name.split('/')[-1]) destination = open(file_path, 'wb+') # Work-around to support file and InMemoeryUploadedFile objects if hasattr(inputFile, 'chunks'): all_file = inputFile.chunks() else: all_file = inputFile # Write everything to the input file represented by destination object. Then return the full path of file name. for chunk in all_file: destination.write(chunk) destination.close() return file_path # Function to setup the environment (input file, solution files and submission files) before running the code. def setup(self): # This is the directory where the student program is copied to. self.temp_dir = tempfile.mkdtemp(prefix="grader", dir=self.eval_dir) os.chdir(self.temp_dir) # Copy student solution files to tmp directory. extract_or_copy(src=self.submission.filePath.file.name, dest=self.temp_dir) # Another temp directory for running solution program. This is the directory where the solution program is copied to. self.solution_temp_dir = tempfile.mkdtemp(prefix="solution", dir=self.eval_dir) directories = [a for a in os.listdir('./') if os.path.isdir(a)] if directories: os.chdir(directories[0]) currentDir = os.getcwd() # Extract the program file and the input file to the student directory, and the input file to the program directory as well. extract_or_copy(src=self.program.program_files.file.name, dest=currentDir) self.submittedFiles = get_file_name_list( name=self.submission.filePath.file.name) input_file = self.store_input_file(self.inputFile, currentDir) shutil.copy(src=input_file, dst=self.solution_temp_dir) # Function to compile the student program. def compile(self): self.exe_file = str(self.program.id) compiler_command = " ".join(pickle.loads( self.program.compiler_command)) + " -o " + self.exe_file self.compileResult = self.commandExecutor.executeCommand( command=compiler_command) # Function to run the student and model solution on the input file. def run(self): # Run student exe file. _, self.actual_stdout = tempfile.mkstemp(dir='./') os.chmod(self.actual_stdout, 0777) runStudentProgram = "./" + self.exe_file + " < " + self.inputFile.name self.actualOutput = self.commandExecutor.safe_execute( command=runStudentProgram, limits=self.get_resource_limit()) # Run solution exe file. shutil.copy(src=self.program.exe_file_name.file.name, dst=self.solution_temp_dir) os.chdir(self.solution_temp_dir) _, self.expected_stdout = tempfile.mkstemp(dir='./') os.chmod(self.expected_stdout, 0777) exe = self.program.exe_file_name.name.split('/')[-1] runSolutionProgram = "./" + exe + " < " + self.inputFile.name self.expectedOutput = self.commandExecutor.safe_execute( command=runSolutionProgram, limits=self.get_resource_limit()) self.passed = self.testPassed() # This function is called after run(), and checks the actual and the expected output. def testPassed(self): # At this point actual o/p is in self.actual_stdout and expected O/P is in self.expected_stdout. # compare them and display result on browser. if not (self.expectedOutput.returnCode == 0 and self.actualOutput.returnCode == 0): return False exp_op = iter(self.expectedOutput.get_stdout()) act_op = iter(self.actualOutput.get_stdout()) if len(self.expectedOutput.get_stdout()) > len( self.actualOutput.get_stdout()): return False for line1, line2 in zip(exp_op, act_op): if line1.strip() != line2.strip(): return False for line2 in act_op: if line2.strip() != "": return False return True # This is the function called to get the result once the input file is uploaded. def getResult(self): # Setup the environment. self.setup() # Compile the program. self.compile() # If compilation is successful then run the program. if self.compileResult.returnCode == 0: self.run() # Clean up the temporary directories. self.cleanUp() return self # Function to clean up the temporary directories. def cleanUp(self): shutil.rmtree(self.temp_dir, ignore_errors=True)