def run_command(self, cmd, env): [output, error, exitcode, timed_out, oom_ed] = \ execute_arglist(cmd, env.tmpdir(), timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE) logger.debug("output: " + output) logger.debug("exitcode: " + str(exitcode)) if error != None and len(error) > 0: logger.debug("error: " + error) output = output + error result = self.create_result(env) if timed_out or oom_ed: # ERROR: Execution timed out logger.error('Execution timeout') # clear log for timeout # because truncating log will result in invalid XML. truncated = False output = '\Execution timed out... (Check for infinite loop in your code)\r\n' + output (output, truncated) = truncated_log(output) # Do not set timout flag in order to handle timeout only as failed testcase. # Student shall be motivated to look for error in his or her code and not in testcode. result.set_log(output, timed_out=timed_out, truncated=truncated, oom_ed=oom_ed, log_format=CheckerResult.TEXT_LOG) result.set_passed(False) return result, output if exitcode != 0: (output, truncated) = truncated_log(output) if exitcode < 0: # append signal message output = output + '\r\nSignal:\r\n' + signal.strsignal(- exitcode) result.set_log(output, timed_out=False, truncated=truncated, oom_ed=False, log_format=CheckerResult.TEXT_LOG) result.set_passed(False) return result, output result.set_passed(True) return result, output
def run(self, env): java_builder = IgnoringJavaBuilder(_flags="", _libs=self.junit_version, _file_pattern=r"^.*\.[jJ][aA][vV][aA]$", _output_flags="", _main_required=False) java_builder._ignore = self.ignore.split(" ") build_result = java_builder.run(env) if not build_result.passed: result = self.create_result(env) result.set_passed(False) result.set_log('<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/>\n'+build_result.log) return result environ = {} environ['UPLOAD_ROOT'] = settings.UPLOAD_ROOT environ['JAVA'] = settings.JVM script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') environ['POLICY'] = os.path.join(script_dir, "junit.policy") cmd = [settings.JVM_SECURE, "-cp", settings.JAVA_LIBS[self.junit_version]+":.", self.runner(), self.class_name] [output, error, exitcode, timed_out, oom_ed] = execute_arglist(cmd, env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs=[script_dir]) result = self.create_result(env) (output, truncated) = truncated_log(output) output = '<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + escape(output) + '</pre>' result.set_log(output, timed_out=timed_out or oom_ed, truncated=truncated, oom_ed=oom_ed) result.set_passed(not exitcode and not timed_out and not oom_ed and self.output_ok(output) and not truncated) return result
def run(self, env): java_builder = IgnoringJavaBuilder( _flags="", _libs=self.junit_version, _file_pattern=r"^.*\.[jJ][aA][vV][aA]$", _output_flags="") java_builder._ignore = self.ignore.split(" ") build_result = java_builder.run(env) if not build_result.passed: result = self.create_result(env) result.set_passed(False) result.set_log( '<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/>\n' + build_result.log) return result environ = {} environ['UPLOAD_ROOT'] = settings.UPLOAD_ROOT environ['JAVA'] = settings.JVM script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') environ['POLICY'] = os.path.join(script_dir, "junit.policy") cmd = [ settings.JVM_SECURE, "-cp", settings.JAVA_LIBS[self.junit_version] + ":.", self.runner(), self.class_name ] [output, error, exitcode, timed_out, oom_ed] = execute_arglist(cmd, working_directory=env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs=[script_dir]) result = self.create_result(env) (output, truncated) = truncated_log(output) output = '<pre>' + escape( self.test_description ) + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + escape( output) + '</pre>' result.set_log(output, timed_out=timed_out or oom_ed, truncated=truncated, oom_ed=oom_ed) result.set_passed(not exitcode and not timed_out and not oom_ed and self.output_ok(output) and not truncated) return result
def run(self, env): """ Runs tests in a special environment. Here's the actual work. This runs the check in the environment ENV, returning a CheckerResult. """ # Setup filename = self.filename if self.filename else self.shell_script.path path = os.path.join(env.tmpdir(), os.path.basename(filename)) copy_file(self.shell_script.path, path) os.chmod(path, 0750) # Run the tests -- execute dumped shell script 'script.sh' filenames = [name for (name, content) in env.sources()] args = [path] + filenames environ = {} environ['USER'] = str(env.user().id) environ['USER_MATR'] = str(env.user().mat_number) environ['SOLUTION_ID'] = str(env.solution().id) environ['HOME'] = env.tmpdir() environ['JAVA'] = settings.JVM environ['JAVA_SECURE'] = settings.JVM_SECURE environ['SCALA'] = settings.SCALA environ['POLICY'] = settings.JVM_POLICY environ['PROGRAM'] = env.program() or '' script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') [output, error, exitcode, timed_out, oom_ed] = execute_arglist( args, working_directory=env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, maxmem=settings.TEST_MAXMEM, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs=[script_dir], ) output = force_unicode(output, errors='replace') result = self.create_result(env) (output, truncated) = truncated_log(output) if self.remove: output = re.sub(self.remove, "", output) if not self.returns_html or truncated or timed_out or oom_ed: output = '<pre>' + escape(output) + '</pre>' result.set_log(output, timed_out=timed_out, truncated=truncated, oom_ed=oom_ed) result.set_passed(not exitcode and not timed_out and not oom_ed and not truncated) return result
def run(self, env): self.copy_files(env) # Save save check configuration config_path = os.path.join(env.tmpdir(), "checks.xml") copy_file(self.configuration.path, config_path) # Run the tests # tests are run unsafe because checkstyle fails when network is missing args = [settings.JVM, "-cp", settings.CHECKSTYLE_VER[self.check_version], "-Dbasedir=.", "com.puppycrawl.tools.checkstyle.Main", "-c", "checks.xml"] + \ [name for (name, content) in env.sources()] # + [" > ", env.tmpdir() + "/output.txt"] [output, error, exitcode, timed_out, oom_ed] = execute_arglist(args, env.tmpdir(), unsafe=True) # Remove Praktomat-Path-Prefixes from result: output = re.sub(r"" + re.escape(env.tmpdir() + "/") + "+", "", output, flags=re.MULTILINE) warnings = str.count(output, '[WARN]') errors = str.count(output, '[ERROR]') result = self.create_result(env) (output, truncated) = truncated_log(output) # logger.debug('Exitcode is ' + str(exitcode)) if ProFormAChecker.retrieve_subtest_results: # simply use plaintext result.set_log(output, timed_out=timed_out, truncated=False, oom_ed=oom_ed, log_format=CheckerResult.TEXT_LOG) regexp = '\[(?<msgtype>[A-Z]+)\]\s(?<filename>\/?(.+\/)*(.+)\.([^\s:]+)):(?<line>[0-9]+)(:(?<column>[0-9]+))?:\s(?<text>.+\.)\s\[(?<short>\w+)\]' result.set_regexp(regexp) else: # old handling (e.g. for LON-CAPA) log = '<pre>' + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + \ escape(output) + '</pre>' # log = '<pre>' + escape(output) + '</pre>' if timed_out: log = log + '<div class="error">Timeout occured!</div>' if oom_ed: log = log + '<div class="error">Out of memory!</div>' result.set_log(log) result.set_passed(not timed_out and not oom_ed and not exitcode and warnings <= self.allowedWarnings and errors <= self.allowedErrors and not truncated) # result.set_passed(not timed_out and not oom_ed and not exitcode and (not re.match('Starting audit...\nAudit done.', output) == None)) return result
def run(self, env): """ Runs tests in a special environment. Here's the actual work. This runs the check in the environment ENV, returning a CheckerResult. """ # Setup filename = self.filename if self.filename else self.shell_script.path path = os.path.join(env.tmpdir(),os.path.basename(filename)) copy_file(self.shell_script.path, path) os.chmod(path,0750) # Run the tests -- execute dumped shell script 'script.sh' filenames = [name for (name,content) in env.sources()] args = [path] + filenames environ = {} environ['USER'] = str(env.user().id) environ['USER_MATR'] = str(env.user().mat_number) environ['SOLUTION_ID'] = str(env.solution().id) environ['HOME'] = env.tmpdir() environ['JAVA'] = settings.JVM environ['JAVA_SECURE'] = settings.JVM_SECURE environ['SCALA'] = settings.SCALA environ['POLICY'] = settings.JVM_POLICY environ['PROGRAM'] = env.program() or '' script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)),'scripts') [output, error, exitcode,timed_out, oom_ed] = execute_arglist( args, working_directory=env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, maxmem=settings.TEST_MAXMEM, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs = [script_dir], ) output = force_unicode(output, errors='replace') result = self.create_result(env) (output,truncated) = truncated_log(output) if self.remove: output = re.sub(self.remove, "", output) if not self.returns_html or truncated or timed_out or oom_ed: output = '<pre>' + escape(output) + '</pre>' result.set_log(output,timed_out=timed_out,truncated=truncated,oom_ed=oom_ed) result.set_passed(not exitcode and not timed_out and not oom_ed and not truncated) return result
def run(self, env): filecopy_result = self.run_file(env) if not filecopy_result.passed: return filecopy_result if self.require_safe: safe_builder = IgnoringHaskellBuilder(_flags="-XSafe", _file_pattern = r"^.*\.[hH][sS]$", _main_required=False) safe_builder._ignore=self.ignore.split(" ") + [self.path_relative_to_sandbox()] safe_build_result = safe_builder.run(env) if not safe_build_result.passed: result = self.create_result(env) result.set_passed(False) result.set_log('<pre>' + escape(self.test_description) + '\n\n======== Test Results (Safe) ======\n\n</pre><br/>\n'+safe_build_result.log) return result test_builder = TestOnlyBuildingBuilder(_flags="-main-is "+self.module_name(), _libs="test-framework test-framework-quickcheck2 test-framework-hunit") test_builder._testsuite_filename=self.path_relative_to_sandbox() test_build_result = test_builder.run(env) if not test_build_result.passed: result = self.create_result(env) result.set_passed(False) result.set_log('<pre>' + escape(self.test_description) + '\n\n======== Test Results (Building all) ======\n\n</pre><br/>\n'+test_build_result.log) return result environ = {} environ['UPLOAD_ROOT'] = settings.UPLOAD_ROOT cmd = ["./"+self.module_binary_name(), "--maximum-generated-tests=1000"] [output, error, exitcode,timed_out, oom_ed] = execute_arglist(cmd, env.tmpdir(),environment_variables=environ,timeout=settings.TEST_TIMEOUT,fileseeklimit=settings.TEST_MAXFILESIZE) result = self.create_result(env) (output,truncated) = truncated_log(output) output = '<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + escape(output) + '</pre>' if self.include_testcase_in_report in ["FULL","DL"]: testsuit_template = get_template('checker/checker/haskell_test_framework_report.html') output += testsuit_template.render(Context({'showSource' : (self.include_testcase_in_report=="FULL"), 'testfile' : self.file, 'testfilename' : self.path_relative_to_sandbox(), 'testfileContent': encoding.get_unicode(self.file.read())})) result.set_log(output,timed_out=timed_out or oom_ed,truncated=truncated) result.set_passed(not exitcode and not timed_out and not oom_ed and self.output_ok(output) and not truncated) return result
def handle_compile_error(self, env, output, error, timed_out, oom_ed, regexp = None): logger.error("output: " + output) output = self.remove_sandbox_paths(output, env) result = self.create_result(env) result.set_passed(False) if error != None and len(error) > 0: logger.error("error: " + error) output = output + error (output, truncated) = truncated_log(output) output = escape(output) filenames = [name for name in self.get_file_names(env)] submissionfiles = set(filenames).intersection([solutionfile.path() for solutionfile in env.solution().solutionfile_set.all()]) log = self.build_compilation_error_log(output, '', submissionfiles, regexp) result.set_log(log["log"], timed_out=timed_out or oom_ed, truncated=truncated, oom_ed=oom_ed, log_format=log["format"]) # result.set_log(log["log"], log_format=log["format"]) return result
def run(self, env): # copy files and unzip zip file if submission consists of just a zip file. self.prepare_run(env) # compile build_result = self.compile_make(env) if build_result != True: return build_result # remove source code files extensions = ('.c', '.h', '.a', '.o', 'CMakeCache.txt', 'Makefile', 'makefile', 'CMakeLists.txt', 'cmake_install.cmake') self.remove_source_files(env, extensions) # copy shared objects self.copy_shared_objects(env) # run test logger.debug('run ' + self.class_name) cmd = [self.class_name] #[output, error, exitcode, timed_out, oom_ed] = \ # execute_arglist(cmd, env.tmpdir(), timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE) #logger.debug(output) #logger.debug("exitcode: " + str(exitcode)) # get result (result, output) = self.run_command(cmd, env) if not result.passed: # error return result (output, truncated) = truncated_log(output) result.set_log(output, timed_out=False, truncated=truncated, oom_ed=False, log_format=CheckerResult.TEXT_LOG) result.set_passed(self.output_ok(output) and not truncated) return result
def run(self, env): """ Runs tests in a special environment. Here's the actual work. This runs the check in the environment ENV, returning a CheckerResult. """ # Setup # copy files and unzip zip file if submission consists of just a zip file. self.prepare_run(env) test_dir = env.tmpdir() replace = [(u'PROGRAM', env.program())] if env.program() else [] copy_file(self.doctest.path, os.path.join(test_dir, os.path.basename(self.doctest.path))) script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') # copy python interpreter with all its shared libraries into sandbox # since it is not available because of chroot in restrict # todo: make properly copy_file('/usr/bin/python3', test_dir + '/python3') self.copy_shared_objects(env) # todo: make properly # python3 instead of 3.8 and prepare outside checker createpathonlib = "(cd / && tar -cf - usr/lib/python3.8) | (cd " + test_dir + " && tar -xf -)" os.system(createpathonlib) # Run the tests -- execute dumped shell script 'script.sh' cmd = ["./python3", os.path.basename(self.doctest.name), "-v"] environ = dict() environ['USER'] = env.user().get_full_name() environ['HOME'] = test_dir for (name, content) in env.sources(): if self.checkSubmission(content): result = CheckerResult(checker=self) result.set_passed(False) result.set_log( "Bitte überarbeiten Sie Ihren Programmcode: " "Betriebssystem kritische Befehle sind verboten") return result # (output, error, exitcode) = execute(args, working_directory=test_dir, environment_variables=environ) [output, error, exitcode, timed_out, oom_ed] = execute_arglist( cmd, env.tmpdir(), environment_variables=environ, # use_default_user_configuration=True, timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE, # extradirs=[script_dir] ) # cleanup sandbox # todo: make properly shutil... try: os.system('rm -rf ' + test_dir + '/lib') os.system('rm -rf ' + test_dir + '/lib64') os.system('rm -rf ' + test_dir + '/usr/lib') os.system('rm -rf ' + test_dir + '/__pycache__') os.system('rm ' + test_dir + '/python3') except: logger.error('error while cleaning up python sandbox') logger.debug(output) logger.debug(error) result = self.create_result(env) (output, truncated) = truncated_log(output) if self.remove: output = re.sub(self.remove, "", output) # Remove Praktomat-Path-Prefixes from result: output = re.sub(r"^" + re.escape(env.tmpdir()) + "/+", "", output, flags=re.MULTILINE) if ProFormAChecker.retrieve_subtest_results: # plain text output result.set_log(output, timed_out=timed_out, truncated=truncated, log_format=CheckerResult.TEXT_LOG) else: if not self.returns_html: output = '<pre>' + output + '</pre>' output = '<pre>' + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + \ escape(output) + '</pre>' result.set_log(output, timed_out=timed_out, truncated=truncated) result.set_passed(not exitcode and not timed_out and self.output_ok_positiv(output) and not truncated) return result
def run(self, env): self.copy_files(env) # compile test logger.debug('JUNIT Checker build') java_builder = IgnoringJavaBuilder( _flags="", _libs=self.junit_version, _file_pattern=r"^.*\.[jJ][aA][vV][aA]$", _output_flags="", _main_required=False) # add JAR files from test task for file in self.files.all(): if file.file.path.lower().endswith('.jar'): java_builder.add_custom_lib(file) java_builder._ignore = self.ignore.split(" ") build_result = java_builder.run(env) if not build_result.passed: logger.info('could not compile JUNIT test') # logger.debug("log: " + build_result.log) result = self.create_result(env) result.set_passed(False) result.set_log(build_result.log, log_format=(CheckerResult.FEEDBACK_LIST_LOG if ProFormAChecker.retrieve_subtest_results else CheckerResult.NORMAL_LOG)) # result.set_log('<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/>\n'+build_result.log) return result # delete all java files in the sandbox in order to avoid the student getting the test source code :-) [output, error, exitcode, timed_out, oom_ed] = \ execute_arglist(['find', '.' , '-name', '*.java', '-delete'], env.tmpdir(), unsafe=True) if exitcode != 0: logger.error('exitcode for java files deletion :' + str(exitcode)) logger.error(output) logger.error(error) # run test logger.debug('JUNIT Checker run') environ = {} environ['UPLOAD_ROOT'] = settings.UPLOAD_ROOT environ['JAVA'] = settings.JVM script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') environ['POLICY'] = os.path.join(script_dir, "junit.policy") if self.junit_version == 'junit5': # JUNIT5 [cmd, use_run_listener ] = self.get_run_command_junit5(java_builder.libs()[1]) else: # JUNIT4 [cmd, use_run_listener ] = self.get_run_command_junit4(java_builder.libs()[1]) # use Java security manager instead of restrict application [output, error, exitcode, timed_out, oom_ed] = \ execute_arglist(cmd, env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs=[script_dir], unsafe=True) # Remove deprecated warning for Java 17 and security manager output = JUnitChecker.remove_depricated_warning(output) # logger.debug('JUNIT output:' + str(output)) logger.debug('JUNIT error:' + str(error)) logger.debug('JUNIT exitcode:' + str(exitcode)) result = self.create_result(env) truncated = False # show normal console output in case of: # - timeout (created by Checker) # - not using RunListener # - exitcode <> 0 with RunListener (means internal error) if timed_out or oom_ed: # ERROR: Execution timed out logger.error('Execution timeout') if use_run_listener: # clear log for timeout with Run Listener # because truncating log will result in invalid XML. output = '' truncated = False output = '\Execution timed out... (Check for infinite loop in your code)\r\n' + output (output, truncated) = truncated_log(output) # Do not set timout flag in order to handle timeout only as failed testcase. # Student shall be motivated to look for error in his or her code and not in testcode. result.set_log(output, timed_out=False, truncated=truncated, oom_ed=oom_ed, log_format=CheckerResult.TEXT_LOG) result.set_passed(False) return result #import chardet #encoding = chardet.detect(output) #logger.debug('JUNIT output encoding:' + encoding['encoding']) if use_run_listener: # RUN LISTENER if exitcode == 0: # normal detailed results # todo: Unterscheiden zwischen Textlistener (altes Log-Format) und Proforma-Listener (neues Format) result.set_log(output, timed_out=timed_out, truncated=False, oom_ed=oom_ed, log_format=CheckerResult.PROFORMA_SUBTESTS) else: result.set_internal_error(True) # no XML output => truncate (output, truncated) = truncated_log(output) result.set_log("RunListener Error: " + output, timed_out=timed_out, truncated=truncated, oom_ed=oom_ed, log_format=CheckerResult.TEXT_LOG) else: # show standard log output (output, truncated) = truncated_log(output) output = '<pre>' + escape(self.test_description) + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + \ escape(output) + '</pre>' result.set_log(output, timed_out=timed_out or oom_ed, truncated=truncated, oom_ed=oom_ed) result.set_passed(not exitcode and self.output_ok(output) and not truncated) return result
def run(self, env): # Setup self.copy_files(env) test_dir = env.tmpdir() replace = [(u'PROGRAM', env.program())] if env.program() else [] copy_file(self.testFile.path, os.path.join(test_dir, os.path.basename(self.testFile.path))) script_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'scripts') # check: only one submission file allowed result = self.create_result(env) if len(env.sources()) > 1: result.set_log("Sie dürfen nur eine Datei angegeben!") result.set_passed(False) return result # check submission for (name, content) in env.sources(): if not (self.secureSubmission(content)): result.set_passed(False) result.set_log("Bitte keine IO-Befehle verwenden") return result else: #concat test #try: self.conCat(test_dir, content, self.testFile) #except UnicodeEncodeError: # result.set_passed(False) # result.set_log("Special characters can pose a problem. Vermeiden Sie Umlaute im Source Code " # "und verwenden Sie kein <, > oder & in XML Dokumenten.") # return result # complete test cmd = [ settings.JVM, '-cp', settings.SETLXJAR, "org.randoom.setlx.pc.ui.SetlX", "concat.stlx" ] # (output, error, exitcode) = execute(args, env.tmpdir()) environ = {} environ['UPLOAD_ROOT'] = settings.UPLOAD_ROOT [output, error, exitcode, timed_out, oom_ed] = execute_arglist(cmd, env.tmpdir(), environment_variables=environ, timeout=settings.TEST_TIMEOUT, fileseeklimit=settings.TEST_MAXFILESIZE, extradirs=[script_dir], unsafe=True) # [output, error, exitcode, timed_out] = execute_arglist(cmd, env.tmpdir(), # use_default_user_configuration=True, # timeout=settings.TEST_TIMEOUT, # fileseeklimit=settings.TEST_MAXFILESIZE, # extradirs=[script_dir]) extradirs=[script_dir]) (output, truncated) = truncated_log(output) # Remove Praktomat-Path-Prefixes from result: # output = re.sub(r"^"+re.escape(env.tmpdir())+"/+", "", output, flags=re.MULTILINE) output = re.sub(r"" + re.escape(env.tmpdir() + "/") + "+", "", output, flags=re.MULTILINE) passed = True if len(output.strip()) == 0: output = "no output" passed = False if ProFormAChecker.retrieve_subtest_results: # plain text output if passed and (RXFAIL.search(output) is not None or exitcode): # add regular expression in case of an error regexp = 'line\ (?<line>[0-9]+)(:(?<column>[0-9]+))?\s(?<text>.+)' result.set_regexp(regexp) result.set_log(output, timed_out=timed_out, truncated=truncated, log_format=CheckerResult.TEXT_LOG) else: output = '<pre>' + '\n\n======== Test Results ======\n\n</pre><br/><pre>' + \ escape(output) + '</pre>' result.set_log(output, timed_out=timed_out, truncated=truncated) result.set_passed(passed and not exitcode and not timed_out and (RXFAIL.search(output) is None) and not truncated) return result