def RunTestCase(name, lang, type): baseName = util.ExtractName(name) #compile and run compiler = cmdline.LookUpWildCard('metaiv', lang, type, 'compiler') flags = cmdline.LookUpWildCard('metaiv', lang, type, 'cflags') cmd = compiler + " -o driver driver.cc " + name + " " + flags (exitCode, dummy1, dummy2) = util.RunCommand(cmd, 0, "Problem whith compiling") ok = (exitCode == 0) if ok: if not os.path.exists("driver"): report.Error("Driveri binary is not created") (exitCode, stdout, stderr) = util.RunCommand("./driver" + binext, 0, "Error running c++ generated c++ binary", true) ok = (exitCode == 0) #compare results with expected result files if ok: ok = util.WriteFile(baseName + ".res", stdout) else: report.Error("Output", None, stdout, stderr) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: resfile.CompareResult(name, baseName + ".res", resFile, None) return ok
def RunTestCase(name, lang, type, coverageFile): stdout = None stderr = None baseName = util.ExtractName(name) report.setTestCaseName(name) ok = true #run the test cases cmd = cmdline.LookUpWildCard('cpp', lang, 'spec', 'interpreter') cmd = cmd + " -D -P -a -b -R " + coverageFile exitCode = util.RunCommand( cmd, 0, "Possible core dump while interpreting specification.") # See if a result file was created if ok: if not os.path.exists(baseName + ".arg.res"): report.Error("No result file generated for test case " + name, "Maybe it failed before") ok = false if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".arg.res", resFile, None) return ok
def RunSpecTestCases(fullNames, lang, dtc, coverageFile): # remove files we expect output in to. for fullName in fullNames: util.RemoveTestFiles(fullName, [".arg.pt", ".arg.res", ".arg.err", ".arg.msg"]) # run the test cases interpreter = cmdline.LookUpWildCard('ip', lang, 'spec', 'interpreter', dtc) cmd = interpreter + " -a -b -R " + coverageFile exitCode = util.RunCommand(cmd, 0, "Possible core dump while interpreting specification.") okNames = [] # Now validate the results for fullName in fullNames: bn = util.ExtractName(fullName) semResName = bn + ".arg.res" resName = bn + ".res" errName = bn + ".arg.msg" report.setTestCaseName(fullName) # See if a result file was created if not os.path.exists(semResName) and not os.path.exists(errName): report.Error("No result or error file generated for test case " + fullName, "Maybe the interpreting toolbox failed for one of the previous test cases") continue # Find expected result file ok = true expResultFile = resfile.FindResFile(fullName) if expResultFile == None: ok = false if os.path.exists(errName): # See if an error file is generated. data = util.ReadFile(errName) if data == None: continue # Note the order of the next binary expression ensures that # CompareRunTimeError is executed even though no expected result was # found! This is necesary as this function generates the result one # can copy in place ok = CompareRunTimeError(fullName, expResultFile, data) and ok else: # Strip sem values ok = ok and StripSemValue(fullName, lang, dtc) # validate expected result then if ok: interpreter = cmdline.LookUpWildCard('ip', lang, 'spec', 'spec-compare', dtc) ok = resfile.CompareResult(fullName, resName, expResultFile, interpreter) if ok: okNames.append(fullName) return okNames
def CompileRunAndCompare(fullName, lang, type, modules): global packageMap baseName = util.ExtractName(fullName) ok = true if ok: ok = CreateArgFile(fullName, lang, modules) standardlibs = convert.GetStandardLibs() libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir')) for lib in standardlibs: libfile = lib + ".java" util.CopyFile(libdir + "/" + libfile, libfile) if ok: ok = CompileJavaFiles(fullName, lang, type, modules) interpreter = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'interpreter')) if ok: # Execute the binary flags = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'rtflags')) (exitCode, stdout, stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0, "Error running Java " + "generated binary", true) print("java " + flags + " TMAIN run") ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) lib = "" if ok == None: # It was not a runtime error interpreter = lib + cmdline.LookUpWildCard('java', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok
def CompileRunAndCompare(fullName, lang, type, modules): baseName = util.ExtractName(fullName) ok = true stdout = None stderr = None if ok: CreateUserdefFile(modules) if ok: ok = CreateArgFile(fullName, lang, modules) if ok: ok = CompileCPPFiles(fullName, lang, type, modules) if ok: # Execute the binary (exitCode, stdout, stderr) = util.RunCommand("icode", 0, "Error running c++ generated binary", true) ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) else: report.Error("Output", None, stdout, stderr) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) if ok == None: # It was not a runtime error interpreter = cmdline.LookUpWildCard('cpp', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok
def CompareRunTimeError(fullName, expResultFile, output): report.Progress(4, "Searching for run time error.") match = re.search("(Run-Time Error *[0-9]+:.*)\"",output, re.M) if not match: match = re.search("(Run-Time Error *[0-9]+:.*)",output, re.M) if not match: report.Error("While searching for runtime error, no runtime error information found for '" + fullName + "'", "Error running interpreter, maybe syntax error") return false resultFile = util.ExtractName(fullName)+".res" ok = util.WriteFile(resultFile,match.group(1)) if not ok: return false if expResultFile: return resfile.CompareResult(fullName, resultFile, expResultFile, None) else: return false
def RunApiTestCase(name, lang, toolMediator): data = util.ReadFile(name) baseName = util.ExtractName(name) neutralName = re.sub('.*/', '', name) # path neutralized! result = baseName + ":\n" last = "" symTable = {} symTable['GENPATH'] = os.path.expanduser(cmdline.LookUp('api-gen-path')) symTable['SOURCEPATH'] = os.path.expanduser( cmdline.LookUp('api-source-path')) print(symTable['GENPATH']) print(symTable['SOURCEPATH']) commands = data.split("\n") line, test = 0, 0 empty = re.compile('^\s*(#.*)|(\s*)$') interactive_mode = cmdline.LookUp('api-run-interactive') for cmd in commands: print("Command: " + cmd) if interactive_mode == 'yes': print("Press <return>") raw_input() line = line + 1 # don't process empty lines if not empty.match(cmd): test = test + 1 try: cmd = re.sub('\$\$', last, cmd) # replace $$ by last result cmd = re.sub('\s+', ' ', cmd).strip() # replace multiple spaces by one cmd = re.sub('\s*\(\s*', '(', cmd) # delete spaces around parentheses cmd = re.sub('\s*\)\s*', ')', cmd) cmd = re.sub('\s*\[\s*', '[', cmd) # delete spaces around parentheses cmd = re.sub('\s*\]\s*', ']', cmd) cmd = re.sub('\s*,\s*', ',', cmd) # delete spaces around commas cmd = re.sub('\s*=\s*', '=', cmd) # delete spaces around = # find variable assignment m = re.compile('^\$(?P<lval>\w+)=(?P<rval>.*)$').match(cmd) if m != None: lval = m.group('lval') cmd = m.group('rval') else: lval = "" # find variable usages and replace them by value m = re.compile('^.*\$(?P<id>\w+).*$').match(cmd) if m != None: while m != None: var = "\$" + m.group('id') try: repl = symTable[m.group('id')] except KeyError: raise Syntax("Unknown variable name $" + var) cmd = re.sub(var, repl, cmd) m = re.compile('^.*\$(?P<id>\w+).*$').match(cmd) result = result + "-- " + str(test) + "(in line " + str( line) + "):\n" # print (cmd) # parse and execute the command last = toolMediator.ParseCommand(cmd, name, line) lines = last.split('\n') pathexp = re.compile('^.*/(.*\.(vdm|vpp|api))$') for l in lines: # neutralize path result = result + pathexp.sub(r'\1', l) + "\n" last = re.sub('\n$', '', last) last = re.sub('\n', ',', last).strip() if lval != "": symTable[lval] = last except CORBA.COMM_FAILURE: raise CORBA.COMM_FAILURE except CORBA.BAD_PARAM: result = result + "CORBA.BAD_PARAM in " + neutralName + ", line " + str( line) + "\n" except ToolboxAPI.APIError: _, e, _ = sys.exc_info() result = result + "APIError in " + neutralName + ", line " + str( line) + ": " + e.msg + "\n" except VDM.VDMError: _, e, _ = sys.exc_info() result = result + "VDMError in " + neutralName + ", line " + str( line) + ": " + str(e.err) + "\n" except Syntax: _, e, _ = sys.exc_info() result = result + "Syntax error in " + neutralName + ", line " + str( line) + ": " + e.GetMessage() + "\n" except CORBA.UserException: result = result + "CORBA.UserException: in " + baseName + ".api, line " + str( line) + "\n" except CORBA.SystemException: _, e, _ = sys.exc_info() print(e) result = result + "CORBA.SystemException: in " + baseName + ".api, line " + str( line) + "\n" except Exception: _, e, _ = sys.exc_info() print("Python exception") print(e.args) result = result + "Python exception in " + baseName + ".api, line " + str( line) + "\n" except: result = result + "Unknown exception in " + baseName + ".api, line " + str( line) + "\n" ok = util.WriteFile(baseName + ".tmp", result) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".tmp", resFile, None) if util.CleanFile(ok): util.DeleteFiles([baseName + ".tmp"]) return ok
result = result + "APIError in " + neutralName + ", line " + ` line ` + ": " + e.msg + "\n" except VDM.VDMError, e: result = result + "VDMError in " + neutralName + ", line " + ` line ` + ": " + ` e.err ` + "\n" except Syntax, e: result = result + "Syntax error in " + neutralName + ", line " + ` line ` + ": " + e.GetMessage( ) + "\n" except CORBA.UserException: result = result + "CORBA.UserException: in " + baseName + ".api, line " + ` line ` + "\n" except CORBA.SystemException, e: print e result = result + "CORBA.SystemException: in " + baseName + ".api, line " + ` line ` + "\n" except Exception, e: print "Python exception" print e.args result = result + "Python exception in " + baseName + ".api, line " + ` line ` + "\n" except: result = result + "Unknown exception in " + baseName + ".api, line " + ` line ` + "\n" ok = util.WriteFile(baseName + ".tmp", result) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".tmp", resFile, None) if util.CleanFile(ok): util.DeleteFiles([baseName + ".tmp"]) return ok
def RunImplTestCase(fullName, lang, dtc): # Remove the files we expect output in to util.RemoveTestFiles(fullName, [".arg.res"]) # Run the test interpreter = cmdline.LookUpWildCard('ip', lang, 'impl', 'interpreter', dtc) localName = util.ExtractName(fullName) + ".vdm" outputFile = util.ExtractName(fullName) + ".res" argFile = util.ExtractName(fullName) + ".arg" logFile = argFile + ".logfile" ok = SetupArgFile(fullName,lang) if not ok: return false if dtc == 'dtcon': dtccmd = "-DPIQ" else: dtccmd = "" cmd = interpreter + " -i " + dtccmd + " -O " + outputFile + " " + argFile + " " + localName util.DeleteFiles([outputFile]) if lang == "rt": if(os.path.exists("logfile")): os.unlink("logfile") # Now run the interpreter (exitCode, stdout, stderr) = util.RunCommand(cmd, None, None, true) if lang == "rt": if(os.path.exists("logfile")): os.rename('logfile', logFile); if not os.path.exists(outputFile): util.WriteFile(outputFile,stdout); exitCode = 1 expResultFile = resfile.FindResFile(fullName) if expResultFile == None: ok = false if ok: if exitCode != 0: ## Maybe a runtime error occured. # Note the order of the next binary expression ensures that # CompareRunTimeError is executed even though no expected result was # found! This is necesary as this function generates the result one # can copy in place ok = CompareRunTimeError(fullName, expResultFile, stdout) and ok return ok # Was output produced? if not os.path.exists(outputFile): report.Error("'" + cmd + "' didn't produce the expected result file: '" + outputFile + "'", "Either command was malformed, or the interpreter crashed.") return false interpreter = cmdline.LookUpWildCard('ip', lang, 'impl', 'spec-compare', dtc) ok = resfile.CompareResult(fullName, outputFile, expResultFile, interpreter) return ok
def CompileRunAndCompare(fullName, lang, type, modules): global packageMap baseName = util.ExtractName(fullName) ok = true if ok: ok = CreateArgFile(fullName, lang, modules) standardlibs = convert.GetStandardLibs() #libdir = cmdline.LookUp('java-stdlib-dir') libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir')) for lib in standardlibs: libfile = lib + ".java" util.CopyFile(libdir + "/" + libfile, libfile) if ok: ok = CompileJavaFiles(fullName, lang, type, modules) for mod in packageMap.keys(): packagePaths = string.split(packageMap[mod], '.') packageDir = string.join(packagePaths, '/') if not os.path.exists(packageDir): os.makedirs(packageDir) report.Progress(4, "cp " + mod + "*.class " + packageDir + "/.") exitCode = os.system("cp " + mod + "*.class " + packageDir + "/.") interpreter = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'interpreter')) if ok: # Execute the binary #flags = cmdline.LookUpWildCard('java',lang,type,'rtflags') flags = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'rtflags')) #if util.IsWindowsOS(): # flags = string.replace(flags, ":", ";") (exitCode, stdout, stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0, "Error running Java " + "generated binary", true) print("java " + flags + " TMAIN run") ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) lib = "" if ok == None: # It was not a runtime error interpreter = lib + cmdline.LookUpWildCard('java', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok
def RunSpecTestCases(fullNames, lang, coverageFile): global vdmde, speccompare, backend, javac, java, vppde global staticSemanticsOnly, parserOnly testvdm = "test-j2v.vdm" if parserOnly == "yes": testvdm = "test-javamain.vdm" if staticSemanticsOnly == "yes": testvdm = "test-jss.vdm" testvdm = setup.BaseDir + "/../j2v-test/" + testvdm cmd = vdmde + " -DP -a -b -R " + coverageFile + " " + testvdm (exitCode, dummy1, dummy2) = util.RunCommand( cmd, 0, "Possible core dump while interpreting specification.", true) ok = exitCode == 0 if exitCode != 0: report.Error("Toolbox returns an error ", "'" + dummy1 + "'") if ok and dummy1.find("Run-Time Error") != -1: report.Error( "Toolbox returns an run-time error. Look for the latest .arg.err file ", "") modules = [] if parserOnly == "no" and ok: # # Run backend for the VDM++ AS value to get # VDM++ concrete syntax # for fullName in fullNames: bn = GetTestName(fullName) semResName = bn + ".arg.res" if not os.path.exists(semResName): continue if staticSemanticsOnly == "yes": util.CopyFile(semResName, fullName + "/" + semResName) report.setTestCaseName(fullName) modules.append(fullName) # # The second arg should be directory where to put all # generated .vpp # if staticSemanticsOnly == "no": if AS2CS(semResName, os.getcwd()): # # Run java to produce expected result # expRes = os.getcwd() + "/" + GetTestName( fullName) + ".res.exp" # # If test directory contains its own MAIN.java, use it, # otherwise, use the generic MAIN.java in the current dir # j2vutilJava = setup.BaseDir + "/../j2v-test/J2VUTIL.java" genericMainJava = setup.BaseDir + "/../j2v-test/MAIN.java" mainJava = fullName + "/MAIN.java" if not os.path.exists(mainJava): mainJava = genericMainJava javaFiles = fullName + "/*.java " + mainJava + " " + j2vutilJava + " " # # Compile MAIN.java and J2VUTIL.java using # classes of test java files prepared earlier # if not util.IsWindowsOS(): cmd = javac + " -d " + os.getcwd() + " " + javaFiles else: cmd = javac + " -classpath ./VDM.jar -d ./ " + javaFiles (exitCode, dummy1, dummy2) = util.RunCommand(cmd, 0, "Javac returns errors. ") if exitCode != 0: break if not util.IsWindowsOS(): cmd = java + " -classpath $CLASSPATH:" + os.getcwd( ) + " MAIN" else: cmd = java + " MAIN" (exitCode, dummy1, dummy2) = util.RunCommand(cmd, 0, "Java returns errors. ", true) if exitCode != 0: break if not util.WriteFile(expRes, dummy1): break # # Run vppde to produce the result itself # prefix = os.getcwd() + "/" + GetTestName(fullName) res = bn + ".res" vpparg = "vpp.arg" # # Create arg file # util.WriteFile(vpparg, "new A().Test()") objectVpp = setup.BaseDir + "/../j2v-test/Object.vpp" vppFiles = "" if not util.IsWindowsOS(): FID = os.popen("find " + os.getcwd() + " -type f -name \*.vpp") else: FID = os.popen("find ./ -type f -name \*.vpp") line = FID.readline() while (line != ''): vppFiles = vppFiles + line[:-1] + " " line = FID.readline() vppFiles = vppFiles + " " + objectVpp cmd = vppde + " -iD -O " + res + " " + vpparg + " " + vppFiles (exitCode, dummy1, dummy2) = util.RunCommand(cmd, 0, "Vppde returns errors. ", true) if exitCode != 0: report.Error("Vppde returns errors " + fullName, "") filesToDelete = [] FID = os.popen("ls *.vpp *.class 2>/dev/null") line = FID.readline() while (line != ''): filesToDelete.append(line[:-1]) line = FID.readline() util.DeleteFiles(filesToDelete) okNames = [] # # Now validate the results # for fullName in fullNames: argErr = bn + ".arg.err" semResName = bn + ".arg.res" resName = bn + ".res" expSemResName = fullName + "/" + bn + ".arg.res.exp" expResName = bn + ".res.exp" report.setTestCaseName(fullName) ok = true if ok and staticSemanticsOnly == "no" and parserOnly == "no": if not os.path.exists(resName): report.Error( "No result file generated for test case " + fullName, "Maybe the code generation failed for " + "one of the previous test cases") ok = false if ok: if staticSemanticsOnly == "no" and parserOnly == "no": if os.path.exists(expResName): ok = resfile.CompareResult(bn, resName, expResName, speccompare, false) else: if staticSemanticsOnly == "yes" and parserOnly == "no": if os.path.exists(semResName): ok = resfile.CompareResult(bn, semResName, expSemResName, speccompare, false) else: report.Error( "No result file generated for test case " + fullName, "Look for the latest .arg.err file ") ok = false if parserOnly == "yes": if os.path.exists(argErr): errorText = util.ReadFile(argErr) if errorText != "": report.Error("Parser failure " + fullName, errorText) if ok: okNames.append(fullName) return (okNames, modules)