def RunTestCase(name, lang, type): baseName = util.ExtractName(name) #compile and run compiler = cmdline.LookUpWildCard('metaiv', lang, type, 'compiler') flags = cmdline.LookUpWildCard('metaiv', lang, type, 'cflags') cmd = compiler + " -o driver driver.cc " + name + " " + flags (exitCode, dummy1, dummy2) = util.RunCommand(cmd, 0, "Problem whith compiling") ok = (exitCode == 0) if ok: if not os.path.exists("driver"): report.Error("Driveri binary is not created") (exitCode, stdout, stderr) = util.RunCommand("./driver" + binext, 0, "Error running c++ generated c++ binary", true) ok = (exitCode == 0) #compare results with expected result files if ok: ok = util.WriteFile(baseName + ".res", stdout) else: report.Error("Output", None, stdout, stderr) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: resfile.CompareResult(name, baseName + ".res", resFile, None) return ok
def RunTestCase(name, lang, type, coverageFile): stdout = None stderr = None baseName = util.ExtractName(name) report.setTestCaseName(name) ok = true #run the test cases cmd = cmdline.LookUpWildCard('cpp', lang, 'spec', 'interpreter') cmd = cmd + " -D -P -a -b -R " + coverageFile exitCode = util.RunCommand( cmd, 0, "Possible core dump while interpreting specification.") # See if a result file was created if ok: if not os.path.exists(baseName + ".arg.res"): report.Error("No result file generated for test case " + name, "Maybe it failed before") ok = false if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".arg.res", resFile, None) return ok
def RunImplTestCase(fullName, lang, posdef): ok = convert.ConvertLanguage(lang, fullName) # Remove the files we expect output in to if ok: util.RemoveTestFiles(fullName, [".arg.res"]) # Run the test if ok: interpreter = cmdline.LookUpWildCard('tc', lang, 'impl', 'interpreter', posdef) if posdef == 'pos': defFlg = "" else: defFlg = " -d " localName = util.ExtractName(fullName) + ".vdm" resName = util.ExtractName(fullName) + ".arg.res" cmd = interpreter + defFlg + " -t -f " + localName + " 2>" + resName # Now run the interpreter (exitCode, stdout, stderr) = util.RunCommand(cmd, None, None, true, true) # Ensure that the interpreter actually did type check anything. # That is in contract to if it stoped due to a syntax error or a core dump. # if re.search("Type checking [^\n]* done", stdout) == None: # report.Error("text 'type check ... done' not found on stdout while running the type checker", # "This might be due to a core dump, a syntax error or the like\n" + # "This does anyway indicate that the type checking was never done", # stdout, stderr) # ok = false if ok: expResName = resfile.FindResFile(fullName) # read the result from the result file, and translate it to a list of numbers # result = TranslateResultImpl(stdout) result = TranslateResultImpl(util.ReadFile(resName)) if result == None: ok = false if ok: if expResName == None: print("Not validating result (2)") if util.KeepFile(false): WriteResult(fullName, result) ok = false if ok: # Validate the result. report.Progress( 4, "Validating result with result file: '" + expResName + "'") ok = ValidateResult(fullName, expResName, result, stdout, stderr) if util.KeepFile(ok): WriteResult(fullName, result) return ok
def FindResFile(fullName): (files, posList) = resfile.FindFile(fullName, "res-pog") path = os.path.dirname(fullName) if len(files) == 1: report.Progress(4, "Expected result file is: " + path + "/" + files[0]) return path + "/" + files[0] else: return resfile.FindResFile(fullName)
def RunImplTestCase(fullName, lang, dtc): # Remove the files we expect output in to util.RemoveTestFiles(fullName, [".arg.res"]) # Run the test interpreter = cmdline.LookUpWildCard('pog', lang, 'impl', 'interpreter') localName = util.ExtractName(fullName) + ".vdm" outputFile = util.ExtractName(fullName) + ".res" argFile = util.ExtractName(fullName) + ".arg" ok = SetupArgFile(fullName, lang) if not ok: return false if dtc == 'dtcon': dtccmd = "-DPIQ" else: dtccmd = "" cmd = interpreter + " -G -O " + outputFile + " " + localName util.DeleteFiles([outputFile]) # Now run the interpreter (exitCode, stdout, stderr) = util.RunCommand(cmd, None, None, true) expResultFile = resfile.FindResFile(fullName) if expResultFile == None: ok = false if ok: if exitCode != 0: ## Maybe a runtime error occured. # Note the order of the next binary expression ensures that # CompareRunTimeError is executed even though no expected result was # found! This is necesary as this function generates the result one # can copy in place ok = CompareRunTimeError(fullName, expResultFile, stdout) and ok return ok # Was output produced? if not os.path.exists(outputFile): report.Error( ` cmd ` + " didn't produce the expected result file: " + ` outputFile `, "Either command was malformed, or the interpreter crashed.") return false interpreter = cmdline.LookUpWildCard('pog', lang, 'impl', 'spec-compare') # ok = resfile.CompareResult(fullName, outputFile, expResultFile, interpreter) ok = CompareResult(fullName, outputFile, expResultFile, interpreter) return ok
def RunSpecTestCases(fullNames, lang, dtc, coverageFile): # remove files we expect output in to. for fullName in fullNames: util.RemoveTestFiles(fullName, [".arg.pt", ".arg.res", ".arg.err", ".arg.msg"]) # run the test cases interpreter = cmdline.LookUpWildCard('ip', lang, 'spec', 'interpreter', dtc) cmd = interpreter + " -a -b -R " + coverageFile exitCode = util.RunCommand(cmd, 0, "Possible core dump while interpreting specification.") okNames = [] # Now validate the results for fullName in fullNames: bn = util.ExtractName(fullName) semResName = bn + ".arg.res" resName = bn + ".res" errName = bn + ".arg.msg" report.setTestCaseName(fullName) # See if a result file was created if not os.path.exists(semResName) and not os.path.exists(errName): report.Error("No result or error file generated for test case " + fullName, "Maybe the interpreting toolbox failed for one of the previous test cases") continue # Find expected result file ok = true expResultFile = resfile.FindResFile(fullName) if expResultFile == None: ok = false if os.path.exists(errName): # See if an error file is generated. data = util.ReadFile(errName) if data == None: continue # Note the order of the next binary expression ensures that # CompareRunTimeError is executed even though no expected result was # found! This is necesary as this function generates the result one # can copy in place ok = CompareRunTimeError(fullName, expResultFile, data) and ok else: # Strip sem values ok = ok and StripSemValue(fullName, lang, dtc) # validate expected result then if ok: interpreter = cmdline.LookUpWildCard('ip', lang, 'spec', 'spec-compare', dtc) ok = resfile.CompareResult(fullName, resName, expResultFile, interpreter) if ok: okNames.append(fullName) return okNames
def RunSpecTestCases(names, lang, posdef, coverageFile): # remove files we expect output in to. for fullName in names: util.RemoveTestFiles(fullName, [".arg.pt", ".arg.res"]) # run the test cases interpreter = cmdline.LookUpWildCard('tc', lang, 'spec', 'interpreter', posdef) cmd = interpreter + " -a -b -I -D -P -R " + coverageFile exitCode = util.RunCommand( cmd, 0, "Possible core dump while interpreting specification.", false, true) okNames = [] # Now validate the results for fullName in names: bn = util.ExtractName(fullName) resName = bn + ".arg.res" report.setTestCaseName(fullName) # See if a result file was created if not os.path.exists(resName): report.Error( "No result generated for test case " + fullName, "Maybe the interpreting toolbox failed for one of the previous test cases" ) continue # read the result from the result file, and translate it to a list of numbers result = TranslateResultSpec(fullName) if result == None: continue # Find the expected result file expResName = resfile.FindResFile(fullName) if expResName == None: if util.KeepFile(false): WriteResult(fullName, result) continue # Validate the result. report.Progress( 4, "Validating result with result file: " + ` expResName `) ok = ValidateResult(fullName, expResName, result, None, None) if ok: okNames.append(fullName) if util.KeepFile(ok): WriteResult(fullName, result) return okNames
def CompileRunAndCompare(fullName, lang, type, modules): global packageMap baseName = util.ExtractName(fullName) ok = true if ok: ok = CreateArgFile(fullName, lang, modules) standardlibs = convert.GetStandardLibs() libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir')) for lib in standardlibs: libfile = lib + ".java" util.CopyFile(libdir + "/" + libfile, libfile) if ok: ok = CompileJavaFiles(fullName, lang, type, modules) interpreter = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'interpreter')) if ok: # Execute the binary flags = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'rtflags')) (exitCode, stdout, stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0, "Error running Java " + "generated binary", true) print("java " + flags + " TMAIN run") ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) lib = "" if ok == None: # It was not a runtime error interpreter = lib + cmdline.LookUpWildCard('java', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok
def CompileRunAndCompare(fullName, lang, type, modules): baseName = util.ExtractName(fullName) ok = true stdout = None stderr = None if ok: CreateUserdefFile(modules) if ok: ok = CreateArgFile(fullName, lang, modules) if ok: ok = CompileCPPFiles(fullName, lang, type, modules) if ok: # Execute the binary (exitCode, stdout, stderr) = util.RunCommand("icode", 0, "Error running c++ generated binary", true) ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) else: report.Error("Output", None, stdout, stderr) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) if ok == None: # It was not a runtime error interpreter = cmdline.LookUpWildCard('cpp', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok
def RunApiTestCase(name, lang, toolMediator): data = util.ReadFile(name) baseName = util.ExtractName(name) neutralName = re.sub('.*/', '', name) # path neutralized! result = baseName + ":\n" last = "" symTable = {} symTable['GENPATH'] = os.path.expanduser(cmdline.LookUp('api-gen-path')) symTable['SOURCEPATH'] = os.path.expanduser( cmdline.LookUp('api-source-path')) print(symTable['GENPATH']) print(symTable['SOURCEPATH']) commands = data.split("\n") line, test = 0, 0 empty = re.compile('^\s*(#.*)|(\s*)$') interactive_mode = cmdline.LookUp('api-run-interactive') for cmd in commands: print("Command: " + cmd) if interactive_mode == 'yes': print("Press <return>") raw_input() line = line + 1 # don't process empty lines if not empty.match(cmd): test = test + 1 try: cmd = re.sub('\$\$', last, cmd) # replace $$ by last result cmd = re.sub('\s+', ' ', cmd).strip() # replace multiple spaces by one cmd = re.sub('\s*\(\s*', '(', cmd) # delete spaces around parentheses cmd = re.sub('\s*\)\s*', ')', cmd) cmd = re.sub('\s*\[\s*', '[', cmd) # delete spaces around parentheses cmd = re.sub('\s*\]\s*', ']', cmd) cmd = re.sub('\s*,\s*', ',', cmd) # delete spaces around commas cmd = re.sub('\s*=\s*', '=', cmd) # delete spaces around = # find variable assignment m = re.compile('^\$(?P<lval>\w+)=(?P<rval>.*)$').match(cmd) if m != None: lval = m.group('lval') cmd = m.group('rval') else: lval = "" # find variable usages and replace them by value m = re.compile('^.*\$(?P<id>\w+).*$').match(cmd) if m != None: while m != None: var = "\$" + m.group('id') try: repl = symTable[m.group('id')] except KeyError: raise Syntax("Unknown variable name $" + var) cmd = re.sub(var, repl, cmd) m = re.compile('^.*\$(?P<id>\w+).*$').match(cmd) result = result + "-- " + str(test) + "(in line " + str( line) + "):\n" # print (cmd) # parse and execute the command last = toolMediator.ParseCommand(cmd, name, line) lines = last.split('\n') pathexp = re.compile('^.*/(.*\.(vdm|vpp|api))$') for l in lines: # neutralize path result = result + pathexp.sub(r'\1', l) + "\n" last = re.sub('\n$', '', last) last = re.sub('\n', ',', last).strip() if lval != "": symTable[lval] = last except CORBA.COMM_FAILURE: raise CORBA.COMM_FAILURE except CORBA.BAD_PARAM: result = result + "CORBA.BAD_PARAM in " + neutralName + ", line " + str( line) + "\n" except ToolboxAPI.APIError: _, e, _ = sys.exc_info() result = result + "APIError in " + neutralName + ", line " + str( line) + ": " + e.msg + "\n" except VDM.VDMError: _, e, _ = sys.exc_info() result = result + "VDMError in " + neutralName + ", line " + str( line) + ": " + str(e.err) + "\n" except Syntax: _, e, _ = sys.exc_info() result = result + "Syntax error in " + neutralName + ", line " + str( line) + ": " + e.GetMessage() + "\n" except CORBA.UserException: result = result + "CORBA.UserException: in " + baseName + ".api, line " + str( line) + "\n" except CORBA.SystemException: _, e, _ = sys.exc_info() print(e) result = result + "CORBA.SystemException: in " + baseName + ".api, line " + str( line) + "\n" except Exception: _, e, _ = sys.exc_info() print("Python exception") print(e.args) result = result + "Python exception in " + baseName + ".api, line " + str( line) + "\n" except: result = result + "Unknown exception in " + baseName + ".api, line " + str( line) + "\n" ok = util.WriteFile(baseName + ".tmp", result) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".tmp", resFile, None) if util.CleanFile(ok): util.DeleteFiles([baseName + ".tmp"]) return ok
result = result + "APIError in " + neutralName + ", line " + ` line ` + ": " + e.msg + "\n" except VDM.VDMError, e: result = result + "VDMError in " + neutralName + ", line " + ` line ` + ": " + ` e.err ` + "\n" except Syntax, e: result = result + "Syntax error in " + neutralName + ", line " + ` line ` + ": " + e.GetMessage( ) + "\n" except CORBA.UserException: result = result + "CORBA.UserException: in " + baseName + ".api, line " + ` line ` + "\n" except CORBA.SystemException, e: print e result = result + "CORBA.SystemException: in " + baseName + ".api, line " + ` line ` + "\n" except Exception, e: print "Python exception" print e.args result = result + "Python exception in " + baseName + ".api, line " + ` line ` + "\n" except: result = result + "Unknown exception in " + baseName + ".api, line " + ` line ` + "\n" ok = util.WriteFile(baseName + ".tmp", result) if ok: resFile = resfile.FindResFile(name) ok = (resFile != None) if ok: ok = resfile.CompareResult(name, baseName + ".tmp", resFile, None) if util.CleanFile(ok): util.DeleteFiles([baseName + ".tmp"]) return ok
def CompileRunAndCompare(fullName, lang, type, modules): global packageMap baseName = util.ExtractName(fullName) ok = true if ok: ok = CreateArgFile(fullName, lang, modules) standardlibs = convert.GetStandardLibs() #libdir = cmdline.LookUp('java-stdlib-dir') libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir')) for lib in standardlibs: libfile = lib + ".java" util.CopyFile(libdir + "/" + libfile, libfile) if ok: ok = CompileJavaFiles(fullName, lang, type, modules) for mod in packageMap.keys(): packagePaths = string.split(packageMap[mod], '.') packageDir = string.join(packagePaths, '/') if not os.path.exists(packageDir): os.makedirs(packageDir) report.Progress(4, "cp " + mod + "*.class " + packageDir + "/.") exitCode = os.system("cp " + mod + "*.class " + packageDir + "/.") interpreter = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'interpreter')) if ok: # Execute the binary #flags = cmdline.LookUpWildCard('java',lang,type,'rtflags') flags = os.path.expandvars( cmdline.LookUpWildCard('java', lang, type, 'rtflags')) #if util.IsWindowsOS(): # flags = string.replace(flags, ":", ";") (exitCode, stdout, stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0, "Error running Java " + "generated binary", true) print("java " + flags + " TMAIN run") ok = (exitCode == 0) if ok: ok = util.WriteFile(baseName + ".res", stdout) if ok: resFile = resfile.FindResFile(fullName) ok = (resFile != None) if ok: ok = CompareRunTimeError(fullName, resFile) lib = "" if ok == None: # It was not a runtime error interpreter = lib + cmdline.LookUpWildCard('java', lang, type, 'spec-compare') # Compare the result ok = resfile.CompareResult(fullName, baseName + ".res", resFile, interpreter) return ok