def CompareResult(fullName, outputFile, resFile, interpreter, structTest=true):
report.Progress(
4, "Comparing result for " + ` fullName ` + " using diff method")
# compare the results using normal "diff"
actualResult = util.ReadFile(outputFile)
expectedResult = util.ReadFile(resFile)
# Remove duplicate white spaces and line breaks, spaces around commas and parenthesis.
actualResult = string.strip(re.sub("\s+", " ", actualResult))
expectedResult = string.strip(re.sub("\s+", " ", expectedResult))
actualResult = string.strip(re.sub("\s*,\s*", ",", actualResult))
expectedResult = string.strip(re.sub("\s*,\s*", ",", expectedResult))
actualResult = string.strip(re.sub("\s*\(\s*", "(", actualResult))
expectedResult = string.strip(re.sub("\s*\(\s*", "(", expectedResult))
actualResult = string.strip(re.sub("\s*\)\s*", ")", actualResult))
expectedResult = string.strip(re.sub("\s*\)\s*", ")", expectedResult))
if actualResult == expectedResult:
return true
# Hmmm we need to try to compare using VDMDE then.
if structTest and interpreter != None:
report.Progress(
4, "Comparing result for " + ` fullName ` + " by build VDM value")
# template = util.ReadFile(setup.BaseDir+"/../pogtestcases/compare-pog.vdm")
template = util.ReadFile(setup.BaseDir + "/templates/compare-pog.vdm")
if template == None:
return false
data = util.SubString('<<EXPECTED_RESULT>>', expectedResult, template)
data = util.SubString('<<ACTUAL_RESULT>>', actualResult, data)
ok = util.WriteFile("compare.vdm", data)
if not ok:
return false
ok = util.WriteFile(".vdmtest", "compare.arg")
# util.CopyFile(setup.BaseDir+"/../pogtestcases/compare-pog.arg","compare.arg")
# util.CopyFile(setup.BaseDir+"/general-files/compare.arg","compare.arg")
util.CopyFile(setup.BaseDir + "/general-files/compare-pog.arg",
"compare.arg")
# cmd = interpreter + " -a -b compare.vdm"
cmd = interpreter + " -i compare.arg compare.vdm"
(exitcode, dummy1, dummy2) = util.RunCommand(cmd)
ok = (exitcode == 0)
else:
ok = false
if not ok:
report.Error(
"Actual result is different from expected result for " +
` fullName `, "expected result : " + expectedResult + "\n" +
"actual result : " + actualResult)
if util.CleanFile(ok):
util.DeleteFiles(["comapre.arg"])
return ok
def CopyFilesForMerging(fullName, modules):
global packageMap
# Copy any files needed for merging
dir = os.path.dirname(fullName)
for mod in modules:
javaFile = dir + "/" + mod + ".java"
if os.path.exists(javaFile):
util.CopyFile(javaFile, "./" + mod + ".java")
package = GetPackage("./" + mod + ".java")
if package != "":
packageMap[mod] = package
externalJavaFile = dir + "/external_" + mod + ".java"
if os.path.exists(externalJavaFile):
util.CopyFile(externalJavaFile, "./external_" + mod + ".java")
package = GetPackage("./external_" + mod + ".java")
if package != "":
packageMap[mod] = package
def CompileRunAndCompare(fullName, lang, type, modules):
global packageMap
baseName = util.ExtractName(fullName)
ok = true
if ok:
ok = CreateArgFile(fullName, lang, modules)
standardlibs = convert.GetStandardLibs()
libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir'))
for lib in standardlibs:
libfile = lib + ".java"
util.CopyFile(libdir + "/" + libfile, libfile)
if ok:
ok = CompileJavaFiles(fullName, lang, type, modules)
interpreter = os.path.expandvars(
cmdline.LookUpWildCard('java', lang, type, 'interpreter'))
if ok:
# Execute the binary
flags = os.path.expandvars(
cmdline.LookUpWildCard('java', lang, type, 'rtflags'))
(exitCode, stdout,
stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0,
"Error running Java " + "generated binary",
true)
print("java " + flags + " TMAIN run")
ok = (exitCode == 0)
if ok:
ok = util.WriteFile(baseName + ".res", stdout)
if ok:
resFile = resfile.FindResFile(fullName)
ok = (resFile != None)
if ok:
ok = CompareRunTimeError(fullName, resFile)
lib = ""
if ok == None: # It was not a runtime error
interpreter = lib + cmdline.LookUpWildCard('java', lang, type,
'spec-compare')
# Compare the result
ok = resfile.CompareResult(fullName, baseName + ".res", resFile,
interpreter)
return ok
def SetupArgFile(fullName,lang):
argNm = util.StripExt(fullName)+".arg"
localNm = util.ExtractName(fullName)+".arg"
if os.path.exists(argNm):
data = util.ReadFile(argNm)
if data == None:
return false
if lang != 'sl':
data = re.sub("^(.*)`(.*)\s*$", "new \\1().\\2", data, re.M)
util.DeleteFiles([localNm])
return util.WriteFile(localNm, data)
else:
argFile = sys.path[0]+"/ip-files/iprun-" + lang + ".arg"
return util.CopyFile(argFile,localNm)
def CompileRunAndCompare(fullName, lang, type, modules):
global packageMap
baseName = util.ExtractName(fullName)
ok = true
if ok:
ok = CreateArgFile(fullName, lang, modules)
standardlibs = convert.GetStandardLibs()
#libdir = cmdline.LookUp('java-stdlib-dir')
libdir = os.path.expandvars(cmdline.LookUp('java-stdlib-dir'))
for lib in standardlibs:
libfile = lib + ".java"
util.CopyFile(libdir + "/" + libfile, libfile)
if ok:
ok = CompileJavaFiles(fullName, lang, type, modules)
for mod in packageMap.keys():
packagePaths = string.split(packageMap[mod], '.')
packageDir = string.join(packagePaths, '/')
if not os.path.exists(packageDir):
os.makedirs(packageDir)
report.Progress(4, "cp " + mod + "*.class " + packageDir + "/.")
exitCode = os.system("cp " + mod + "*.class " + packageDir + "/.")
interpreter = os.path.expandvars(
cmdline.LookUpWildCard('java', lang, type, 'interpreter'))
if ok:
# Execute the binary
#flags = cmdline.LookUpWildCard('java',lang,type,'rtflags')
flags = os.path.expandvars(
cmdline.LookUpWildCard('java', lang, type, 'rtflags'))
#if util.IsWindowsOS():
# flags = string.replace(flags, ":", ";")
(exitCode, stdout,
stderr) = util.RunCommand(interpreter + " " + flags + " TMAIN", 0,
"Error running Java " + "generated binary",
true)
print("java " + flags + " TMAIN run")
ok = (exitCode == 0)
if ok:
ok = util.WriteFile(baseName + ".res", stdout)
if ok:
resFile = resfile.FindResFile(fullName)
ok = (resFile != None)
if ok:
ok = CompareRunTimeError(fullName, resFile)
lib = ""
if ok == None: # It was not a runtime error
interpreter = lib + cmdline.LookUpWildCard('java', lang, type,
'spec-compare')
# Compare the result
ok = resfile.CompareResult(fullName, baseName + ".res", resFile,
interpreter)
return ok
def RunSpecTestCases(fullNames, lang, coverageFile):
global vdmde, speccompare, backend, javac, java, vppde
global staticSemanticsOnly, parserOnly
testvdm = "test-j2v.vdm"
if parserOnly == "yes":
testvdm = "test-javamain.vdm"
if staticSemanticsOnly == "yes":
testvdm = "test-jss.vdm"
testvdm = setup.BaseDir + "/../j2v-test/" + testvdm
cmd = vdmde + " -DP -a -b -R " + coverageFile + " " + testvdm
(exitCode, dummy1, dummy2) = util.RunCommand(
cmd, 0, "Possible core dump while interpreting specification.", true)
ok = exitCode == 0
if exitCode != 0:
report.Error("Toolbox returns an error ", "'" + dummy1 + "'")
if ok and dummy1.find("Run-Time Error") != -1:
report.Error(
"Toolbox returns an run-time error. Look for the latest .arg.err file ",
"")
modules = []
if parserOnly == "no" and ok:
#
# Run backend for the VDM++ AS value to get
# VDM++ concrete syntax
#
for fullName in fullNames:
bn = GetTestName(fullName)
semResName = bn + ".arg.res"
if not os.path.exists(semResName):
continue
if staticSemanticsOnly == "yes":
util.CopyFile(semResName, fullName + "/" + semResName)
report.setTestCaseName(fullName)
modules.append(fullName)
#
# The second arg should be directory where to put all
# generated .vpp
#
if staticSemanticsOnly == "no":
if AS2CS(semResName, os.getcwd()):
#
# Run java to produce expected result
#
expRes = os.getcwd() + "/" + GetTestName(
fullName) + ".res.exp"
#
# If test directory contains its own MAIN.java, use it,
# otherwise, use the generic MAIN.java in the current dir
#
j2vutilJava = setup.BaseDir + "/../j2v-test/J2VUTIL.java"
genericMainJava = setup.BaseDir + "/../j2v-test/MAIN.java"
mainJava = fullName + "/MAIN.java"
if not os.path.exists(mainJava):
mainJava = genericMainJava
javaFiles = fullName + "/*.java " + mainJava + " " + j2vutilJava + " "
#
# Compile MAIN.java and J2VUTIL.java using
# classes of test java files prepared earlier
#
if not util.IsWindowsOS():
cmd = javac + " -d " + os.getcwd() + " " + javaFiles
else:
cmd = javac + " -classpath ./VDM.jar -d ./ " + javaFiles
(exitCode, dummy1,
dummy2) = util.RunCommand(cmd, 0,
"Javac returns errors. ")
if exitCode != 0:
break
if not util.IsWindowsOS():
cmd = java + " -classpath $CLASSPATH:" + os.getcwd(
) + " MAIN"
else:
cmd = java + " MAIN"
(exitCode, dummy1,
dummy2) = util.RunCommand(cmd, 0, "Java returns errors. ",
true)
if exitCode != 0:
break
if not util.WriteFile(expRes, dummy1):
break
#
# Run vppde to produce the result itself
#
prefix = os.getcwd() + "/" + GetTestName(fullName)
res = bn + ".res"
vpparg = "vpp.arg"
#
# Create arg file
#
util.WriteFile(vpparg, "new A().Test()")
objectVpp = setup.BaseDir + "/../j2v-test/Object.vpp"
vppFiles = ""
if not util.IsWindowsOS():
FID = os.popen("find " + os.getcwd() +
" -type f -name \*.vpp")
else:
FID = os.popen("find ./ -type f -name \*.vpp")
line = FID.readline()
while (line != ''):
vppFiles = vppFiles + line[:-1] + " "
line = FID.readline()
vppFiles = vppFiles + " " + objectVpp
cmd = vppde + " -iD -O " + res + " " + vpparg + " " + vppFiles
(exitCode, dummy1,
dummy2) = util.RunCommand(cmd, 0,
"Vppde returns errors. ", true)
if exitCode != 0:
report.Error("Vppde returns errors " + fullName, "")
filesToDelete = []
FID = os.popen("ls *.vpp *.class 2>/dev/null")
line = FID.readline()
while (line != ''):
filesToDelete.append(line[:-1])
line = FID.readline()
util.DeleteFiles(filesToDelete)
okNames = []
#
# Now validate the results
#
for fullName in fullNames:
argErr = bn + ".arg.err"
semResName = bn + ".arg.res"
resName = bn + ".res"
expSemResName = fullName + "/" + bn + ".arg.res.exp"
expResName = bn + ".res.exp"
report.setTestCaseName(fullName)
ok = true
if ok and staticSemanticsOnly == "no" and parserOnly == "no":
if not os.path.exists(resName):
report.Error(
"No result file generated for test case " + fullName,
"Maybe the code generation failed for " +
"one of the previous test cases")
ok = false
if ok:
if staticSemanticsOnly == "no" and parserOnly == "no":
if os.path.exists(expResName):
ok = resfile.CompareResult(bn, resName, expResName,
speccompare, false)
else:
if staticSemanticsOnly == "yes" and parserOnly == "no":
if os.path.exists(semResName):
ok = resfile.CompareResult(bn, semResName,
expSemResName, speccompare,
false)
else:
report.Error(
"No result file generated for test case " + fullName,
"Look for the latest .arg.err file ")
ok = false
if parserOnly == "yes":
if os.path.exists(argErr):
errorText = util.ReadFile(argErr)
if errorText != "":
report.Error("Parser failure " + fullName,
errorText)
if ok:
okNames.append(fullName)
return (okNames, modules)
def CompareResult(fullName, outputFile, resFile, interpreter, structTest=true):
# compare the results using normal "diff"
actualResult = util.ReadFile(outputFile)
expectedResult = util.ReadFile(resFile)
# Remove duplicate white spaces and line breaks, spaces around commas and parenthesis.
actualResult = re.sub("\s+", " ", actualResult).strip()
expectedResult = re.sub("\s+", " ", expectedResult).strip()
actualResult = re.sub("\s*,\s*", ",", actualResult).strip()
expectedResult = re.sub("\s*,\s*", ",", expectedResult).strip()
actualResult = re.sub("\s*\(\s*", "(", actualResult).strip()
expectedResult = re.sub("\s*\(\s*", "(", expectedResult).strip()
actualResult = re.sub("\s*\)\s*", ")", actualResult).strip()
expectedResult = re.sub("\s*\)\s*", ")", expectedResult).strip()
if actualResult == expectedResult:
return true
# Expected Result is Run Time Error
if expectedResult.startswith("Run-Time Error"):
report.Error(
"Actual result is different from expected result for '" +
fullName + "'", "expected result : " + expectedResult + "\n" +
"actual result : " + actualResult)
return false
if expectedResult.startswith(
"<UTIL.NumberToInt>:") or actualResult.startswith(
"<UTIL.NumberToInt>:"):
report.Error(
"Actual result is different from expected result for '" +
fullName + "'", "expected result : " + expectedResult + "\n" +
"actual result : " + actualResult)
return false
if expectedResult.startswith(
"The construct is not supported:") or actualResult.startswith(
"The construct is not supported:"):
report.Error(
"Actual result is different from expected result for '" +
fullName + "'", "expected result : " + expectedResult + "\n" +
"actual result : " + actualResult)
return false
report.Progress(
4, "Comparing result for '" + fullName + "' using diff method")
# Hmmm we need to try to compare using VDMDE then.
if structTest and interpreter != None:
report.Progress(
4, "Comparing result for '" + fullName + "' by build VDM value")
template = util.GetTemplate("compare-result")
if template == None:
return false
data = util.SubString('<<EXPECTED_RESULT>>', expectedResult, template)
data = util.SubString('<<ACTUAL_RESULT>>', actualResult, data)
ok = util.WriteFile("compare.vdm", data)
if not ok:
return false
util.CopyFile(setup.BaseDir + "/general-files/compare.arg",
"compare.arg")
cmd = interpreter + " -i compare.arg compare.vdm"
(exitcode, dummy1, dummy2) = util.RunCommand(cmd)
ok = (exitcode == 0)
else:
ok = false
if not ok:
report.Error(
"Actual result is different from expected result for '" +
fullName + "'", "expected result : " + expectedResult + "\n" +
"actual result : " + actualResult)
if util.CleanFile(ok):
util.DeleteFiles(["comapre.arg"])
return ok