def checkOutputs(fldr):
# get a list of all of the subfolders
logName = 'checkOutputs.log'
lf = open(os.path.join(fldr, logName), 'w')
sub_dirs = sorted(os.listdir(fldr))
for sd in sub_dirs:
# only operate on the directories
currFldr = os.path.join(fldr, sd)
if os.path.isdir(currFldr):
lf.write(centerText(" In folder: {} ".format(sd)) + "\n")
# get a list of all of the files in there that have "output" in the name
# in order so can compare two adjacent files
files = sorted(glob.glob1(currFldr, 'output*'))
for x in range(0, len(files), 2):
f1 = files[x]
f2 = files[x + 1]
lf.write("Checking similarity of \n\t{} and\n\t{}\n".format(
f1, f2))
# run diff on these
p0 = sp.Popen([
'diff',
os.path.join(currFldr, f1),
os.path.join(currFldr, f2)
],
stdout=sp.PIPE)
rslt = p0.communicate()[0]
rc = p0.returncode
if rc:
# print(rslt.decode())
lf.write("*** Files are not the same! ***\n")
lf.write('\n')
lf.close()
return
def printTestResult(results, logFile):
logFile.write(centerText(" Test " + results["name"] + " ") + "\n")
logFile.write("\n")
for x in range(0, 2):
for k, log in results["test_log" + str(x)].items():
# print(k + " " + str(type(log)))
comp_name = os.path.basename(log["compile_log"][0]).split(".")[0]
if log["compile_result"] != 0:
logFile.write("Error compiling " + comp_name)
logFile.write(", see \n\t" + log["compile_log"][0] + "\n")
else:
logFile.write("Passed compilation " + comp_name + "\n")
# for key, val in log.items():
# print(" " + key + " " + str(type(val)))
for i in range(len(log["exec_result"])):
cmd = results["commands"][x][i]
rc = log["exec_result"][i]
# print(cmd)
# print(rc)
if rc == 0:
logFile.write("Passed command " + cmd + "\n")
else:
logFile.write("Failed running command \n\t" + cmd +
"\n")
logFile.write("\n")
def run(fldr, trgts, srcs, auxf, cmds, logFile, nowFolder):
progFolder = os.getcwd()
# sometimes there is only one target that takes different parameters
if len(trgts) == 1:
trgts.append(trgts[0])
srcs.append(srcs[0])
# we are creating a dictionary to track everything
run_log = {
"name": os.path.basename(os.path.dirname(fldr)),
"folder": fldr,
"targets": trgts,
"sources": srcs,
"commands": cmds,
"dependencies": auxf
}
# import any file dependencies
aux_list = importFileDependencies(auxf, fldr, progFolder)
# make a folder for this set of files
subFolder = makeSubFolder(fldr, progFolder, nowFolder)
# many of the tests need to be linked with the math library
# this will probably change later to be more versatile
xcflags = '""'
xlflags = '"-lm"'
# these are in quotes so it works for sure on the command line
passes = ['""', '"-TMR"']
for i in range(0, 2): # 0-2, exclusive of 2
test_name = "test_log" + str(i)
run_log[test_name] = {} # need to compare across executions
# going to have to do some tricky dictionary nesting
for p in passes:
# things specific to each run
run_name = "run_log" + p.strip('"')
run_log[test_name][run_name] = {}
runMakeClean()
command = ("make compile OPT_PASSES=" + p + " SRCFOLDER=\"" +
fldr + "\" SRCFILES=\"" + srcs[i] + "\" XCFLAGS=" +
xcflags + " XLFLAGS=" + xlflags + " TARGET=" + trgts[i])
lname = trgts[i] + ".log"
# check for file existing already; if so, rename
# TODO: this check is seems to be always returning true
# also doesn't catch when '-TMR' is appended to the front of the name
# put the check when moving files
# if os.path.exists(os.path.join(subFolder, lname)):
# lname = trgts[i] + '_2.log'
with open(lname, 'w') as lf:
lf.write(command + "\n")
lf.flush()
p1 = sp.Popen(shlex.split(command),
stdout=lf,
stderr=sp.STDOUT)
p1.wait()
rc = p1.returncode
run_log[test_name][run_name]["compile_result"] = rc
# the following list will be empty if compilation failed
exec_results = []
# if success, then rc = 0
if not rc:
# now execute it
for cmd in cmds[i]:
exec_results.append(os.system(cmd))
run_log[test_name][run_name]["exec_result"] = exec_results
# move the resulting files
# note: these are not always .txt files
run_log[test_name][run_name]["files_created"] = moveFiles(
subFolder, "output", trgts[i] + p.strip('"'))
# delete the executable
os.system("rm " + trgts[i])
else:
# make note of error
lf.write("\n" +
centerText(" compilation error, did not run "))
# move the log file
newLogFileName = moveFiles(subFolder, lname, p.strip('"'))
run_log[test_name][run_name]["compile_log"] = newLogFileName
# end inner for loop
# end outer for loop
# remove any dependencies copied in
if auxf:
for aux in aux_list:
os.system("rm " + aux.split("/")[len(aux.split("/")) - 1])
return run_log
def printTestHead(f, t, s, c, x, logFile):
logFile.write(centerText(" Test # " + str(x)) + "\n")
str1 = "Testing in\n\t{}\nMaking targets\n{}From source\n{}Running Commands\n{}".format(
f, listAsString(t), listAsString(s), listAsString(c))
logFile.write(str1)
def main():
# command line processing
parser = setUpArgs()
args = parser.parse_args()
# set up folder references
progFolder = os.getcwd()
fldr = os.path.join(progFolder,"stressTests")
# print("tests go in " + fldr)
if args.clean:
print("Removing old log files...")
os.system("rm " + fldr + "/*.log")
print("Removing old .bc files...")
os.system("rm " + fldr + "/*.bc")
print("Removing old .ll files...")
os.system("rm " + fldr + "/*.ll")
sys.exit(0)
passes = ['"-DWC"', '"-TMR"']
numRuns = args.tests[0]
# header
logFileName = "rand_log_" + getDateString() + ".log"
printIntro(logFileName, numRuns)
# keep track of everything
overall_summary = []
with open(os.path.join(fldr,logFileName), 'w') as lf:
# do this 'n' number of times
for x in range(0,numRuns):
title = ' Test #{} '.format(x)
lf.write(centerText(title) + '\n')
lf.flush()
# file name definitions
target = "stress_random_" + str(x)
rand_file = target + ".clang.ll"
output_file = rand_file.replace("ll", "bc")
opt_file = output_file.replace("clang", "opt")
opt_dis_file = opt_file.replace("bc", "ll")
run_summary = {"header" : title, "target" : target}
updateConsoleDisplay(target, args.tests[0], x)
# create random .ll files
rc0 = createRandomIRFile(fldr, rand_file)
# stuff = input("enter anything to continue")
# convert to .bc files
rc1 = llvmAsseble(progFolder, fldr, rand_file, output_file)
# stuff2 = input("enter anything to continue")
# run through opt
p = sample(passes, 1)[0]
rc2 = llvmOptimizer(fldr, output_file, opt_file, target, p, lf)
run_summary["passes"] = p
run_summary["opt_rc"] = rc2
# move .ll files to test folder
newLLfiles = moveFiles(fldr, opt_dis_file, "")
if rc2 == 0:
# print("moved files: \n\t" + str(newLLfiles))
# os.system("rm " + fldr + "/*.bc")
if newLLfiles:
# remove .ll files because passed test
os.system("rm " + newLLfiles[0])
else:
lf.write(centerText(' failed compilation! ') + "\n")
lf.write('\tsee file ' + rand_file + '\n')
lf.flush()
# get rid of .bc files in the top folder
runMakeClean()
lf.write("\n")
overall_summary.append(run_summary)
# print the summary at the end
lf.flush()
print("Summarizing data.... ")
print("Complete! (" + str(numRuns) + "/" + str(numRuns) + ") " )
sum_head = centerText('') + '\n' + centerText(' Summary: ') + '\n' + centerText('') + '\n\n'
lf.write(sum_head)
successes = 0
for summary in overall_summary:
lf.write('----' + summary["header"] + '----\n')
lf.write('Target: \t\t' + summary["target"] + '\n')
lf.write('Running pass: \t' + summary["passes"] + '\n')
if summary["opt_rc"] == 0:
lf.write('Result: \t\tSuccess\n\n')
successes += 1
else:
lf.write('Result: \t\tFailure\n\n')
lf.write("---- Overall ----\n")
lf.write("Successes: {}/{}\n".format(successes, numRuns))
return