def fitness_function(chromosome):
try:
if fitness_function.vectorProperties.base_type == TestVectorProperties.BaseType[2]:
# Sometimes this conversion fails and I don't see why?
# Just catch it and move on
chromosome.genomeList = [chr(val) for val in chromosome.genomeList]
except TypeError:
pass
fitness_function.run += 1
traceFile = "%s.%s.%d" % (os.path.basename(fitness_function.binary), "trace", fitness_function.run)
cmd = '%s --debug-flags=Fetch --trace-file=%s %s --cpu-type=timing -c %s -o "%s"' % \
(config.Arguments.gem5_simulator, traceFile, config.Arguments.gem5_config, fitness_function.binary, ' '.join(map(str, chromosome.genomeList)))
debug.debug_message("Running '%s' on gem5" % cmd, 1)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
returncode = proc.wait()
if returncode:
debug.exit_message("Running '%s' failed" % cmd)
gem5_trace = os.path.abspath(os.getcwd()) + os.sep + config.Arguments.m5_trace_directory + os.sep + traceFile
assert os.path.exists(gem5_trace), "Expected to find gem5 trace in '%s' but it is not there" % gem5_trace
firstLines = os.popen("head -1 %s" % gem5_trace).readlines()
lastLines = os.popen("tail -1 %s" % gem5_trace).readlines()
assert len(firstLines) == 1
assert len(lastLines) == 1
firstLine = firstLines[0]
lastLine = lastLines[0]
time1 = shlex.split(firstLine)[0]
time2 = shlex.split(lastLine)[0]
time1 = time1[:-1]
time2 = time2[:-1]
score = int(time2) - int(time1)
debug.debug_message("Score = %d" % score, 1)
fitness_function.gem5traces.append(compress_trace(gem5_trace))
return score
def read_file(self):
locale.setlocale(locale.LC_ALL, 'en_US.UTF8')
with open(config.Arguments.test_specification_file, 'r') as f:
for line in f:
index = line.find('=')
if index == -1:
debug.exit_message("Found an invalid line '%s' in the test specification file" % line)
else:
lhs = line[:index].strip()
rhs = line[index+1:].strip()
if lhs.lower() == "type":
self.base_type = rhs
elif lhs.lower() == "length":
try:
self.length = int(rhs)
except:
debug.exit_message("The length of the test vector must be a non-negative integer. It is '%s'." % rhs)
elif lhs.lower() == "lower":
try:
self.lower = locale.atoi(rhs)
except:
debug.exit_message("The lower bound on the range of elements in the test vector must be an integer. It is '%s'." % rhs)
elif lhs.lower() == "upper":
try:
self.upper = locale.atoi(rhs)
except:
debug.exit_message("The upper bound on the range of elements in the test vector must be an integer. It is '%s'." % rhs)
else:
debug.exit_message("Do not understand the line '%s' in the test specification file" % line)
def set_rootID(self):
for v in self:
if v.number_of_predecessors() == 0:
if self.rootID != vertices.dummyID:
debug.exit_message("There are multiple tree roots")
else:
self.rootID = v.vertexID
if self.rootID == vertices.dummyID:
debug.exit_message("No tree root found")
def set_rootID(self):
for v in self:
if v.number_of_predecessors() == 0:
if self.rootID != vertices.dummyID:
debug.exit_message("There are multiple tree roots")
else:
self.rootID = v.vertexID
if self.rootID == vertices.dummyID:
debug.exit_message("No tree root found")
def disassemble_program(binary):
debug.verbose_message("Disassembling program", __name__)
disassembly_filename = binary + ".dis"
with open(disassembly_filename, "w") as disassembly:
cmd = "%s %s -d" % (config.Arguments.objdump, binary)
proc = subprocess.Popen(cmd, shell=True, stdout=disassembly, stderr=sys.stderr)
returncode = proc.wait()
if returncode:
debug.exit_message("Disassembling '%s' failed" % binary)
return disassembly_filename
def __parse(self, traceFiles):
runID = 0
for filename in traceFiles:
parsing = False
with gzip.open(filename, 'r') as f:
runID += 1
self._allruns.add(runID)
debug.debug_message(
"Analysing gem5 trace file '%s'" % filename, 1)
for line in f:
lexemes = shlex.split(line)
PCLexeme = lexemes[-1]
assert len(
PCLexeme
) == 11, "Unable to parse program counter %s" % PCLexeme
try:
time = int(lexemes[0][:-1])
PCLexeme = PCLexeme[5:]
PC = int(PCLexeme, 16)
if PC == self.__firstAddr:
self.__time1 = time
startTime = time
parsing = True
self.__currentContextv = self.__contextg.getVertex(
self.__contextg.getRootID())
self.__currentCFG = self._program.getCFG(
self.__currentContextv.getName())
self.__currentLNT = self._program.getLNT(
self.__currentContextv.getName())
self.__currentPathg = self._program.getPathInfoGraph(
self.__currentContextv.getName())
self.__predBB = None
self.__currentBB = self.__currentCFG.getVertex(
self.__currentCFG.getEntryID())
self._analyseCFGVertex(self.__currentPathg,
self.__currentLNT,
self.__currentBB.vertexID)
if parsing:
self.__parseAddress(time, PC, runID)
if PC == self.__lastAddr:
# Stop parsing
parsing = False
# Compute the HWMT
totalTime = time - startTime
self._longestTime = max(self._longestTime,
totalTime)
# Falsify conjectures
self._endOfFunction(self.__currentCFG,
self.__currentLNT,
self.__currentPathg)
except ValueError:
debug.exit_message(
"Cannot cast %s into an integer: it is not a hexadecimal string"
% PCLexeme)
def set_rootID(self):
without_predecessors = []
for v in self:
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
if len(without_predecessors) > 1:
debug.exit_message("Too many roots found: %s" % ','.join(str(vertexID) for vertexID in without_predecessors))
elif len(without_predecessors) == 0:
debug.exit_message("No root found")
else:
self.rootID = without_predecessors[0]
def find_and_set_root(self):
without_predecessors = set()
for v in self:
if v.number_of_predecessors() == 0:
without_predecessors.add(v.vertexID)
if len(without_predecessors) == 0:
debug.exit_message("Could not find program entry point as there are no functions without predecessors")
elif len(without_predecessors) > 1:
debug.exit_message("Call graph has too many entry points: %s" (','.join(str(vertexID) for vertexID in without_predecessors)))
else:
self.rootID = list(without_predecessors)[0]
assert self.rootID, "Unable to set root ID of call graph"
def run(self):
try:
self.compile()
if self.status == enums.Status.passed:
# Fitness is inversely proportional to execution time
self.fitness = 1/self.execution_time
debug.verbose_message("Individual %d: execution time = %f, fitness = %f" \
% (self.ID, self.execution_time, self.fitness), __name__)
else:
self.fitness = 0
except internal_exceptions.FailedCompilationException as e:
debug.exit_message(e)
def set_rootID(self):
without_predecessors = []
for v in self:
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
if len(without_predecessors) > 1:
debug.exit_message(
"Too many roots found: %s" %
','.join(str(vertexID) for vertexID in without_predecessors))
elif len(without_predecessors) == 0:
debug.exit_message("No root found")
else:
self.rootID = without_predecessors[0]
def disassemble_program(binary):
debug.verbose_message("Disassembling program", __name__)
disassembly_filename = binary + ".dis"
with open(disassembly_filename, 'w') as disassembly:
cmd = "%s %s -d" % (config.Arguments.objdump, binary)
proc = subprocess.Popen(cmd,
shell=True,
stdout=disassembly,
stderr=sys.stderr)
returncode = proc.wait()
if returncode:
debug.exit_message("Disassembling '%s' failed" % binary)
return disassembly_filename
def set_entry_and_exit(self):
without_predecessors = []
without_successors = []
for v in self:
if v.number_of_successors() == 0:
without_successors.append(v.vertexID)
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
if len(without_predecessors) == 0:
debug.exit_message("No entry point found")
elif len(without_predecessors) > 1:
debug.exit_message(
"Too many entry points found: %s" %
(','.join(str(vertexID) for vertexID in without_predecessors)))
else:
self.entryID = without_predecessors[0]
if len(without_successors) == 0:
debug.exit_message("No exit point found")
elif len(without_successors) > 1:
debug.exit_message(
"Too many exit points found: %s" %
(','.join(str(vertexID) for vertexID in without_successors)))
else:
self.exitID = without_successors[0]
def the_command_line():
parser = argparse.ArgumentParser(description="Compute super block cfgs")
parser.add_argument("program_file",
help="a file containing program information (with '.txt' extension)")
parser.add_argument("-d",
"--debug",
type=int,
help="debug mode",
default=0)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDrawGraph files",
default=False)
parser.add_argument("--generate-traces",
type=int,
help="generate traces for the given program",
default=0,
metavar="<INT>")
parser.add_argument("--parse-trace",
help="parse this trace file",
metavar="<FILE>")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.parse_args(namespace=config.Arguments)
config.Arguments.basename = os.path.splitext(os.path.basename(config.Arguments.program_file))[0]
config.Arguments.basepath = os.path.abspath(os.path.dirname(config.Arguments.program_file))
config.Arguments.program_file = os.path.abspath(config.Arguments.program_file)
if not config.Arguments.program_file.endswith(".txt"):
debug.exit_message("Please pass a program file with a '%s' suffix" % ".txt")
if config.Arguments.parse_trace is not None:
config.Arguments.parse_trace = os.path.abspath(config.Arguments.parse_trace)
assert os.path.exists(config.Arguments.parse_trace), "Trace file '%s' does not exist" % config.Arguments.parse_trace
assert os.path.getmtime(config.Arguments.program_file) <= os.path.getmtime(config.Arguments.parse_trace), "Program file modified AFTER trace file generation"
def compile_program(program):
debug.verbose_message("Compiling program", __name__)
optimisation = ""
extraFlags = ""
if config.Arguments.flags:
for flag in config.Arguments.flags:
extraFlags += "-%s " % flag
if re.match(r"O[0-3]+", flag):
optimisation = flag
binary = program[:-2] + optimisation
cmd = "%s -fno-stack-protector -static %s %s -o %s" % (config.Arguments.GCC, extraFlags, program, binary)
debug.debug_message("Compiling with command '%s'" % cmd, 1)
proc = subprocess.Popen(cmd, shell=True, stdout=sys.stdout, stderr=sys.stderr)
returncode = proc.wait()
if returncode:
debug.exit_message("Compiling '%s' with '%s' failed" % (program, cmd))
return binary
def run_gem5(binary):
test_vector_properties = TestVectorProperties()
run = get_next_trace_file_number(binary) + 1
# Now run the program n times
random_test_vectors = RandomGeneration(test_vector_properties)
gem5traces = []
for i in xrange(run, config.Arguments.tests + run):
traceFile = "%s.%s.%d.gz" % (os.path.basename(binary), "trace", i)
cmd = '%s --debug-flags=Fetch --trace-file=%s %s --cpu-type=timing -c %s -o "%s"' % \
(config.Arguments.gem5_simulator, traceFile, config.Arguments.gem5_config, binary, random_test_vectors.next_test_vector())
debug.debug_message("Running '%s' on gem5" % cmd, 1)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
returncode = proc.wait()
if returncode:
debug.exit_message("Running '%s' failed" % cmd)
gem5_trace = os.path.abspath(os.getcwd()) + os.sep + config.Arguments.m5_trace_directory + os.sep + traceFile
assert os.path.exists(gem5_trace), "Expected to find gem5 trace in '%s' but it is not there" % gem5_trace
gem5traces.append(gem5_trace)
return gem5traces
def compile_program(program):
debug.verbose_message("Compiling program", __name__)
optimisation = ""
extraFlags = ""
if config.Arguments.flags:
for flag in config.Arguments.flags:
extraFlags += "-%s " % flag
if re.match(r'O[0-3]+', flag):
optimisation = flag
binary = program[:-2] + optimisation
cmd = "%s -fno-stack-protector -static %s %s -o %s" % (
config.Arguments.GCC, extraFlags, program, binary)
debug.debug_message("Compiling with command '%s'" % cmd, 1)
proc = subprocess.Popen(cmd,
shell=True,
stdout=sys.stdout,
stderr=sys.stderr)
returncode = proc.wait()
if returncode:
debug.exit_message("Compiling '%s' with '%s' failed" % (program, cmd))
return binary
def __parse (self, traceFiles):
runID = 0
for filename in traceFiles:
parsing = False
with gzip.open(filename, 'r') as f:
runID += 1
self._allruns.add(runID)
debug.debug_message("Analysing gem5 trace file '%s'" % filename, 1)
for line in f:
lexemes = shlex.split(line)
PCLexeme = lexemes[-1]
assert len(PCLexeme) == 11, "Unable to parse program counter %s" % PCLexeme
try:
time = int(lexemes[0][:-1])
PCLexeme = PCLexeme[5:]
PC = int(PCLexeme, 16)
if PC == self.__firstAddr:
self.__time1 = time
startTime = time
parsing = True
self.__currentContextv = self.__contextg.getVertex(self.__contextg.getRootID())
self.__currentCFG = self._program.getCFG(self.__currentContextv.getName())
self.__currentLNT = self._program.getLNT(self.__currentContextv.getName())
self.__currentPathg = self._program.getPathInfoGraph(self.__currentContextv.getName())
self.__predBB = None
self.__currentBB = self.__currentCFG.getVertex(self.__currentCFG.getEntryID())
self._analyseCFGVertex(self.__currentPathg, self.__currentLNT, self.__currentBB.vertexID)
if parsing:
self.__parseAddress (time, PC, runID)
if PC == self.__lastAddr:
# Stop parsing
parsing = False
# Compute the HWMT
totalTime = time - startTime
self._longestTime = max(self._longestTime, totalTime)
# Falsify conjectures
self._endOfFunction(self.__currentCFG, self.__currentLNT, self.__currentPathg)
except ValueError:
debug.exit_message("Cannot cast %s into an integer: it is not a hexadecimal string" % PCLexeme)
def solve(self):
debug.debug_message("Solving ILP", __name__, 10)
self.write_to_file()
cmd = "lp_solve %s" % self.filename
start = timeit.default_timer()
proc = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
return_code = proc.wait()
end = timeit.default_timer()
self.solve_time = end - start
if return_code != 0:
debug.exit_message("Running '%s' failed" % cmd)
for line in proc.stdout.readlines():
if line.startswith("Value of objective function"):
lexemes = shlex.split(line)
self.wcet = long(decimal.Decimal(lexemes[-1]))
elif line.startswith(LpSolve.edge_prefix) or line.startswith(LpSolve.vertex_prefix):
lexemes = shlex.split(line)
assert len(lexemes) == 2, "Incorrectly detected variable execution count line '%s'" % line
self.variable_execution_counts[lexemes[0]] = int(lexemes[1])
def set_entry_and_exit(self):
without_predecessors = []
without_successors = []
for v in self:
if v.number_of_successors() == 0:
without_successors.append(v.vertexID)
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
entryID = None
if len(without_predecessors) == 0:
debug.exit_message("CFG '%s' does not have an entry point" %
self.name)
elif len(without_predecessors) > 1:
debug_info = ""
for bbID in without_predecessors:
bb = self.getVertex(bbID)
debug_info += bb.__str__()
debug.exit_message("CFG '%s' has too many entry points: %s" %
(self.name, debug_info))
else:
entryID = self.getNextVertexID()
entryv = vertices.CFGVertex(entryID, True)
self.addVertex(entryv)
self.set_entryID(entryID)
self.addEdge(entryID, without_predecessors[0])
exitID = None
if len(without_successors) == 0:
debug.exit_message("CFG '%s' does not have an exit point" %
self.name)
elif len(without_successors) > 1:
debug_info = ""
for bbID in without_successors:
bb = self.getVertex(bbID)
debug_info += bb.__str__()
debug.exit_message("CFG '%s' has too many exit points: %s" %
(self.name, debug_info))
else:
exitID = self.getNextVertexID()
exitv = vertices.CFGVertex(exitID, True)
self.addVertex(exitv)
self.set_exitID(exitID)
self.addEdge(without_successors[0], exitID)
assert entryID, "Unable to set entry ID"
assert exitID, "Unable to set exit ID"
self.addEdge(exitID, entryID)
def check_is_tree(self):
without_predecessors = set()
for v in self:
if v.number_of_predecessors() == 0:
without_predecessors.add(v)
elif v.number_of_predecessors() > 1:
debug.exit_message("%d does not have a unique parent" % v.vertexID)
if len(without_predecessors) == 0:
debug.exit_message("No root vertex found")
elif len(without_predecessors) > 1:
debug.exit_message("Too many root vertices")
def check_is_tree(self):
without_predecessors = set()
for v in self:
if v.number_of_predecessors() == 0:
without_predecessors.add(v)
elif v.number_of_predecessors() > 1:
debug.exit_message("%d does not have a unique parent" %
v.vertexID)
if len(without_predecessors) == 0:
debug.exit_message("No root vertex found")
elif len(without_predecessors) > 1:
debug.exit_message("Too many root vertices")
def set_entry_and_exit(self):
without_predecessors = []
without_successors = []
for v in self:
if v.number_of_successors() == 0:
without_successors.append(v.vertexID)
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
entryID = None
if len(without_predecessors) == 0:
debug.exit_message("CFG '%s' does not have an entry point" % self.name)
elif len(without_predecessors) > 1:
debug_info = ""
for bbID in without_predecessors:
bb = self.getVertex(bbID)
debug_info += bb.__str__()
debug.exit_message("CFG '%s' has too many entry points: %s" % (self.name, debug_info))
else:
entryID = self.getNextVertexID()
entryv = vertices.CFGVertex(entryID, True)
self.addVertex(entryv)
self.set_entryID(entryID)
self.addEdge(entryID, without_predecessors[0])
exitID = None
if len(without_successors) == 0:
debug.exit_message("CFG '%s' does not have an exit point" % self.name)
elif len(without_successors) > 1:
debug_info = ""
for bbID in without_successors:
bb = self.getVertex(bbID)
debug_info += bb.__str__()
debug.exit_message("CFG '%s' has too many exit points: %s" % (self.name, debug_info))
else:
exitID = self.getNextVertexID()
exitv = vertices.CFGVertex(exitID, True)
self.addVertex(exitv)
self.set_exitID(exitID)
self.addEdge(without_successors[0], exitID)
assert entryID, "Unable to set entry ID"
assert exitID, "Unable to set exit ID"
self.addEdge(exitID, entryID)
def set_entry_and_exit(self):
without_predecessors = []
without_successors = []
for v in self:
if v.number_of_successors() == 0:
without_successors.append(v.vertexID)
if v.number_of_predecessors() == 0:
without_predecessors.append(v.vertexID)
if len(without_predecessors) == 0:
debug.exit_message("No entry point found")
elif len(without_predecessors) > 1:
debug.exit_message("Too many entry points found: %s" % (','.join(str(vertexID) for vertexID in without_predecessors)))
else:
self.entryID = without_predecessors[0]
if len(without_successors) == 0:
debug.exit_message("No exit point found")
elif len(without_successors) > 1:
debug.exit_message("Too many exit points found: %s" % (','.join(str(vertexID) for vertexID in without_successors)))
else:
self.exitID = without_successors[0]
def set_gem5_variables():
# Need a gem5 environment variable
gem5_home = "GEM5_HOME"
try:
config.Arguments.gem5_basepath = os.environ[gem5_home]
if not os.path.exists(os.path.abspath(config.Arguments.gem5_basepath)):
debug.exit_message("Your gem5 base directory '%s' does not exist" %
config.Arguments.gem5_basepath)
config.Arguments.gem5_simulator = config.Arguments.gem5_basepath + os.sep + "build" + os.sep + "ARM" + os.sep + "gem5.opt"
if not os.path.exists(config.Arguments.gem5_simulator):
debug.exit_message(
"""Unable to find '%s' in your gem5 distribution, which is the optimised arm
configuration of gem5. Ensure that you have built this version using
'scons arm/build/gem5.opt' in '%s'""" \
% (config.Arguments.gem5_simulator, config.Arguments.gem5_basepath))
config.Arguments.gem5_config = config.Arguments.gem5_basepath + os.sep + "configs" + os.sep + "example" + os.sep + "se.py"
if not os.path.exists(config.Arguments.gem5_config):
debug.exit_message(
"The gem5 configuration file '%s' does not exist" %
config.Arguments.gem5_config)
except KeyError:
debug.exit_message(
"You need to set environment variable '%s' to simulate the program using gem5"
% gem5_home)
def set_gem5_variables():
# Need a gem5 environment variable
gem5_home = "GEM5_HOME"
try:
config.Arguments.gem5_basepath = os.environ[gem5_home]
if not os.path.exists(os.path.abspath(config.Arguments.gem5_basepath)):
debug.exit_message("Your gem5 base directory '%s' does not exist" % config.Arguments.gem5_basepath)
config.Arguments.gem5_simulator = (
config.Arguments.gem5_basepath + os.sep + "build" + os.sep + "ARM" + os.sep + "gem5.opt"
)
if not os.path.exists(config.Arguments.gem5_simulator):
debug.exit_message(
"""Unable to find '%s' in your gem5 distribution, which is the optimised arm
configuration of gem5. Ensure that you have built this version using
'scons arm/build/gem5.opt' in '%s'"""
% (config.Arguments.gem5_simulator, config.Arguments.gem5_basepath)
)
config.Arguments.gem5_config = (
config.Arguments.gem5_basepath + os.sep + "configs" + os.sep + "example" + os.sep + "se.py"
)
if not os.path.exists(config.Arguments.gem5_config):
debug.exit_message("The gem5 configuration file '%s' does not exist" % config.Arguments.gem5_config)
except KeyError:
debug.exit_message("You need to set environment variable '%s' to simulate the program using gem5" % gem5_home)
def the_command_line():
new_filename_prefix = "program"
def clean():
for paths, dirs, files in os.walk(os.path.abspath(os.curdir)):
for filename in files:
if re.match(r'%s[0-9]+\.txt' % new_filename_prefix, filename):
full_path = os.path.join(paths, filename)
debug.verbose_message("Removing '%s'" % full_path, __name__)
os.remove(full_path)
def create_filename():
files = [f for f in os.listdir(os.curdir) if os.path.isfile(os.path.join(os.curdir,f))]
numbers = set()
for filename in files:
if re.match(r'%s[0-9]+\.txt' % new_filename_prefix, filename):
filenumbers = re.findall(r'[0-9]+', filename)
assert len(filenumbers) == 1
filenumber = filenumbers[0]
numbers.add(int(filenumber))
for i in xrange(1,sys.maxint):
if i not in numbers:
return os.path.abspath('%s%d.txt' % (new_filename_prefix, i))
assert False
class SubprogramsAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 0:
raise argparse.ArgumentTypeError("The number of subprograms must be a positive integer")
setattr(namespace, self.dest, value)
class BasicBlockAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 0:
raise argparse.ArgumentTypeError("The number of basic blocks per program must be a positive integer")
setattr(namespace, self.dest, value)
class FanOutAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 1:
raise argparse.ArgumentTypeError("The maximum fan out of a basic block must be at least two to allow branches")
setattr(namespace, self.dest, value)
# The command-line parser and its options
parser = argparse.ArgumentParser(description="Generate random program structure (call graph and CFGs)")
parser.add_argument("directory",
help="write the program file to this directory")
parser.add_argument("-f",
"--filename",
help="write the program to this file name",
default=create_filename())
parser.add_argument("-c",
"--clean",
metavar="",
type=clean,
help="clean files from previous runs",
default=False)
parser.add_argument("-d",
"--debug",
type=int,
help="debug mode",
default=0)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDraw files to visualise graphs",
default=False)
parser.add_argument("--subprograms",
action=SubprogramsAction,
type=int,
help="number of subprograms",
default=1,
metavar="<INT>")
parser.add_argument("--loops",
type=int,
help="maximum number of loops in a CFG",
default=0,
metavar="<INT>")
parser.add_argument("--self-loops",
type=int,
help="maximum number of self-loops in a CFG",
default=0,
metavar="<INT>")
parser.add_argument("--nesting-depth",
type=int,
help="maximum nesting depth of loops",
default=1,
metavar="<INT>")
parser.add_argument("--fan-out",
action=FanOutAction,
type=int,
help="select maximum fan out of a CFG vertex",
default=random.randint(2,10),
metavar="<INT>")
parser.add_argument("--basic-blocks",
type=int,
action=BasicBlockAction,
help="maximum number of vertices in a CFG",
default=10,
metavar="<INT>")
parser.add_argument("--breaks",
action="store_true",
help="allow break-like constructs in the CFG",
default=False)
parser.add_argument("--continues",
action="store_true",
help="allow continue-like constructs in the CFG",
default=False)
parser.add_argument("--unstructured",
action="store_true",
help="add unstructured edges to the CFG",
default=False)
# WCET options
wcet_group = parser.add_argument_group("WCET arguments")
wcet_group.add_argument("--add-WCET-information",
action="store_true",
help="add WCET information (execution times and execution counts) when writing the program to file",
default=False)
wcet_group.add_argument("--maximum-loop-bound",
type=int,
help="maximum allowed loop bound for a single iteration of the parent loop",
default=10,
metavar="<INT>")
wcet_group.add_argument("--maximum-execution-time",
type=int,
help="maximum allowed execution time for a basic block",
default=10,
metavar="<INT>")
# Profiling options
profiling_group = parser.add_argument_group("Program profiling arguments")
profiling_group.add_argument("--add-profile-information",
action="store_true",
help="add profile information (program points whose execution frequencies we want to count) when writing the program to file",
default=False)
# Tracing options
tracing_group = parser.add_argument_group("Program tracing arguments")
tracing_group.add_argument("--generate-traces",
type=int,
help="generate this number of execution traces",
metavar="<INT>",
default=0)
tracing_group.add_argument("--add-timestamps",
action="store_true",
help="attach timestamps to program points in the trace",
default=False)
tracing_group.add_argument("--maximum-number-of-loop-iterations",
type=int,
help="ensure a loop never iterates more than this value",
metavar="<INT>",
default=10)
tracing_group.add_argument("--maximum-number-of-calls",
type=int,
help="ensure the number of procedure calls exceeds this value",
metavar="<INT>",
default=5)
parser.parse_args(namespace=config.Arguments)
if config.Arguments.basic_blocks < config.Arguments.loops * 2:
debug.exit_message("The number of vertices in a CFG must be at least twice the number of loops")
config.Arguments.basename = os.path.splitext(os.path.basename(config.Arguments.filename))[0]
config.Arguments.basepath = os.path.abspath(config.Arguments.directory)
def the_command_line():
def comma_separated_list(the_list):
try:
return the_list.split(',')
except:
raise argparse.ArgumentTypeError("Invalid compiler flags")
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="Run C programs on gem5 and analyse traces")
parser.add_argument(
"program_file",
help=
"either a program to compile (with '.c.' extension) or a pre-compiled binary"
)
parser.add_argument("gem5_traces", nargs='*', help="previous gem5 runs")
parser.add_argument("-C",
"--compile",
action="store_true",
help="only compile program",
default=False)
parser.add_argument("--compiler-flags",
type=comma_separated_list,
help="flags to be passed to the compiler",
dest="flags",
metavar="<FLAGS>")
parser.add_argument("-d",
"--debug",
action="store",
type=int,
help="debug mode",
metavar="<INT>",
default=0)
parser.add_argument(
"--exclusive-size",
action="store",
type=int,
help="size of subsets of mutually exclusive basic blocks to compute",
metavar="<INT>")
parser.add_argument(
"-G",
"--ga",
action="store_true",
help="use a genetic algorithm to generate test vectors",
default=False)
parser.add_argument(
"-I",
"--inline",
action="store_true",
help="do analysis with fully inlined program where applicable",
default=False)
parser.add_argument(
"-r",
"--root",
action="store",
help=
"the function that is the entry point of the analysis. [This should not be 'main']",
metavar="<FUNCTION>")
parser.add_argument("-T",
"--number-of-tests",
action="store",
type=int,
dest="tests",
help="the number of times to run the application",
metavar="<INT>",
default=1)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDrawGraph files",
default=False)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.parse_args(namespace=config.Arguments)
config.Arguments.basename = os.path.splitext(
os.path.basename(config.Arguments.program_file))[0]
config.Arguments.basepath = os.path.abspath(
os.path.dirname(config.Arguments.program_file))
config.Arguments.program_file = os.path.abspath(
config.Arguments.program_file)
if not os.path.exists(config.Arguments.program_file):
debug.exit_message(
"The first command-line argument must be a file: '%s' does not exist."
% config.Arguments.program_file)
elif not os.path.isfile(config.Arguments.program_file):
debug.exit_message(
"The first command-line argument must be a file: '%s' is not a file"
% config.Arguments.program_file)
config.Arguments.test_specification_file = os.path.splitext(
config.Arguments.program_file)[0] + '.test'
if not os.path.exists(config.Arguments.test_specification_file):
debug.exit_message(
"Expected to find the test specification file '%s' but it is not there"
% config.Arguments.test_specification_file)
def the_command_line():
new_filename_prefix = "program"
def clean():
for paths, dirs, files in os.walk(os.path.abspath(os.curdir)):
for filename in files:
if re.match(r'%s[0-9]+\.txt' % new_filename_prefix, filename):
full_path = os.path.join(paths, filename)
debug.verbose_message("Removing '%s'" % full_path,
__name__)
os.remove(full_path)
def create_filename():
files = [
f for f in os.listdir(os.curdir)
if os.path.isfile(os.path.join(os.curdir, f))
]
numbers = set()
for filename in files:
if re.match(r'%s[0-9]+\.txt' % new_filename_prefix, filename):
filenumbers = re.findall(r'[0-9]+', filename)
assert len(filenumbers) == 1
filenumber = filenumbers[0]
numbers.add(int(filenumber))
for i in xrange(1, sys.maxint):
if i not in numbers:
return os.path.abspath('%s%d.txt' % (new_filename_prefix, i))
assert False
class SubprogramsAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 0:
raise argparse.ArgumentTypeError(
"The number of subprograms must be a positive integer")
setattr(namespace, self.dest, value)
class BasicBlockAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 0:
raise argparse.ArgumentTypeError(
"The number of basic blocks per program must be a positive integer"
)
setattr(namespace, self.dest, value)
class FanOutAction(argparse.Action):
def __call__(self, parser, namespace, value, option_string=None):
if value <= 1:
raise argparse.ArgumentTypeError(
"The maximum fan out of a basic block must be at least two to allow branches"
)
setattr(namespace, self.dest, value)
# The command-line parser and its options
parser = argparse.ArgumentParser(
description="Generate random program structure (call graph and CFGs)")
parser.add_argument("directory",
help="write the program file to this directory")
parser.add_argument("-f",
"--filename",
help="write the program to this file name",
default=create_filename())
parser.add_argument("-c",
"--clean",
metavar="",
type=clean,
help="clean files from previous runs",
default=False)
parser.add_argument("-d",
"--debug",
type=int,
help="debug mode",
default=0)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDraw files to visualise graphs",
default=False)
parser.add_argument("--subprograms",
action=SubprogramsAction,
type=int,
help="number of subprograms",
default=1,
metavar="<INT>")
parser.add_argument("--loops",
type=int,
help="maximum number of loops in a CFG",
default=0,
metavar="<INT>")
parser.add_argument("--self-loops",
type=int,
help="maximum number of self-loops in a CFG",
default=0,
metavar="<INT>")
parser.add_argument("--nesting-depth",
type=int,
help="maximum nesting depth of loops",
default=1,
metavar="<INT>")
parser.add_argument("--fan-out",
action=FanOutAction,
type=int,
help="select maximum fan out of a CFG vertex",
default=random.randint(2, 10),
metavar="<INT>")
parser.add_argument("--basic-blocks",
type=int,
action=BasicBlockAction,
help="maximum number of vertices in a CFG",
default=10,
metavar="<INT>")
parser.add_argument("--breaks",
action="store_true",
help="allow break-like constructs in the CFG",
default=False)
parser.add_argument("--continues",
action="store_true",
help="allow continue-like constructs in the CFG",
default=False)
parser.add_argument("--unstructured",
action="store_true",
help="add unstructured edges to the CFG",
default=False)
# WCET options
wcet_group = parser.add_argument_group("WCET arguments")
wcet_group.add_argument(
"--add-WCET-information",
action="store_true",
help=
"add WCET information (execution times and execution counts) when writing the program to file",
default=False)
wcet_group.add_argument(
"--maximum-loop-bound",
type=int,
help=
"maximum allowed loop bound for a single iteration of the parent loop",
default=10,
metavar="<INT>")
wcet_group.add_argument(
"--maximum-execution-time",
type=int,
help="maximum allowed execution time for a basic block",
default=10,
metavar="<INT>")
# Profiling options
profiling_group = parser.add_argument_group("Program profiling arguments")
profiling_group.add_argument(
"--add-profile-information",
action="store_true",
help=
"add profile information (program points whose execution frequencies we want to count) when writing the program to file",
default=False)
# Tracing options
tracing_group = parser.add_argument_group("Program tracing arguments")
tracing_group.add_argument("--generate-traces",
type=int,
help="generate this number of execution traces",
metavar="<INT>",
default=0)
tracing_group.add_argument(
"--add-timestamps",
action="store_true",
help="attach timestamps to program points in the trace",
default=False)
tracing_group.add_argument(
"--maximum-number-of-loop-iterations",
type=int,
help="ensure a loop never iterates more than this value",
metavar="<INT>",
default=10)
tracing_group.add_argument(
"--maximum-number-of-calls",
type=int,
help="ensure the number of procedure calls exceeds this value",
metavar="<INT>",
default=5)
parser.parse_args(namespace=config.Arguments)
if config.Arguments.basic_blocks < config.Arguments.loops * 2:
debug.exit_message(
"The number of vertices in a CFG must be at least twice the number of loops"
)
config.Arguments.basename = os.path.splitext(
os.path.basename(config.Arguments.filename))[0]
config.Arguments.basepath = os.path.abspath(config.Arguments.directory)
def the_command_line():
def comma_separated_list(the_list):
try:
return the_list.split(",")
except:
raise argparse.ArgumentTypeError("Invalid compiler flags")
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter, description="Run C programs on gem5 and analyse traces"
)
parser.add_argument(
"program_file", help="either a program to compile (with '.c.' extension) or a pre-compiled binary"
)
parser.add_argument("gem5_traces", nargs="*", help="previous gem5 runs")
parser.add_argument("-C", "--compile", action="store_true", help="only compile program", default=False)
parser.add_argument(
"--compiler-flags",
type=comma_separated_list,
help="flags to be passed to the compiler",
dest="flags",
metavar="<FLAGS>",
)
parser.add_argument("-d", "--debug", action="store", type=int, help="debug mode", metavar="<INT>", default=0)
parser.add_argument(
"--exclusive-size",
action="store",
type=int,
help="size of subsets of mutually exclusive basic blocks to compute",
metavar="<INT>",
)
parser.add_argument(
"-G", "--ga", action="store_true", help="use a genetic algorithm to generate test vectors", default=False
)
parser.add_argument(
"-I",
"--inline",
action="store_true",
help="do analysis with fully inlined program where applicable",
default=False,
)
parser.add_argument(
"-r",
"--root",
action="store",
help="the function that is the entry point of the analysis. [This should not be 'main']",
metavar="<FUNCTION>",
)
parser.add_argument(
"-T",
"--number-of-tests",
action="store",
type=int,
dest="tests",
help="the number of times to run the application",
metavar="<INT>",
default=1,
)
parser.add_argument("-u", "--udraw", action="store_true", help="generate uDrawGraph files", default=False)
parser.add_argument("-v", "--verbose", action="store_true", help="be verbose", default=False)
parser.parse_args(namespace=config.Arguments)
config.Arguments.basename = os.path.splitext(os.path.basename(config.Arguments.program_file))[0]
config.Arguments.basepath = os.path.abspath(os.path.dirname(config.Arguments.program_file))
config.Arguments.program_file = os.path.abspath(config.Arguments.program_file)
if not os.path.exists(config.Arguments.program_file):
debug.exit_message(
"The first command-line argument must be a file: '%s' does not exist." % config.Arguments.program_file
)
elif not os.path.isfile(config.Arguments.program_file):
debug.exit_message(
"The first command-line argument must be a file: '%s' is not a file" % config.Arguments.program_file
)
config.Arguments.test_specification_file = os.path.splitext(config.Arguments.program_file)[0] + ".test"
if not os.path.exists(config.Arguments.test_specification_file):
debug.exit_message(
"Expected to find the test specification file '%s' but it is not there"
% config.Arguments.test_specification_file
)
def get_binary_and_program():
file_ext = os.path.splitext(config.Arguments.program_file)[1]
if file_ext:
if file_ext == '.c':
if not distutils.spawn.find_executable(config.Arguments.GCC):
debug.exit_message(
"Unable to find arm GCC cross compiler '%s' on your path" %
config.Arguments.GCC)
if not distutils.spawn.find_executable(config.Arguments.objdump):
debug.exit_message(
"Unable to find arm GCC object dump facility '%s' on your path"
% config.Arguments.objdump)
if not config.Arguments.root:
debug.exit_message(
"To compile and analyse a C file, you must supply a root function via -r."
)
binary = compile_program(config.Arguments.program_file)
disassembly = disassemble_program(binary)
program = arm.Disassembler(disassembly,
config.Arguments.root).program
program_input_output.write_disassembled_program_to_file(
program, disassembly)
return binary, program
else:
debug.exit_message(
"Unable to compile '%s' because its extension is not '%s'" %
(config.Arguments.program_file, '.c'))
else:
binary = config.Arguments.program_file
config.Arguments.program_file = binary + ".txt"
if not os.access(binary, os.X_OK):
debug.exit_message(
"The argument '%s' does not have execute permissions" % binary)
if not os.path.exists(config.Arguments.program_file):
debug.exit_message(
"Expected to find file with program information in '%s' but it is not there"
% config.Arguments.program_file)
program = program_input_output.read_file(config.Arguments.program_file)
return binary, program
def the_command_line ():
ilp = "--ilp"
clp = "--clp"
region_based = "--region-based"
region_based_super_block_CFG = "--region-based-super-block-CFG"
all_calculations = [ilp, clp, region_based, region_based_super_block_CFG]
parser = argparse.ArgumentParser(description="Compute WCET using implicit path enumeration defined on the super block CFG")
parser.add_argument("program_file",
help="a file containing program information (with '.txt' extension)")
parser.add_argument("-d",
"--debug",
type=int,
help="debug mode",
default=0)
parser.add_argument("--function",
help="do a WCET calculation for this function only",
default=None)
parser.add_argument("--keep-temps",
action="store_true",
help="keep any files generated during the analysis",
default=False)
parser.add_argument("--log-to-file",
help="log output to this file")
parser.add_argument("--randomise-WCET-data",
action="store_true",
help="randomise data used in WCET calculation, overriding any data supplied in the program file",
default=False)
parser.add_argument("--repeat-calculation",
type=int,
help="repeat the calculation this many times",
default=1,
metavar="<INT>")
parser.add_argument("--shuffle-constraints",
action="store_true",
help="before repeating the solve stage of a constraint-based WCET calculation, shuffle the constraints",
default=False)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDraw files to visualise graphs",
default=False)
parser.add_argument(clp,
action="store_true",
help="use constraint logic programming to solve WCET estimation constraint system",
default=False)
parser.add_argument(ilp,
action="store_true",
help="use integer linear programming to solve WCET estimation constraint system",
default=False)
parser.add_argument(region_based,
action="store_true",
help="use region-based calculation on CFG to compute a WCET estimate",
default=False)
parser.add_argument(region_based_super_block_CFG,
action="store_true",
help="use region-based calculation on super block CFG to compute a WCET estimate",
default=False)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.parse_args(namespace=config.Arguments)
if not (config.Arguments.ilp or
config.Arguments.clp or
config.Arguments.region_based or
config.Arguments.region_based_super_block_CFG):
debug.exit_message("You must calculate specify how to calculate a WCET estimate with one or more of the following: %s" %
','.join(all_calculations))
config.Arguments.basename = os.path.splitext(os.path.basename(config.Arguments.program_file))[0]
config.Arguments.basepath = os.path.abspath(os.path.dirname(config.Arguments.program_file))
def the_command_line():
ilp = "--ilp"
clp = "--clp"
region_based = "--region-based"
region_based_super_block_CFG = "--region-based-super-block-CFG"
all_calculations = [ilp, clp, region_based, region_based_super_block_CFG]
parser = argparse.ArgumentParser(
description=
"Compute WCET using implicit path enumeration defined on the super block CFG"
)
parser.add_argument(
"program_file",
help="a file containing program information (with '.txt' extension)")
parser.add_argument("-d",
"--debug",
type=int,
help="debug mode",
default=0)
parser.add_argument("--function",
help="do a WCET calculation for this function only",
default=None)
parser.add_argument("--keep-temps",
action="store_true",
help="keep any files generated during the analysis",
default=False)
parser.add_argument("--log-to-file", help="log output to this file")
parser.add_argument(
"--randomise-WCET-data",
action="store_true",
help=
"randomise data used in WCET calculation, overriding any data supplied in the program file",
default=False)
parser.add_argument("--repeat-calculation",
type=int,
help="repeat the calculation this many times",
default=1,
metavar="<INT>")
parser.add_argument(
"--shuffle-constraints",
action="store_true",
help=
"before repeating the solve stage of a constraint-based WCET calculation, shuffle the constraints",
default=False)
parser.add_argument("-u",
"--udraw",
action="store_true",
help="generate uDraw files to visualise graphs",
default=False)
parser.add_argument(
clp,
action="store_true",
help=
"use constraint logic programming to solve WCET estimation constraint system",
default=False)
parser.add_argument(
ilp,
action="store_true",
help=
"use integer linear programming to solve WCET estimation constraint system",
default=False)
parser.add_argument(
region_based,
action="store_true",
help="use region-based calculation on CFG to compute a WCET estimate",
default=False)
parser.add_argument(
region_based_super_block_CFG,
action="store_true",
help=
"use region-based calculation on super block CFG to compute a WCET estimate",
default=False)
parser.add_argument("-v",
"--verbose",
action="store_true",
help="be verbose",
default=False)
parser.parse_args(namespace=config.Arguments)
if not (config.Arguments.ilp or config.Arguments.clp
or config.Arguments.region_based
or config.Arguments.region_based_super_block_CFG):
debug.exit_message(
"You must calculate specify how to calculate a WCET estimate with one or more of the following: %s"
% ','.join(all_calculations))
config.Arguments.basename = os.path.splitext(
os.path.basename(config.Arguments.program_file))[0]
config.Arguments.basepath = os.path.abspath(
os.path.dirname(config.Arguments.program_file))
config.Arguments.test_specification_file = os.path.splitext(config.Arguments.program_file)[0] + ".test"
if not os.path.exists(config.Arguments.test_specification_file):
debug.exit_message(
"Expected to find the test specification file '%s' but it is not there"
% config.Arguments.test_specification_file
)
if __name__ == "__main__":
the_command_line()
debug.verbose_message(
"%s Analysing program '%s' %s" % ("*" * 10, config.Arguments.program_file, "*" * 10), __name__
)
time1 = timing.log("COMPILING BEGIN")
binary, program = get_binary_and_program()
time2 = timing.log("COMPILING END")
if config.Arguments.compile:
debug.exit_message("DONE")
if config.Arguments.gem5_traces:
check_trace_files()
else:
set_gem5_variables()
if config.Arguments.ga:
debug.verbose_message("Using GA to generate test vectors", __name__)
config.Arguments.gem5_traces.extend(testing.runGAGem5(binary))
else:
debug.verbose_message("Running program on gem5 with %d tests" % config.Arguments.tests, __name__)
config.Arguments.gem5_traces.extend(testing.run_gem5(binary))
do_analysis(program)
def get_binary_and_program():
file_ext = os.path.splitext(config.Arguments.program_file)[1]
if file_ext:
if file_ext == ".c":
if not distutils.spawn.find_executable(config.Arguments.GCC):
debug.exit_message("Unable to find arm GCC cross compiler '%s' on your path" % config.Arguments.GCC)
if not distutils.spawn.find_executable(config.Arguments.objdump):
debug.exit_message(
"Unable to find arm GCC object dump facility '%s' on your path" % config.Arguments.objdump
)
if not config.Arguments.root:
debug.exit_message("To compile and analyse a C file, you must supply a root function via -r.")
binary = compile_program(config.Arguments.program_file)
disassembly = disassemble_program(binary)
program = arm.Disassembler(disassembly, config.Arguments.root).program
program_input_output.write_disassembled_program_to_file(program, disassembly)
return binary, program
else:
debug.exit_message(
"Unable to compile '%s' because its extension is not '%s'" % (config.Arguments.program_file, ".c")
)
else:
binary = config.Arguments.program_file
config.Arguments.program_file = binary + ".txt"
if not os.access(binary, os.X_OK):
debug.exit_message("The argument '%s' does not have execute permissions" % binary)
if not os.path.exists(config.Arguments.program_file):
debug.exit_message(
"Expected to find file with program information in '%s' but it is not there"
% config.Arguments.program_file
)
program = program_input_output.read_file(config.Arguments.program_file)
return binary, program
if not os.path.exists(config.Arguments.test_specification_file):
debug.exit_message(
"Expected to find the test specification file '%s' but it is not there"
% config.Arguments.test_specification_file)
if __name__ == "__main__":
the_command_line()
debug.verbose_message(
"%s Analysing program '%s' %s" %
('*' * 10, config.Arguments.program_file, '*' * 10), __name__)
time1 = timing.log("COMPILING BEGIN")
binary, program = get_binary_and_program()
time2 = timing.log("COMPILING END")
if config.Arguments.compile:
debug.exit_message("DONE")
if config.Arguments.gem5_traces:
check_trace_files()
else:
set_gem5_variables()
if config.Arguments.ga:
debug.verbose_message("Using GA to generate test vectors",
__name__)
config.Arguments.gem5_traces.extend(testing.runGAGem5(binary))
else:
debug.verbose_message(
"Running program on gem5 with %d tests" %
config.Arguments.tests, __name__)
config.Arguments.gem5_traces.extend(testing.run_gem5(binary))
do_analysis(program)
