def write_substages_file(self):
stage = self.stage
substages = self._substage_names()
tracename = Main.get_config("trace_traces")
try:
cdb = Main.get_config("calltrace_db", self.stage)
except KeyError:
cdb = None
if cdb and os.path.exists(cdb):
path = Main.get_config("policy_trace_el", self.stage)
if os.path.exists(path):
return
calltrace_path = Main.get_config("calltrace_db", self.stage)
failed = False
substage_linenos = []
for s in substages:
if s == "_start":
n = 0
else:
if not failed:
try:
n = self.get_function_lineno(s, calltrace_path)
except subprocess.CalledProcessError:
print "Did not find %s in %s" % (s, calltrace_path)
failed = True
n = self.get_function_lineno(
s, calltrace_path, True)
substage_linenos.append(n)
outf = open(path, "w")
outf.write("(setq substages '(%s))\n" %
" ".join([str(i) for i in substage_linenos]))
outf.close()
def get_raw_files(self, noprepare):
stage = self.stage
substages = self._substage_numbers()
name = self._substage_names()
try:
substageresultsdir = Main.get_config("policy_trace_fnlist_dir",
self.stage)
except KeyError:
substageresultsdir = None
tracename = Main.get_config("trace_traces")
try:
calltrace_path = Main.get_config("calltrace_db", self.stage)
except KeyError:
calltrace_path = None
if calltrace_path and os.path.exists(
calltrace_path) and substageresultsdir:
if not noprepare:
el_path = Main.get_config("policy_trace_el", self.stage)
if os.path.exists(substageresultsdir):
return {}
pymacs_request.ask_emacs(
'(create-substage-calltraces "%s" "%s" "%s")' %
(calltrace_path, el_path, substageresultsdir))
origdir = os.getcwd()
os.chdir(substageresultsdir)
files = [
os.path.join(substageresultsdir, f) for f in glob.glob("*.txt")
]
if files:
files.sort()
return {i: files[i] for i in range(0, len(files))}
return {}
def addr2disasm(addr, stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
srcdir = Main.get_config("temp_bootloader_src_dir")
cmd = '%sgdb --cd=%s ' \
'-ex "disassemble/r 0x%x,+1" --batch '\
'--nh --nx %s 2>/dev/null' % (cc,
srcdir,
addr,
elf)
# print cmd
output = Main.shell.run_cmd(cmd)
voutput = output.split('\n')[1].strip() # get line with disassembly
voutput = voutput.split('\t')
# try:
# convert to a hex string and then decode it to it is an array of bytes
value = (''.join(voutput[1].split())).decode('hex')
# except Exception:
instr = ' '.join(voutput[2:])
# get function name if possible
foutput = output.split(':')[0]
if foutput.lower() == '0x%x:':
func = '' # not located in a function
else:
rgx = re.compile(r'<([A-Za-z0-9_]+)(\+\d+){0,1}>')
res = re.search(rgx, foutput)
if res is None:
func = ''
else:
func = res.group(1)
return (value, instr, func)
def populate_contents_table(self):
stage = self.stage
substages = self._substage_numbers()
substagesnamename = self._substage_names()
tracename = Main.get_config("trace_traces")
if 'framac' in tracename:
row = self.contents_table.row
tracefile = Main.get_config("framac_callstacks", self.stage)
if os.path.exists(tracefile):
results = self.parse_frama_c_call_trace_stages(
tracefile, self.substage_file_path)
for s in substages:
for f in results[s]:
row["substagenum"] = s
row["functionname"] = f
row.append()
elif "baremetal" not in tracename:
raws = self.get_raw_files(False)
for (num, f) in raws.iteritems():
fopen = open(f, "r")
contents = fopen.read()
row = self.contents_table.row
for n in contents.split():
row["substagenum"] = num
row["functionname"] = n
row.append()
fopen.close()
self.contents_table.flush()
self.contents_table.cols.substagenum.reindex()
def from_line(cls, line, stage):
res = cls.regexp.match(line.strip())
if res is None:
raise Exception("%s is not a framac dst result")
else:
min_value = int(res.group(1), 0)
max_value = int(res.group(2), 0)
if max_value > 0xFFFFFFFF:
max_value = 0xFFFFFFFF
lvalue = res.group(3)
path = res.group(4)
# somewhat of a hack to get relative path
root = Main.get_bootloader_cfg().software_cfg.root
if path.startswith(root):
path = os.path.relpath(path, root)
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
elif path.startswith(
Main.test_suite_path): # not sure why this happens
path = os.path.relpath(path, Main.test_suite_path)
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
elif path.startswith("/tmp/tmp"):
path = "/".join(path.split("/")[3:])
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
lineno = int(res.group(5))
callstack = res.group(6)
# somewhat of a hack for muxconf
return cls(path,
lineno,
lvalue, [intervaltree.Interval(min_value, max_value)],
callstack=callstack,
stage=stage)
def disassemble(what, stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
srcdir = Main.get_config("temp_bootloader_src_dir")
cmd = '%sgdb --cd=%s ' \
'-ex "disassemble %s" --batch --nh --nx %s 2>/dev/null' % (cc,
srcdir,
what,
elf)
output = Main.shell.run_multiline_cmd(cmd)
return output
def init_with_test_instance(self):
# update stage start/end info now that it has been calculated
self.stage = Main.stage_from_name(self.stage.stagename)
if self.substages:
self.policy_file = Main.get_config("policy_file", self.stage)
self.regions_file = Main.get_config("regions_file", self.stage)
self.substages_entrypoints = substage.SubstagesInfo.substage_names(
self.stage)
self._entrypoint = self.stage.entrypoint
self._exitpoint = self.stage.exitpc
if self._startpoint is None:
self._startpoint = self._entrypoint
def _create_var_table(self, substage=-1):
fields = ["startaddr", "size", "kind", "name"]
vtab = self.var_table
if vtab.nrows > 0:
return
cc = Main.cc
stage = self.stage
sname = stage.stagename
elf = Main.get_config("stage_elf", stage)
cmd = "%snm -n -S %s" % (cc, elf)
f = StringIO.StringIO(Main.shell.run_cmd(cmd))
reader = csv.DictReader(f,
fields,
delimiter=" ",
lineterminator="\n",
skipinitialspace=True)
row = vtab.row
for r in reader:
if r['name'] is None:
continue # this means there is no listed size
else:
row['name'] = r['name'].strip()
row['startaddr'] = int(r['startaddr'].strip(), 16)
row['endaddr'] = row['startaddr'] + int(r['size'].strip(), 16)
row['rawkind'] = r['kind'].strip()
k = row['rawkind'].lower()
if ('t' == k) or ('w' == k):
row['kind'] = getattr(addr_space.var_type, 'text')
else:
row['kind'] = getattr(addr_space.var_type, 'staticvar')
row['perms'] = getattr(addr_space.var_perms, 'rw')
row['substage'] = substage
row.append()
vtab.flush()
def __init__(self,
path,
lineno,
lvalue,
values,
pc=None,
origpc=None,
substage_name=None,
callstack="",
stage=None):
self.path = path
self.pc = pc
self.origpc = origpc
self.lineno = lineno
self.values = intervaltree.IntervalTree()
for v in values:
self.values.add(v)
self.lvalue = lvalue
self.stage = stage
if substage_name is None and callstack:
policy = Main.get_config('policy_file', self.stage)
self.substages = substage.SubstagesInfo.substage_names(self.stage)
self.substages[0] = "frama_go"
called_fns = callstack.split("->")
called_fns = filter(len, called_fns)
called_fns.reverse()
for f in called_fns:
if f in self.substages:
substage_name = self.substages.index(f)
break
self.substage = substage_name
def get_line_addr(line, start, stage, debug=False, srcdir=None):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
if srcdir:
srcdir = "--cd=%s " % (srcdir)
cmd = "%sgdb %s -ex 'info line %s' --batch --nh --nx %s 2>/dev/null" % (cc,
srcdir,
line, elf)
if debug:
print cmd
output = Main.shell.run_multiline_cmd(cmd)
if debug:
print output
output = output[0]
assembly = False
if ("but contains no code." in output) and (".S\" is at address" in output):
if (".S\" is at address" in output): # is assembly
assembly = True
readdr = re.compile("is at address (0x[0-9a-fA-F]{0,8})")
elif start:
readdr = re.compile("starts at address (0x[0-9a-fA-F]{0,8})")
else:
readdr = re.compile("and ends at (0x[0-9a-fA-F]{0,8})")
if not readdr:
return -1
addrg = readdr.search(output)
if not addrg:
return -1
res = int(addrg.group(1), 0)
if assembly and (not start):
res += 1 # give something larger for end endress for non-includive range
return res
def addr2funcnameobjdump(addr, stage, debug=False):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
cmd = "%sobjdump -D -w --start-address=0x%x --stop-address=0x%x %s 2>/dev/null" \
% (cc, addr, addr + 4, elf)
if debug:
print cmd
output = Main.shell.run_cmd(cmd).split("\n")
if debug:
print output
addrre = re.compile("%x" % addr)
output = [l for l in output if addrre.match(l)]
if debug:
print output
name = output[0].strip()
func = ""
if debug:
print name
rgx = re.compile(r'%x <([A-Za-z0-9_]+)(\+0x[0-9a-fA-F]+){0,1}>:' % addr)
res = re.search(rgx, name)
if res is None:
func = ''
else:
func = res.group(1)
return func
def __init__(self, stage, intervaltype=SUBSTAGEONLY):
self.h5file = None
self.h5mmap = None
self.h5group = None
self.var_table = None
self.h5mmapgroup = None
self.trace_intervals_table = None
self.contents_table = None
self.unresolved_interval_table = None
self.interval_type = intervaltype
self.stage = stage
self.substage_mmap_info_table = None
self.substage_mmap_addr_table = None
self.substage_reloc_info_table = None
self.substage_info_table = None
self.substage_region_policy_table = None
self.substage_file_path = Main.get_config("policy_file", self.stage)
self.mmap_file = Main.get_config("regions_file", self.stage)
def calculate_intervals(self, substages):
tracename = Main.get_config("trace_traces")
frama_c = "framac" in tracename
if frama_c:
intervals = self.calculate_framac_intervals(substages)
else:
intervals = self.calculate_trace_intervals(substages, tracename)
return intervals
def _resolve_var_addr(self, s, allregions, values):
val = s
split = s.split('.')
trysymbol = False
if '.'.join(split[:-1]) in allregions.iterkeys():
val = self._resolve_region_relative(s, allregions)
return val
if s in values.iterkeys():
val = values[s]
return val
if config.Main.stage_from_name(split[0]):
stage = config.Main.stage_from_name(split[0])
if not stage.post_build_setup_done:
stage.elf = Main.get_config('stage_elf', stage)
stage.image = Main.get_config('stage_image', stage)
stage.post_build_setup()
if len(split) > 1:
attr = split[1]
val = getattr(stage, attr, val)
return val
if s.startswith('.'):
end = '.end'
start = '.start'
if s.endswith(end):
(start,
val) = utils.get_section_location(re.sub(end, '', s),
self.stage)
elif s.endswith(start):
(val,
end) = utils.get_section_location(re.sub(start, '', s),
self.stage)
if type(val) == int and val < 0:
val = s
return val
if not s.startswith('0x'):
val = utils.get_symbol_location(s, self.stage, nm=True)
if val < 0:
val = s
return val
else:
val = int(s, 16)
return val
def __init__(self, srcfile, isasm, stage, path=""):
self.srcfile = srcfile
self.isasm = isasm
self.stagename = stage
if len(path) < 1:
root = Main.get_config("temp_target_src_dir")
if not root:
raise Exception("dont have temp soruce files yet")
else:
self.path = path
self.lineno = -1
def __init__(self, h5file, group, stage, name, desc=""):
self.h5file = h5file
self.group = group
self.stage = stage
self._name = name
self.desc = desc
self.tables = {}
self._mux_name = "set_muxconf_regs"
(self._mux_start, self._mux_end) = utils.get_symbol_location_start_end(
self._mux_name, self.stage)
self.thumbranges = Main.get_config("thumb_ranges", self.stage)[0]
self._mux_start += 2
self._mux_end -= 2
if self.thumbranges.overlaps_point(self._mux_start):
self.cs = capstone.Cs(CS_ARCH_ARM, CS_MODE_THUMB)
self.cs.detail = True
self._thumb = True
self.emu = Uc(UC_ARCH_ARM, UC_MODE_THUMB)
else:
self.cs = capstone.Cs(CS_ARCH_ARM, CS_MODE_ARM)
self.cs.detail = True
self._thumb = False
self.emu = Uc(UC_ARCH_ARM, UC_MODE_ARCH)
elf = Main.get_config("stage_elf", stage)
entrypoint = self._mux_start
headers = pure_utils.get_program_headers(Main.cc, elf)
for h in headers:
if h['filesz'] > 0:
codeaddr = h['virtaddr']
break
alignedstart = self._mux_start & 0xFFFFF0000
size = 2 * 1024 * 1024
fileoffset = alignedstart
code = open(elf, "rb").read()[self._mux_start -
fileoffset:self._mux_end - fileoffset]
self.emu.mem_map(0x40000000, 0x70000000)
self.emu.mem_write(self._mux_start, code)
self.emu.reg_write(0x40000000, ARM_REG_SP)
self.open()
def addr2functionname(addr, stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
srcdir = Main.get_config("temp_bootloader_src_dir")
cmd = '%sgdb --cd=%s ' \
'-ex "x/i 0x%x" --batch --nh --nx %s 2>/dev/null' % (cc,
srcdir,
addr,
elf)
output = Main.shell.run_cmd(cmd)
output = output.split('\n')[0].strip()
output = output.split(':')[0]
if output.lower() == '0x%x:':
return '' # not located in a function
else:
rgx = re.compile(r'<([A-Za-z0-9_]+)(\+\d+){0,1}>')
res = re.search(rgx, output)
if res is None:
return ''
else:
return res.group(1)
def finalize(self, args):
substage_names = {
s.stagename: self._stages[s.stagename].substages
for s in self.stage_order
if self._stages[s.stagename].substages is not None
}
stages = [s.stagename for s in self.stage_order]
doit_manager.TaskManager(False,
False,
stages,
substage_names,
False,
None,
self.test_trace_name,
False, [],
self.test_instance_name,
hook=True)
tmpdir = Main.get_config("temp_bootloader_src_dir")
gdb.execute("dir %s" % tmpdir)
gdb.execute("set substitute-path %s %s" %
(Main.get_bootloader_cfg().software_cfg.root, tmpdir))
self.hw = Main.get_config("trace_hw")
if self.hw.name == "bbxmqemu":
self.isbaremetal = False
else:
self.isbaremetal = True
if len(self.stage_order) == 0:
self.stage_order = [
Main.stage_from_name(s) for s in list(self._stages.iterkeys())
]
for stage in self.stage_order:
s = self._stages[stage.stagename]
s.init_with_test_instance()
# self.f_hooks = list(set(self.f_hooks))
for f in self.f_hooks:
f(args)
def get_symbol_location_start_end(name, stage, debug=False):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
start = get_symbol_location(name, stage, debug)
if start >= 0:
cmd = '%sreadelf -W -s %s | grep %s 2>/dev/null' % (cc, elf, name)
if debug:
print cmd
output = Main.shell.run_cmd(cmd)
if debug:
print output
size = int(output.split()[2])
return (start, start + size)
else:
return (0, 0)
def get_c_function_names(stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
cmd = '%sreadelf -W -s %s 2>/dev/null' % (cc, elf)
output = Main.shell.run_multiline_cmd(cmd)
regexp = re.compile("\s+\d+:\s+(?P<addr>[a-fA-f0-9]{8})"
"\s+\w*\s+(?P<t>[A-Z]+)\s+\w*\s+\w*\s+\w*\s+(?P<name>[\w_\-\.]+)\s*$")
results = []
for l in output:
m = regexp.search(l)
if m is not None:
(addr, name, t) = (m.groupdict()['addr'], m.groupdict()['name'], m.groupdict()['t'])
if t == "FUNC":
results.append((name, int(addr, 16)))
return results
def __init__(self, controller, needs_relocation, stage, **kwargs):
self.stage = stage
self.needs_relocation = needs_relocation
self.controller = controller
self.stophooks = self.controller.lookup_bp_hooks(self)
self.final_events = []
for (k, v) in kwargs.iteritems():
setattr(self, k, v)
if controller.isbaremetal:
pc = controller.get_reg_value("lr")
(ts, arms, ds) = Main.get_config("thumb_ranges", self.stage)
if not (ts.search(pc) or arms.search(pc)):
self.breakpoint = None
return
self.breakpoint = gdb.FinishBreakpoint.__init__(self, internal=True)
def set_mode(self):
if self.isbaremetal:
addr = self.get_reg_value('pc')
(ts, arms, ds) = Main.get_config("thumb_ranges",
self.current_stage)
# hack
if utils.addr2functionname(addr,
self.current_stage) == "clear_bss":
typ = "thumb"
elif arms.search(addr):
typ = "arm"
else:
typ = "thumb"
if not typ == self.core_state:
self.core_state = typ
gdb.execute("mon arm core_state %s" % typ, to_string=True)
def label_search(cls, label=None, root=""):
labels = []
if len(root) == 0:
root = Main.get_config("temp_bootloader_src_dir")
if not root:
raise Exception("dont have temp soruce files yet")
#root = Main.get_bootloader_root()
for (dirpath, dirs, files) in os.walk(root):
for filename in fnmatch.filter(files, "*.[chsS]"):
fullpath = os.path.join(dirpath, filename)
filepath = fullpath[len(root)+1:]
if os.path.isfile(fullpath): # just in case
filelabels = cls._get_labels(filepath, root, name="",
stage="", checkreqs=False, ltype=label)
if len(filelabels) > 0:
labels.extend(filelabels)
return labels
def symbol_relocation_file(name, offset, stage, path=None, debug=False):
if path is None:
path = tempfile.NamedTemporaryFile("rw").name
elf = Main.get_config("stage_elf", stage)
cc = Main.cc
cmd = "%sobjcopy --extract-symbol -w -N \!%s "\
"--change-addresses=0x%x %s %s 2>/dev/null" % (cc,
name,
offset,
elf,
path)
if debug:
print cmd
output = Main.shell.run_cmd(cmd)
if debug:
print output
return path
def continue_stage(self):
cont = self.controller
cont.disable_breakpoint(self)
if not gdb.current_progspace().filename:
elf = Main.get_config("stage_elf", self.stage)
gdb.execute("file %s" % elf)
cont.gdb_print('loaded file %s\n' % elf)
cont.gdb_print("Inserting breakpoints for %s %s ...\n" %
(self.controller.name, self.stage.stagename))
cont.current_substage = 0
cont.insert_breakpoints(self.stage)
cont.gdb_print("Done setting breakpoints\n")
for s in self.controller.stage_hooks:
s(self, self.stage)
cont.gdb_print("return\n")
def __init__(self, short_name, d, stage, parent=None, values={}):
if parent is None:
parent_type = None
parent_default_perms = None
parent_include_children = None
parent_reclassifiable = None
else:
parent_type = parent.typ
parent_default_perms = parent.default_perms
parent_include_children = parent.include_children
parent_reclassifiable = parent.reclassifiable
self.stage = stage
self.addresses = intervaltree.IntervalTree()
self.short_name = short_name
self.name = get_value(d, 'name')
self._raw_typ = get_value(d, 'type', parent_type).lower()
self._raw_addresses = get_value(d, 'addresses')
self._raw_default_perms = get_value(d, 'default_perms',
parent_default_perms)
self._raw_subregions = get_value(d, 'subregions')
self._raw_include_children = get_value(d, 'include_children',
parent_include_children)
self._raw_reclassifiable = get_value(d, 'reclassifiable',
parent_reclassifiable)
self._csv = get_value(d, 'csv')
if self._csv:
self._csv = os.path.join(Main.get_config('test_instance_root'),
self._csv)
if parent and parent._csv:
# if parent had csv, don't propigate csv definition
self._csv = None
self.contents = get_value(d, 'contents')
self.children_names = [
self.short_name + '.' + s for s in self._raw_subregions.iterkeys()
]
self.parent = parent
self.addresses_resolved = False
self._convert_from_raw(values)
self.resolve_addresses(values=values)
self.reclassification_rules = {0: self.typ}
def addr2disasmobjdump(addr, sz, stage, thumb=True, debug=False):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
cmd = "%sobjdump -D -w --start-address=0x%x --stop-address=0x%x -j .text %s 2>/dev/null" \
% (cc, addr, addr+sz, elf)
if debug:
print cmd
output = Main.shell.run_cmd(cmd).split("\n")
if len(output) < 2:
return (None, None, None, None)
if debug:
print output
addrre = re.compile("%x" % addr)
output = [l for l in output if addrre.match(l)]
name = output[0].strip()
disasm = output[1].strip()
func = ""
rgx = re.compile(r'<([A-Za-z0-9_]+)(\+0x[a-fA-F0-9]+){0,2}>:')
res = re.search(rgx, name)
if res is None:
func = ''
else:
func = res.group(1)
disasm = disasm.split('\t')
# convert to a hex string and then decode it to it is an array of bytes
value = (''. join(disasm[1].split())).decode('hex')
instr = ' '.join(disasm[2:])
if (not thumb) or (len(value) == 2):
value = value[::-1]
else:
sv = value[:2][::-1]
ev = value[2:][::-1]
value = sv+ev
return (value, instr, func)
def _create(self):
dbpath = Main.get_config("trace_db", self.stage)
self._db = database.TraceTable(dbpath, self.stage, True, True)
def _open(self, append=False):
dbpath = Main.get_config("trace_db", self.stage)
self._db = database.TraceTable(dbpath, self.stage, False, True)
print "nwrite %s (%s)" % (self._db.writestable.nrows, self.stage.stagename)
print self._db.outname
def _create(self):
self._db = addr_space.AddrSpaceInfo()
mmapdb_path = Main.get_config("mmapdb")
cvs = Main.get_config("reglist")
self._db.open_dbs(mmapdb_path, True, cvs)