def __init__(self, stage, intervaltype=SUBSTAGEONLY):
self.h5file = None
self.h5mmap = None
self.h5group = None
self.var_table = None
self.h5mmapgroup = None
self.trace_intervals_table = None
self.contents_table = None
self.unresolved_interval_table = None
self.interval_type = intervaltype
self.stage = stage
if isinstance(self.stage, str):
self.stage = Main.stage_from_name(self.stage)
self.substage_mmap_info_table = None
self.substage_mmap_addr_table = None
self.substage_reloc_info_table = None
self.substage_info_table = None
self.substage_region_policy_table = None
if not hasattr(Main.raw.policies, "substages_file"):
raise Exception(
"No substage or region definitions are available for processing"
)
self.substage_file_path = Main.get_policy_config(
"substages_file", self.stage)
self.mmap_file = Main.get_policy_config("regions_file", self.stage)
self.process_trace = False
def from_line(cls, line, stage):
res = cls.regexp.match(line.strip())
if res is None:
raise Exception("%s is not a framac dst result")
else:
min_value = int(res.group(1), 0)
max_value = int(res.group(2), 0)
if max_value > 0xFFFFFFFF:
max_value = 0xFFFFFFFF
lvalue = res.group(3)
path = res.group(4)
# somewhat of a hack to get relative path
root = Main.get_bootloader_cfg().software_cfg.root
if path.startswith(root):
path = os.path.relpath(path, root)
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
elif path.startswith(
Main.test_suite_path): # not sure why this happens
path = os.path.relpath(path, Main.test_suite_path)
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
elif path.startswith("/tmp/tmp"):
path = "/".join(path.split("/")[3:])
path = os.path.join(Main.get_config("temp_bootloader_src_dir"),
path)
lineno = int(res.group(5))
callstack = res.group(6)
# somewhat of a hack for muxconf
return cls(path,
lineno,
lvalue, [intervaltree.Interval(min_value, max_value)],
callstack=callstack,
stage=stage)
def __init__(self, print_build, do_build):
self.init_only = True if len(print_build) + len(do_build) == 0 else False
self.print_build = print_build
self.do_build = do_build
self.builds = list(set(do_build + print_build))
ss = Main.config_class_lookup("Software")
bootloader = Main.get_bootloader_cfg()
self.code_tasks = [CodeTaskList(s, s.name not in self.do_build)
for s in ss
if hasattr(s, "build_required")
and s.build_required
and (s.name in self.builds)]
# always need the bootloader
if bootloader.software not in self.builds:
self.code_tasks.extend(CodeTaskList(s,
s.name not in self.do_build)
for s in ss if s.name == bootloader.software)
if self.init_only:
for c in self.code_tasks:
for t in c.tasks:
t.uptodate = [True]
else:
for c in self.code_tasks:
if c.basename in self.do_build:
for t in c.tasks:
t.uptodate = [False]
def populate_contents_table(self):
stage = self.stage
substages = self._substage_numbers()
substagesnamename = self._substage_names()
tracename = Main.get_config("trace_traces")
if 'framac' in tracename:
row = self.contents_table.row
tracefile = Main.get_config("framac_callstacks", self.stage)
if os.path.exists(tracefile):
results = self.parse_frama_c_call_trace_stages(
tracefile, self.substage_file_path)
for s in substages:
for f in results[s]:
row["substagenum"] = s
row["functionname"] = f
row.append()
elif "baremetal" not in tracename:
raws = self.get_raw_files(False)
for (num, f) in raws.iteritems():
fopen = open(f, "r")
contents = fopen.read()
row = self.contents_table.row
for n in contents.split():
row["substagenum"] = num
row["functionname"] = n
row.append()
fopen.close()
self.contents_table.flush()
self.contents_table.cols.substagenum.reindex()
def addr2disasm(addr, stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
srcdir = Main.get_config("temp_bootloader_src_dir")
cmd = '%sgdb --cd=%s ' \
'-ex "disassemble/r 0x%x,+1" --batch '\
'--nh --nx %s 2>/dev/null' % (cc,
srcdir,
addr,
elf)
# print cmd
output = Main.shell.run_cmd(cmd)
voutput = output.split('\n')[1].strip() # get line with disassembly
voutput = voutput.split('\t')
# try:
# convert to a hex string and then decode it to it is an array of bytes
value = (''.join(voutput[1].split())).decode('hex')
# except Exception:
instr = ' '.join(voutput[2:])
# get function name if possible
foutput = output.split(':')[0]
if foutput.lower() == '0x%x:':
func = '' # not located in a function
else:
rgx = re.compile(r'<([A-Za-z0-9_]+)(\+\d+){0,1}>')
res = re.search(rgx, foutput)
if res is None:
func = ''
else:
func = res.group(1)
return (value, instr, func)
def write_substages_file(self):
stage = self.stage
substages = self._substage_names()
tracename = Main.get_config("trace_traces")
try:
cdb = Main.get_config("calltrace_db", self.stage)
except KeyError:
cdb = None
if cdb and os.path.exists(cdb):
path = Main.get_config("policy_trace_el", self.stage)
if os.path.exists(path):
return
calltrace_path = Main.get_config("calltrace_db", self.stage)
failed = False
substage_linenos = []
for s in substages:
if s == "_start":
n = 0
else:
if not failed:
try:
n = self.get_function_lineno(s, calltrace_path)
except subprocess.CalledProcessError:
print "Did not find %s in %s" % (s, calltrace_path)
failed = True
n = self.get_function_lineno(
s, calltrace_path, True)
substage_linenos.append(n)
outf = open(path, "w")
outf.write("(setq substages '(%s))\n" %
" ".join([str(i) for i in substage_linenos]))
outf.close()
def __init__(self, stage,
intervaltype=SUBSTAGEONLY):
self.h5file = None
self.h5mmap = None
self.h5group = None
self.var_table = None
self.h5mmapgroup = None
self.trace_intervals_table = None
self.contents_table = None
self.unresolved_interval_table = None
self.interval_type = intervaltype
self.stage = stage
if isinstance(self.stage, str):
self.stage = Main.stage_from_name(self.stage)
self.substage_mmap_info_table = None
self.substage_mmap_addr_table = None
self.substage_reloc_info_table = None
self.substage_info_table = None
self.substage_region_policy_table = None
self.valid = True
self.substage_file_path = None
self.mmap_file = None
self.process_trace = False
if not hasattr(Main.raw.policies, "substages_file"):
self.valid = False
return
if not hasattr(Main.raw.policies.substages_file, self.stage.stagename):
self.valid = False
return
self.substage_file_path = Main.get_policy_config("substages_file", self.stage)
self.mmap_file = Main.get_policy_config("regions_file", self.stage)
self.process_trace = False
def get_raw_files(self, noprepare):
stage = self.stage
substages = self._substage_numbers()
name = self._substage_names()
try:
substageresultsdir = Main.get_config("policy_trace_fnlist_dir",
self.stage)
except KeyError:
substageresultsdir = None
tracename = Main.get_config("trace_traces")
try:
calltrace_path = Main.get_config("calltrace_db", self.stage)
except KeyError:
calltrace_path = None
if calltrace_path and os.path.exists(
calltrace_path) and substageresultsdir:
if not noprepare:
el_path = Main.get_config("policy_trace_el", self.stage)
if os.path.exists(substageresultsdir):
return {}
pymacs_request.ask_emacs(
'(create-substage-calltraces "%s" "%s" "%s")' %
(calltrace_path, el_path, substageresultsdir))
origdir = os.getcwd()
os.chdir(substageresultsdir)
files = [
os.path.join(substageresultsdir, f) for f in glob.glob("*.txt")
]
if files:
files.sort()
return {i: files[i] for i in range(0, len(files))}
return {}
def disassemble(what, stage):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
srcdir = Main.get_config("temp_bootloader_src_dir")
cmd = '%sgdb --cd=%s ' \
'-ex "disassemble %s" --batch --nh --nx %s 2>/dev/null' % (cc,
srcdir,
what,
elf)
output = Main.shell.run_multiline_cmd(cmd)
return output
def _calculate_current_id(self):
(gitid, sha) = self.boot_task.get_gitinfo()
cc = Main.cc
cc_name = self.boot_task.build_cfg.compiler_name
ccpath = "%s%s" % (cc, cc_name)
defconfig = Main.get_bootloader_cfg().makecfg
hwclass = Main.get_hardwareclass_config().name
bootsoftware = self.boot_task.build_cfg.name
ccinfo = pure_utils.file_md5(ccpath)
gitinfo = {'local': self.boot_task.build_cfg.root, 'sha1': sha}
return ("%s.%s.%s.%s.%s" %
(hwclass, bootsoftware, defconfig, gitid, ccinfo), gitinfo)
def init_with_test_instance(self):
# update stage start/end info now that it has been calculated
self.stage = Main.stage_from_name(self.stage.stagename)
if self.substages:
self.policy_file = Main.get_config("policy_file", self.stage)
self.regions_file = Main.get_config("regions_file", self.stage)
self.substages_entrypoints = substage.SubstagesInfo.substage_names(
self.stage)
self._entrypoint = self.stage.entrypoint
self._exitpoint = self.stage.exitpc
if self._startpoint is None:
self._startpoint = self._entrypoint
def create_stageexit_table(self):
self.stageexits = self.h5file.create_table(self.group, 'stageexits',
StageExitInfo,
"stage exit info")
sls = WriteSearch.find_labels(labeltool.StageinfoLabel, "EXIT",
self.stage, "")
r = self.stageexits.row
for l in sls:
lineno = self._get_real_lineno(l)
if lineno < 0:
print "couldn't find label at %s" % l
continue
loc = "%s:%d" % (l.filename, lineno)
addr = utils.get_line_addr(
loc,
True,
self.stage,
srcdir=Main.get_runtime_config("temp_target_src_dir"))
success = True if l.name == "success" else False
r['addr'] = addr
r['addrlo'] = utils.addr_lo(addr)
r['addrhi'] = utils.addr_hi(addr)
r['line'] = loc
r['success'] = success
r.append()
self.stageexits.flush()
def __init__(self,
path,
lineno,
lvalue,
values,
pc=None,
origpc=None,
substage_name=None,
callstack="",
stage=None):
self.path = path
self.pc = pc
self.origpc = origpc
self.lineno = lineno
self.values = intervaltree.IntervalTree()
for v in values:
self.values.add(v)
self.lvalue = lvalue
self.stage = stage
if substage_name is None and callstack:
policy = Main.get_config('policy_file', self.stage)
self.substages = substage.SubstagesInfo.substage_names(self.stage)
self.substages[0] = "frama_go"
called_fns = callstack.split("->")
called_fns = filter(len, called_fns)
called_fns.reverse()
for f in called_fns:
if f in self.substages:
substage_name = self.substages.index(f)
break
self.substage = substage_name
def get_real_lineno(cls, l, prev, stage):
lineno = l.lineno
fullpath = os.path.join(l.path, l.filename)
addr = -1
while addr < 0:
if not prev:
lineno = labeltool.SrcLabelTool.get_next_non_label(
lineno, fullpath)
else:
lineno = labeltool.SrcLabelTool.get_prev_non_label(
lineno, fullpath)
# print "lineno %s, %s" % (lineno, fullpath)
if lineno is None:
addr = -1
break
addr = utils.get_line_addr(
"%s:%d" % (l.filename, lineno),
True,
stage,
srcdir=Main.get_runtime_config("temp_target_src_dir"))
if addr < 0:
return -1
else:
return lineno
def _create_var_table(self, substage=-1):
fields = ["startaddr", "size", "kind", "name"]
vtab = self.var_table
if vtab.nrows > 0:
return
cc = Main.cc
stage = self.stage
sname = stage.stagename
elf = Main.get_config("stage_elf", stage)
cmd = "%snm -n -S %s" % (cc, elf)
f = StringIO.StringIO(Main.shell.run_cmd(cmd))
reader = csv.DictReader(f,
fields,
delimiter=" ",
lineterminator="\n",
skipinitialspace=True)
row = vtab.row
for r in reader:
if r['name'] is None:
continue # this means there is no listed size
else:
row['name'] = r['name'].strip()
row['startaddr'] = int(r['startaddr'].strip(), 16)
row['endaddr'] = row['startaddr'] + int(r['size'].strip(), 16)
row['rawkind'] = r['kind'].strip()
k = row['rawkind'].lower()
if ('t' == k) or ('w' == k):
row['kind'] = getattr(addr_space.var_type, 'text')
else:
row['kind'] = getattr(addr_space.var_type, 'staticvar')
row['perms'] = getattr(addr_space.var_perms, 'rw')
row['substage'] = substage
row.append()
vtab.flush()
def get_line_addr(line, start, stage, debug=False, srcdir=None):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
if srcdir:
srcdir = "--cd=%s " % (srcdir)
cmd = "%sgdb %s -ex 'info line %s' --batch --nh --nx %s 2>/dev/null" % (cc,
srcdir,
line, elf)
if debug:
print cmd
output = Main.shell.run_multiline_cmd(cmd)
if debug:
print output
output = output[0]
assembly = False
if ("but contains no code." in output) and (".S\" is at address" in output):
if (".S\" is at address" in output): # is assembly
assembly = True
readdr = re.compile("is at address (0x[0-9a-fA-F]{0,8})")
elif start:
readdr = re.compile("starts at address (0x[0-9a-fA-F]{0,8})")
else:
readdr = re.compile("and ends at (0x[0-9a-fA-F]{0,8})")
if not readdr:
return -1
addrg = readdr.search(output)
if not addrg:
return -1
res = int(addrg.group(1), 0)
if assembly and (not start):
res += 1 # give something larger for end endress for non-includive range
return res
def addr2funcnameobjdump(addr, stage, debug=False):
cc = Main.cc
elf = Main.get_config("stage_elf", stage)
cmd = "%sobjdump -D -w --start-address=0x%x --stop-address=0x%x %s 2>/dev/null" \
% (cc, addr, addr + 4, elf)
if debug:
print cmd
output = Main.shell.run_cmd(cmd).split("\n")
if debug:
print output
addrre = re.compile("%x" % addr)
output = [l for l in output if addrre.match(l)]
if debug:
print output
name = output[0].strip()
func = ""
if debug:
print name
rgx = re.compile(r'%x <([A-Za-z0-9_]+)(\+0x[0-9a-fA-F]+){0,1}>:' % addr)
res = re.search(rgx, name)
if res is None:
func = ''
else:
func = res.group(1)
return func
def uboot_mux_init(self):
self._mux_name = "set_muxconf_regs"
(self._mux_start, self._mux_end) = utils.get_symbol_location_start_end(
self._mux_name, self.stage)
self._mux_start += 2
self._mux_end -= 2
if self.thumbranges.overlaps_point(self._mux_start):
self.cs = capstone.Cs(CS_ARCH_ARM, CS_MODE_THUMB)
self.cs.detail = True
self._thumb = True
self.emu = Uc(UC_ARCH_ARM, UC_MODE_THUMB)
else:
self.cs = capstone.Cs(CS_ARCH_ARM, CS_MODE_ARM)
self.cs.detail = True
self._thumb = False
self.emu = Uc(UC_ARCH_ARM, UC_MODE_ARM)
entrypoint = self._mux_start
headers = pure_utils.get_program_headers(Main.cc, elf)
for h in headers:
if h['filesz'] > 0:
codeaddr = h['virtaddr']
break
alignedstart = self._mux_start & 0xFFFFF0000
size = 2 * 1024 * 1024
fileoffset = alignedstart
elf = stage.elf
code = open(elf, "rb").read()[self._mux_start -
fileoffset:self._mux_end - fileoffset]
hw = Main.get_hardwareclass_config()
for i in hw.addr_range:
self.emu.mem_map(begin, (i.end + 1) - begin)
self.emu.mem_write(self._mux_start, code)
self.emu.reg_write(self.stage.elf.entrypoint, ARM_REG_SP)
def __init__(self, short_name, d, stage, parent=None, values={}):
if parent is None:
parent_type = None
parent_default_perms = None
parent_include_children = None
parent_reclassifiable = None
else:
parent_type = parent.typ
parent_default_perms = parent.default_perms
parent_include_children = parent.include_children
parent_reclassifiable = parent.reclassifiable
self.stage = stage
self.addresses = intervaltree.IntervalTree()
self.short_name = short_name
self.name = get_value(d, 'name')
self._raw_typ = get_value(d, 'type', parent_type).lower()
self._raw_addresses = get_value(d, 'addresses')
self._raw_default_perms = get_value(d, 'default_perms', parent_default_perms)
self._raw_subregions = get_value(d, 'subregions')
self._raw_include_children = get_value(d, 'include_children', parent_include_children)
self._raw_reclassifiable = get_value(d, 'reclassifiable', parent_reclassifiable)
self._csv = get_value(d, 'csv')
if self._csv:
self._csv = Main.populate_from_config(self._csv)
if parent and parent._csv:
# if parent had csv, don't propigate csv definition
self._csv = None
self.contents = get_value(d, 'contents')
self.children_names = [self.short_name + '.' + s for s in self._raw_subregions.iterkeys()]
self.parent = parent
self.addresses_resolved = False
self._convert_from_raw(values)
self.resolve_addresses(values=values)
self.reclassification_rules = {0: self.typ}
def __init__(self):
self.grpname = 'memory'
self.csvs = []
self.reg_csvs = []
for (f, i) in Main.get_hardwareclass_config()._files.iteritems():
if i.type == "mmap":
if getattr(i, "subtype", "") == "registers":
self.reg_csvs.append(Main.populate_from_config(i.path))
else:
self.csvs.append(Main.populate_from_config(i.path))
self.mem_tablename = "memmap"
self.reg_tablename = "regs"
self.h5group = None
self.h5file = None
self.memmap_table = None
self.reg_table = None
def lookup_label_addr(self, label):
lineno = WriteSearch.get_real_lineno(label, False, self.stage)
loc = "%s:%d" % (label.filename, lineno)
return utils.get_line_addr(
loc,
True,
self.stage,
srcdir=Main.get_runtime_config("temp_target_src_dir"))
def instances(cls,
stage,
root=Main.get_target_cfg().software_cfg.root,
quick=False):
files = cls.files
fs = map(lambda s: os.path.join(root, s), files)
fs = map(functools.partial(PreprocessedFileInstance, stage=stage), fs)
return fs
def __init__(self, option_strings, dest, **kwargs):
self.stages = Main.stages
self.stagenames = [s.stagename for s in self.stages]
if len(self.stages) == 1:
self.nargs = 2
else:
self.nargs = 3
name = Main.get_hardwareclass_config().name
path = Main.get_hardwareclass_config().hw_info_path
defaultdir = os.path.join(
Main.get_hardwareclass_config().hw_info_path, name)
self.sdefaults = {}
kwargs['default'] = self.sdefaults
super(SubstageFileAction, self).__init__(option_strings, dest,
**kwargs)
def __init__(self, option_strings, dest, **kwargs):
stages = list(
Main.get_bootloader_cfg().supported_stages.itervalues())
self.stagenames = [s.stagename for s in stages]
self.nargs = 2
defaults = {}
kwargs['default'] = defaults
super(SubstageNameAction, self).__init__(option_strings, dest,
**kwargs)
def _get_line_addr(self, line, start=True, framac=False):
if framac:
return self.get_framac_line_addr(line, start)
else:
return utils.get_line_addr(
line,
start,
self.stage,
srcdir=Main.get_runtime_config("temp_target_src_dir"))
def calculate_intervals(self, substages):
tracename = Main.get_config("trace_traces")
frama_c = "framac" in tracename
if frama_c:
intervals = self.calculate_framac_intervals(substages)
else:
intervals = self.calculate_trace_intervals(substages, tracename)
return intervals
def format_command(self, cmd):
config_type = self.build_cfg.config_type if hasattr(self.build_cfg, "config_type") else ""
config_name = self.build_cfg.config_name if hasattr(self.build_cfg, "config_name") else ""
if config_type and config_name:
cfg = Main.object_config_lookup(config_type, config_name)
keywords = Main.__dict__
keywords.update(cfg.__dict__)
cmd = cmd.format(**keywords)
return cmd
def __init__(self):
self.grpname = 'memory'
self.base_memory_csv = Main.get_hardwareclass_config().base_mem_map
self.mem_tablename = "memmap"
self.reg_tablename = "regs"
self.h5group = None
self.h5file = None
self.memmap_table = None
self.reg_table = None
def __init__(self, stage, intervaltype=SUBSTAGEONLY):
self.h5file = None
self.h5mmap = None
self.h5group = None
self.var_table = None
self.h5mmapgroup = None
self.trace_intervals_table = None
self.contents_table = None
self.unresolved_interval_table = None
self.interval_type = intervaltype
self.stage = stage
self.substage_mmap_info_table = None
self.substage_mmap_addr_table = None
self.substage_reloc_info_table = None
self.substage_info_table = None
self.substage_region_policy_table = None
self.substage_file_path = Main.get_config("policy_file", self.stage)
self.mmap_file = Main.get_config("regions_file", self.stage)
def __init__(self, createdb, stage, verbose=False, readonly=False):
self.verbose = verbose
outfile = Main.get_static_analysis_config("db", stage)
self.stage = stage
self.ia = InstructionAnalyzer()
self.relocstable = None
self.stageexits = None
self.writestable = None
self.smcstable = None
self.srcstable = None
self.funcstable = None
self.longwritestable = None
self.skipstable = None
self.verbose = verbose
(self._thumbranges, self._armranges, self._dataranges) = (None, None,
None)
if createdb:
m = "w"
self.h5file = tables.open_file(outfile,
mode=m,
title="%s target static analysis" %
stage.stagename)
self.group = self.h5file.create_group(
"/", 'staticanalysis',
"%s target static analysis" % stage.stagename)
else:
mo = "a"
self.h5file = tables.open_file(outfile,
mode=mo,
title="%s target static analysis" %
stage.stagename)
self.group = self.h5file.get_node("/staticanalysis")
r2.cd(self.stage.elf, Main.get_runtime_config("temp_target_src_dir"))
def q():
try:
r2.files[self.stage.elf].quit()
except IOError:
pass
atexit.register(q)