def by(**type):
'''Return the segment matching the specified keywords in `type`.'''
searchstring = ', '.join("{:s}={!r}".format(key, value)
for key, value in six.iteritems(type))
res = builtins.list(__iterate__(**type))
if len(res) > 1:
maxaddr = max(builtins.map(operator.attrgetter('endEA'), res) or [1])
caddr = math.ceil(math.log(maxaddr) / math.log(16))
builtins.map(
logging.info,
(("[{:d}] {:0{:d}x}:{:0{:d}x} {:s} {:+#x} sel:{:04x} flags:{:02x}".
format(seg.index, seg.startEA, int(caddr), seg.endEA, int(caddr),
idaapi.get_true_segm_name(seg), seg.size(), seg.sel,
seg.flags)) for seg in res))
logging.warn(
"{:s}.by({:s}) : Found {:d} matching results. Returning the first segment at index {:d} from {:0{:d}x}<>{:0{:d}x} with the name {:s} and size {:+#x}."
.format(__name__, searchstring, len(res), res[0].index,
res[0].startEA, int(caddr), res[0].endEA, int(caddr),
idaapi.get_true_segm_name(res[0]), res[0].size()))
res = next(iter(res), None)
if res is None:
raise E.SearchResultsError(
"{:s}.by({:s}) : Found 0 matching results.".format(
__name__, searchstring))
return res
def list(**type):
'''List all of the enumerations within the database that match the keyword specified by `type`.'''
res = builtins.list(iterate(**type))
maxindex = max(builtins.map(idaapi.get_enum_idx, res) or [1])
maxname = max(
builtins.map(utils.fcompose(idaapi.get_enum_name, len), res) or [0])
maxsize = max(builtins.map(size, res) or [0])
cindex = math.ceil(math.log(maxindex or 1) / math.log(10))
try:
cmask = max(
builtins.map(
utils.fcompose(
mask,
utils.fcondition(utils.fpartial(operator.eq, 0))(
utils.fconstant(1), utils.fidentity), math.log,
functools.partial(operator.mul, 1.0 / math.log(8)),
math.ceil), res) or [database.config.bits() / 4.0])
except:
cmask = 0
for n in res:
six.print_(
"[{:{:d}d}] {:>{:d}s} & {:<{:d}x} ({:d} members){:s}".format(
idaapi.get_enum_idx(n), int(cindex), idaapi.get_enum_name(n),
maxname, mask(n), int(cmask), len(builtins.list(members(n))),
" // {:s}".format(comment(n)) if comment(n) else ''))
return
def by(**type):
'''Return the segment matching the specified keywords in `type`.'''
searchstring = utils.string.kwargs(type)
get_segment_name = idaapi.get_segm_name if hasattr(
idaapi, 'get_segm_name') else idaapi.get_true_segm_name
listable = builtins.list(__iterate__(**type))
if len(listable) > 1:
maxaddr = max(builtins.map(interface.range.end, listable) or [1])
caddr = math.ceil(math.log(maxaddr) / math.log(16))
builtins.map(
logging.info,
((u"[{:d}] {:0{:d}x}:{:0{:d}x} {:s} {:+#x} sel:{:04x} flags:{:02x}"
.format(seg.index, interface.range.start(seg), int(caddr),
interface.range.end(seg), int(caddr),
utils.string.of(get_segment_name(seg)), seg.size(),
seg.sel, seg.flags)) for seg in listable))
logging.warn(
u"{:s}.by({:s}) : Found {:d} matching results. Returning the first segment at index {:d} from {:0{:d}x}<>{:0{:d}x} with the name {:s} and size {:+#x}."
.format(__name__, searchstring, len(listable), listable[0].index,
interface.range.start(listable[0]), int(caddr),
interface.range.end(listable[0]), int(caddr),
utils.string.of(get_segment_name(listable[0])),
listable[0].size()))
res = six.next(iter(listable), None)
if res is None:
raise E.SearchResultsError(
u"{:s}.by({:s}) : Found 0 matching results.".format(
__name__, searchstring))
return res
def list(**type):
'''List all of the segments in the database that match the keyword specified by `type`.'''
res = builtins.list(__iterate__(**type))
maxindex = max(builtins.map(operator.attrgetter('index'), res) or [1])
maxaddr = max(builtins.map(operator.attrgetter('endEA'), res) or [1])
maxsize = max(builtins.map(operator.methodcaller('size'), res) or [1])
maxname = max(
builtins.map(utils.fcompose(idaapi.get_true_segm_name, len), res)
or [1])
cindex = math.ceil(math.log(maxindex or 1) / math.log(10))
caddr = math.ceil(math.log(maxaddr or 1) / math.log(16))
csize = math.ceil(math.log(maxsize or 1) / math.log(16))
for seg in res:
comment = idaapi.get_segment_cmt(seg, 0) or idaapi.get_segment_cmt(
seg, 1)
six.print_(
u"[{:{:d}d}] {:#0{:d}x}<>{:#0{:d}x} : {:<+#{:d}x} : {:>{:d}s} : sel:{:04x} flags:{:02x}{:s}"
.format(
seg.index, int(cindex), seg.startEA, 2 + int(caddr), seg.endEA,
2 + int(caddr), seg.size(), 3 + int(csize),
utils.string.of(idaapi.get_true_segm_name(seg)),
maxname, seg.sel, seg.flags, u"// {:s}".format(
utils.string.of(comment)) if comment else ''))
return
def list(cls, enum):
'''List all the members belonging to the enumeration identified by `enum`.'''
# FIXME: make this consistent with every other .list using the matcher class
eid = by(enum)
res = builtins.list(cls.iterate(eid))
maxindex = max(builtins.map(utils.first, enumerate(res)) or [1])
maxvalue = max(builtins.map(utils.fcompose(member.value, "{:#x}".format, len), res) or [1])
for i, mid in enumerate(res):
six.print_(u"[{:d}] 0x{:>0{:d}x} {:s}".format(i, member.value(mid), maxvalue, member.name(mid)))
return
def list(cls, enum):
'''List all the members belonging to the enumeration identified by `enum`.'''
# FIXME: make this consistent with every other .list using the matcher class
eid = by(enum)
res = builtins.list(cls.iterate(eid))
maxindex = max(builtins.map(utils.first, enumerate(res)) or [1])
maxvalue = max(builtins.map(utils.fcompose(member.value, "{:#x}".format, len), res) or [1])
for i, mid in enumerate(res):
six.print_(u"[{:d}] 0x{:>0{:d}x} {:s}".format(i, member.value(mid), maxvalue, member.name(mid)))
return
def recovermarks():
"""Walk through the tags made by ``colormarks`` and re-create the marks that were found.
This is useful if any marks were accidentally deleted and can be used for
recovering them as long as they were initally tagged properly.
"""
# collect
result = []
for fn, l in database.select('marks'):
m = set((l['marks']) if hasattr(l['marks'], '__iter__') else
[int(x, 16) for x in l['marks'].
split(',')] if type(l['marks']) is str else [l['marks']])
res = [(ea, d['mark']) for ea, d in func.select(fn, 'mark')]
if m != {a for a, _ in res}:
logging.warning(
"{:s} : Ignoring the function tag \"{:s}\" for function {:#x} due to its value being out-of-sync with the contents values ({!s} <> {!s})."
.format('.'.join((__name__, 'recovermarks')), fn,
builtins.map(hex, m),
builtins.map(hex, set(a for a, _ in res))))
result.extend(res)
result.sort(cmp=lambda x, y: cmp(x[1], y[1]))
# discovered marks versus database marks
result = dict(result)
current = {ea: descr for ea, descr in database.marks()}
# create tags
for x, y in result.items():
if x in current:
logging.warning(
"{:#x}: skipping already existing mark : {!r}".format(
x, current[x]))
continue
# x not in current
if x not in current:
logging.info("{:#x}: adding missing mark due to tag : {!r}".format(
x, result[x]))
elif current[x] != result[x]:
logging.info(
"{:#x}: database tag is different than mark description : {!r}"
.format(x, result[x]))
else:
assert current[x] == result[x]
database.mark(x, y)
# marks that aren't reachable in the database
for ea in set(current.viewkeys()).difference(result.viewkeys()):
logging.warning("{:#x}: unreachable mark (global) : {!r}".format(
ea, current[ea]))
# color them
colormarks()
def list(**type):
'''List all of the enumerations within the database that match the keyword specified by `type`.'''
res = builtins.list(iterate(**type))
maxindex = max(builtins.map(idaapi.get_enum_idx, res) or [1])
maxname = max(builtins.map(utils.fcompose(idaapi.get_enum_name, len), res) or [0])
maxsize = max(builtins.map(size, res) or [0])
cindex = math.ceil(math.log(maxindex or 1)/math.log(10))
try: cmask = max(builtins.map(utils.fcompose(mask, utils.fcondition(utils.fpartial(operator.eq, 0))(utils.fconstant(1), utils.fidentity), math.log, functools.partial(operator.mul, 1.0/math.log(8)), math.ceil), res) or [database.config.bits()/4.0])
except: cmask = 0
for n in res:
name = idaapi.get_enum_name(n)
six.print_(u"[{:{:d}d}] {:>{:d}s} & {:<{:d}x} ({:d} members){:s}".format(idaapi.get_enum_idx(n), int(cindex), utils.string.of(name), maxname, mask(n), int(cmask), len(builtins.list(members(n))), u" // {:s}".format(comment(n)) if comment(n) else ''))
return
def by(**type):
'''Return the segment matching the specified keywords in `type`.'''
searchstring = utils.string.kwargs(type)
res = builtins.list(__iterate__(**type))
if len(res) > 1:
maxaddr = max(builtins.map(operator.attrgetter('endEA'), res) or [1])
caddr = math.ceil(math.log(maxaddr)/math.log(16))
builtins.map(logging.info, ((u"[{:d}] {:0{:d}x}:{:0{:d}x} {:s} {:+#x} sel:{:04x} flags:{:02x}".format(seg.index, seg.startEA, int(caddr), seg.endEA, int(caddr), utils.string.of(idaapi.get_true_segm_name(seg)), seg.size(), seg.sel, seg.flags)) for seg in res))
logging.warn(u"{:s}.by({:s}) : Found {:d} matching results. Returning the first segment at index {:d} from {:0{:d}x}<>{:0{:d}x} with the name {:s} and size {:+#x}.".format(__name__, searchstring, len(res), res[0].index, res[0].startEA, int(caddr), res[0].endEA, int(caddr), utils.string.of(idaapi.get_true_segm_name(res[0])), res[0].size()))
res = next(iter(res), None)
if res is None:
raise E.SearchResultsError(u"{:s}.by({:s}) : Found 0 matching results.".format(__name__, searchstring))
return res
def list(**type):
'''List all of the segments in the database that match the keyword specified by `type`.'''
res = builtins.list(__iterate__(**type))
maxindex = max(builtins.map(operator.attrgetter('index'), res) or [1])
maxaddr = max(builtins.map(operator.attrgetter('endEA'), res) or [1])
maxsize = max(builtins.map(operator.methodcaller('size'), res) or [1])
maxname = max(builtins.map(utils.fcompose(idaapi.get_true_segm_name,len), res) or [1])
cindex = math.ceil(math.log(maxindex or 1)/math.log(10))
caddr = math.ceil(math.log(maxaddr or 1)/math.log(16))
csize = math.ceil(math.log(maxsize or 1)/math.log(16))
for seg in res:
comment = idaapi.get_segment_cmt(seg, 0) or idaapi.get_segment_cmt(seg, 1)
six.print_(u"[{:{:d}d}] {:#0{:d}x}<>{:#0{:d}x} : {:<+#{:d}x} : {:>{:d}s} : sel:{:04x} flags:{:02x}{:s}".format(seg.index, int(cindex), seg.startEA, 2+int(caddr), seg.endEA, 2+int(caddr), seg.size(), 3+int(csize), utils.string.of(idaapi.get_true_segm_name(seg)), maxname, seg.sel, seg.flags, u"// {:s}".format(utils.string.of(comment)) if comment else ''))
return
def _collect(addr, result):
process = set()
for blk in builtins.map(func.block, func.block.after(addr)):
if any(blk in coll for coll in (result, sentinel)):
continue
process.add(blk)
for addr, _ in process:
result |= _collect(addr, result | process)
return result
def _collect(addr, result):
process = set()
for blk in builtins.map(func.block, func.block.after(addr)):
if any(blk in coll for coll in (result, sentinel)):
continue
process.add(blk)
for addr, _ in process:
result |= _collect(addr, result | process)
return result
def groupby(sequence, count):
'''Group sequence by number of elements'''
idata = enumerate(sequence)
def fkey(item):
(index, value) = item
return index // count
for key, res in itertools.groupby(idata, fkey):
yield builtins.map(operator.itemgetter(1), res)
return
def __iterate__(**type):
'''Iterate through each segment defined in the database that match the keywords specified by `type`.'''
if not type: type = {'predicate':lambda n: True}
def newsegment(index):
res = idaapi.getnseg(index)
res.index = index
return res
res = builtins.map(newsegment, six.moves.range(idaapi.get_segm_qty()))
for key, value in six.iteritems(type):
res = builtins.list(__matcher__.match(key, value, res))
for item in res: yield item
def parallelMap(f, items, nProcs=None):
if nProcs == 1:
return builtins.map(f, items)
else:
name = parallelMap.i
parallelMap.scope[name] = (f, items)
parallelMap.i += 1
pool = multiprocessing.Pool(nProcs)
result = pool.map(_parallel_func,
((name, i) for i in range(len(items))))
pool.close()
del parallelMap.scope[name]
return result
def __iterate__(**type):
'''Iterate through each segment defined in the database that match the keywords specified by `type`.'''
if not type: type = {'predicate': lambda n: True}
def newsegment(index):
res = idaapi.getnseg(index)
res.index = index
return res
res = builtins.map(newsegment, six.moves.range(idaapi.get_segm_qty()))
for key, value in six.iteritems(type):
res = builtins.list(__matcher__.match(key, value, res))
for item in res:
yield item
def ppmap(processes, function, sequence, *sequences):
"""Split the work of 'function' across the given number of
processes. Set 'processes' to None to let Parallel Python
autodetect the number of children to use.
Although the calling semantics should be identical to
__builtin__.map (even using __builtin__.map to process
arguments), it differs in that it returns a generator instead of a
list. This enables lazy evaluation of the results so that other
work can be done while the subprocesses are still running.
>>> def rangetotal(n): return n, sum(range(n))
>>> list(map(rangetotal, range(1, 6)))
[(1, 0), (2, 1), (3, 3), (4, 6), (5, 10)]
>>> list(ppmap(1, rangetotal, range(1, 6)))
[(1, 0), (2, 1), (3, 3), (4, 6), (5, 10)]
"""
# Create a new server if one isn't already initialized
if not __STATE["server"]:
__STATE["server"] = pp.Server()
def submit(*args):
"""Send a job to the server"""
return __STATE["server"].submit(function, args, globals=globals())
# Merge all the passed-in argument lists together. This is done
# that way because as with the map() function, at least one list
# is required but the rest are optional.
a = [sequence]
a.extend(sequences)
# Set the requested level of multi-processing
__STATE["server"].set_ncpus(processes or "autodetect")
# First, submit all the jobs. Then harvest the results as they
# come available.
return (subproc() for subproc in builtins.map(submit, *a))
def groupby(sequence, count):
'''Group sequence by number of elements'''
key, data = lambda (index, value): index / count, enumerate(sequence)
for key, res in itertools.groupby(data, key):
yield builtins.map(operator.itemgetter(1), res)
return
def groupby(sequence, count):
'''Group sequence by number of elements'''
key, data = lambda (index, value): index / count, enumerate(sequence)
for key, res in itertools.groupby(data, key):
yield builtins.map(operator.itemgetter(1), res)
return
'fitem', 'fhasattr', 'fattributeQ', 'fgetattr', 'fattribute', 'fconstant',
'fpassthru', 'fdefault', 'fpass', 'fidentity', 'fid', 'first', 'second',
'third', 'last', 'fcompose', 'fdiscard', 'fcondition', 'fmap', 'flazy',
'fmemo', 'fpartial', 'fapply', 'fcurry', 'frpartial', 'freverse',
'freversed', 'fexc', 'fexception', 'fcatch', 'fcomplement', 'fnot',
'ilist', 'liter', 'ituple', 'titer', 'itake', 'iget', 'imap', 'ifilter',
'ichain', 'izip', 'count'
]
### functional programming primitives (FIXME: probably better to document these with examples)
# box any specified arguments
fbox = fboxed = lambda *a: a
# return a closure that executes `f` with the arguments unboxed.
funbox = lambda f, *a, **k: lambda *ap, **kp: f(
*(a + builtins.reduce(operator.add, builtins.map(builtins.tuple, ap),
())), **builtins.dict(k.items() + kp.items()))
# return a closure that will check that `object` is an instance of `type`.
finstance = lambda *type: frpartial(builtins.isinstance, type)
# return a closure that will check if its argument has an item `key`.
fhasitem = fitemQ = lambda key: fcompose(
fcatch(frpartial(operator.getitem, key)), iter, next,
fpartial(operator.eq, None))
# return a closure that will get a particular element from an object
fgetitem = fitem = lambda item, *default: lambda object: default[
0] if default and item not in object else object[item]
# return a closure that will check if its argument has an `attribute`.
fhasattr = fattributeQ = lambda attribute: frpartial(hasattr, attribute)
# return a closure that will get a particular attribute from an object
fgetattr = fattribute = lambda attribute, *default: lambda object: getattr(
object, attribute, *default)
def map(function, *sequence):
""" map(function, sequence[, sequence, ...]) -> list
Like the builtin map() function, but splits the workload across a
pool of processes whenever possible.
"""
# IPC stuff
structformat = 'L' #'H' #not everything there is short enough.
structlen = struct.calcsize(structformat)
def sendmessage(myend, message):
"""Send a pickled message across a pipe"""
outobj = pickle.dumps(message)
os.write(myend, struct.pack(structformat, len(outobj)) + outobj)
def recvmessage(myend):
"""Receive a pickled message from a pipe"""
length = struct.unpack(structformat, (os.read(myend, structlen)))[0]
return pickle.loads(os.read(myend, length))
try:
maxchildren = function.parallel_maxchildren
except AttributeError:
return builtins.map(function, *sequence)
# Handle map()'s multi-sequence semantics
if len(sequence) == 1:
if function is None:
return list(sequence[0])
arglist = zip(sequence[0])
else:
arglist = builtins.map(None, *sequence)
if function is None:
return arglist
argindex = 0
finished = 0
outlist = [None] * len(arglist)
# Spawn the worker children. Don't create more than the number of
# values we'll be processing.
fromchild, toparent = os.pipe()
children = []
for childnum in range(min(maxchildren, len(arglist))):
fromparent, tochild = os.pipe()
pid = os.fork()
# Parent?
if pid:
# Do some housekeeping and give the child its first assignment
children.append({
'pid': pid,
'fromparent': fromparent,
'tochild': tochild,
})
sendmessage(tochild, (argindex, arglist[argindex]))
argindex += 1
# Child?
else:
# Since children can't really tell when they've been
# orphaned, set a timeout so that they die if they don't
# hear from the parent in a timely manner.
def timeouthandler(signum, frame):
"""Get out cleanly"""
sys.exit()
oldsignal = signal.signal(signal.SIGALRM, timeouthandler)
# Keep processing values until the parent kills you
while True:
try:
# Wait one second before quitting. Children
# should generally hear from their parent almost
# instantly.
signal.alarm(1)
message = recvmessage(fromparent)
signal.alarm(0)
if message is None:
sys.exit()
index, value = message
sendmessage(toparent, (childnum, index, function(*value)))
except Exception as excvalue:
try:
excvalue.tb = traceback.extract_tb(sys.exc_info()[2])
except Exception as e2:
try:
excvalue.e2 = e2
except Exception as e3:
pass
sendmessage(toparent, (childnum, index, excvalue))
finally:
signal.signal(signal.SIGALRM, oldsignal)
# Keep accepting values back from the children until they've all
# come back
while finished < len(arglist):
returnchild, returnindex, value = recvmessage(fromchild)
if isinstance(value, Exception):
try:
print("Child traceback:")
print(''.join(traceback.format_list(value.tb)))
except Exception as e2:
pass
raise value
outlist[returnindex] = value
finished += 1
# If there are still values left to process, hand one back out
# to the child that just finished
if argindex < len(arglist):
sendmessage(children[returnchild]['tochild'],
(argindex, arglist[argindex]))
argindex += 1
# Kill the child processes
for child in children:
sendmessage(child['tochild'], None)
for child in children:
os.wait()
return outlist
def recovermarks():
"""Walk through the tags made by ``colormarks`` and re-create the marks that were found.
This is useful if any marks were accidentally deleted and can be used for
recovering them as long as they were initally tagged properly.
"""
# collect
result = []
for fn, l in database.select('marks'):
m = set( (l['marks']) if hasattr(l['marks'], '__iter__') else [int(x, 16) for x in l['marks'].split(',')] if type(l['marks']) is str else [l['marks']])
res = [(ea, d['mark']) for ea, d in func.select(fn, 'mark')]
if m != { a for a, _ in res }:
logging.warning("{:s} : Ignoring the function tag \"{:s}\" for function {:#x} due to its value being out-of-sync with the contents values ({!s} <> {!s}).".format('.'.join((__name__, 'recovermarks')), fn, builtins.map(hex, m), builtins.map(hex, set(a for a, _ in res))))
result.extend(res)
result.sort(cmp=lambda x, y: cmp(x[1], y[1]))
# discovered marks versus database marks
result = dict(result)
current = {ea : descr for ea, descr in database.marks()}
# create tags
for x, y in result.items():
if x in current:
logging.warning("{:#x}: skipping already existing mark : {!r}".format(x, current[x]))
continue
# x not in current
if x not in current:
logging.info("{:#x}: adding missing mark due to tag : {!r}".format(x, result[x]))
elif current[x] != result[x]:
logging.info("{:#x}: database tag is different than mark description : {!r}".format(x, result[x]))
else:
assert current[x] == result[x]
database.mark(x, y)
# marks that aren't reachable in the database
for ea in set(current.viewkeys()).difference(result.viewkeys()):
logging.warning("{:#x}: unreachable mark (global) : {!r}".format(ea, current[ea]))
# color them
colormarks()
