def __setstate__(self, state):
self.eprobs = state[0]
self.IS_LEAF = state[1]
self.column = state[2]
self.cutoff = state[3]
self._less_tree = state[4]
self._ge_tree = state[5]
self.g = DirectedGraph()
self.g.add_node(self)
self.probs = 0.5 * self.eprobs
self.probs = self.probs / numpy.sum(self.probs)
if self._less_tree != None:
assert self._ge_tree != None
self.g.union(self._less_tree.g)
self.g.union(self._ge_tree.g)
self._less_tree.g = self.g
self._ge_tree.g = self.g
less_edge = self.__symbolic_column_name__(
self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
def forward_data_flow(source, ea=None, calldepth=0):
if ea == None:
ea = ScreenEA()
_clear_colored()
inst, dst, src = wilds("inst dst src")
w = WildResults()
tainted = VersionedSet()
tainted.version = -1
tainted.add(source)
def _fix_esp(ea, exp):
spd = GetSpd(ea)
return exp.substitute({esp: (esp + spd).simplify()})
fg = FunctionGraph(ea)
# data connections graph
TAINTED = symbols("TAINTED")
dg = DirectedGraph()
dg.connect(TAINTED, source)
for addr, level in fg.walk(ea, depthfirst=True):
if level <= tainted.version:
print "reverting to version %s" % (level - 1)
tainted = tainted.get_version(level - 1)
tainted.version = level
syminst = symdecode(addr)
if syminst.match(inst(dst, src), w) and w.inst in tainted_dst_src_insts:
print "analyzing %s" % (syminst,)
# untaint cleared registers
if syminst.match(XOR(dst, dst)) and w.dst in tainted:
tainted.remove(w.dst)
elif w.src in tainted:
_color(addr)
print "tainting %s" % (w.dst,)
tainted.add(w.dst)
elif w.dst in tainted:
tainted.remove(w.dst)
return tainted
def __setstate__(self, state):
self.eprobs = state[0]
self.IS_LEAF = state[1]
self.column = state[2]
self.cutoff = state[3]
self._less_tree = state[4]
self._ge_tree = state[5]
self.g = DirectedGraph()
self.g.add_node(self)
self.probs = 0.5 * self.eprobs
self.probs = self.probs / numpy.sum(self.probs)
if self._less_tree != None:
assert self._ge_tree != None
self.g.union(self._less_tree.g)
self.g.union(self._ge_tree.g)
self._less_tree.g = self.g
self._ge_tree.g = self.g
less_edge = self.__symbolic_column_name__(self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
class Fuzzy(object):
def __init__(self):
self.graph = DirectedGraph()
def add(self, prop):
self.graph.add_node(prop)
if prop.is_Function:
for a in prop.args:
self.add(a)
self.graph.connect(a, prop)
if str(prop.func) in bidirectional:
self.graph.connect(prop, a)
def implies(self, condition, result):
self.add(condition)
self.add(result)
self.graph.connect(condition, result, "Imp")
def __init__(self,
X,
oracle,
target=0.6,
maxdepth=5,
g=None,
eprobs=None,
bias=0.0):
if g == None:
g = DirectedGraph()
if eprobs == None:
eprobs = numpy.array([1.0, 1.0])
self.g = g
self.eprobs = eprobs
self.IS_LEAF = False
self.column = None
self.cutoff = None
self._less_tree = None
self._ge_tree = None
probs = 0.5 * self.eprobs
probs = probs / numpy.sum(probs)
self.probs = probs
met_target = numpy.all(
[p <= 1.0 - target or p >= target for p in probs])
self.g.add_node(self)
if met_target:
self._make_leaf()
elif X.shape[0] < 10:
self._make_leaf()
elif maxdepth <= 1:
self._make_leaf()
else:
self._make_split(X, oracle, target, maxdepth, bias)
class OneClassTree(object):
def __init__(self, X, oracle, target=0.6, maxdepth=5, g=None, eprobs=None, bias=0.0):
if g == None:
g = DirectedGraph()
if eprobs == None:
eprobs = numpy.array([1.0, 1.0])
self.g = g
self.eprobs = eprobs
self.IS_LEAF = False
self.column = None
self.cutoff = None
self._less_tree = None
self._ge_tree = None
probs = 0.5 * self.eprobs
probs = probs / numpy.sum(probs)
self.probs = probs
met_target = numpy.all([p <= 1.0 - target or p >= target for p in probs])
self.g.add_node(self)
if met_target:
self._make_leaf()
elif X.shape[0] < 10:
self._make_leaf()
elif maxdepth <= 1:
self._make_leaf()
else:
self._make_split(X, oracle, target, maxdepth, bias)
def selected_columns(self):
if self.IS_LEAF:
return set([])
else:
left = self._less_tree.selected_columns()
right = self._ge_tree.selected_columns()
middle = set([self.column])
return middle.union(left).union(right)
def __getstate__(self):
return (self.eprobs, self.IS_LEAF, self.column, self.cutoff, self._less_tree, self._ge_tree)
def __setstate__(self, state):
self.eprobs = state[0]
self.IS_LEAF = state[1]
self.column = state[2]
self.cutoff = state[3]
self._less_tree = state[4]
self._ge_tree = state[5]
self.g = DirectedGraph()
self.g.add_node(self)
self.probs = 0.5 * self.eprobs
self.probs = self.probs / numpy.sum(self.probs)
if self._less_tree != None:
assert self._ge_tree != None
self.g.union(self._less_tree.g)
self.g.union(self._ge_tree.g)
self._less_tree.g = self.g
self._ge_tree.g = self.g
less_edge = self.__symbolic_column_name__(self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
def classify(self, x):
assert x.shape[0] > 0
if x.shape[0] > 1:
rv = numpy.vstack([self.classify(x[i]) for i in range(x.shape[0])])
elif self.IS_LEAF:
rv = self.eprobs
else:
rv = self._less_tree.classify(x) if x[0, self.column] < self.cutoff else self._ge_tree.classify(x)
if len(rv.shape) == 1:
rv = numpy.array([rv])
return rv
def _make_leaf(self):
self.IS_LEAF = True
def _make_split(self, X, oracle, target, maxdepth, bias):
self.IS_LEAF = False
try:
best = select_best_column(X, oracle, bias=bias)
except NoBestColumn:
self._make_leaf()
return
self.column = int(best[0])
self.cutoff = float(best[1])
# get indexes for >= and <
columndata = _toarray(X[:,self.column].T)[0]
geidx = columndata >= self.cutoff
lidx = geidx == False
lX = X[lidx.nonzero()[0],:] if not numpy.all(lidx == False) else numpy.matrix((0, X.shape[1]))
geX = X[geidx.nonzero()[0],:] if not numpy.all(geidx == False) else numpy.matrix((0, X.shape[1]))
ge_eprobs = numpy.array([oracle(self.column, self.cutoff), float(geX.shape[0]) / X.shape[0]], dtype=numpy.double)
l_eprobs = 1.0 - ge_eprobs
ge_eprobs = ge_eprobs * self.eprobs
l_eprobs = l_eprobs * self.eprobs
# need to rebuild oracles to account for this split
def loracle(col, cut):
if col != self.column:
return oracle(col, cut)
if cut >= self.cutoff:
return 0.0
else:
pless = (1.0 - oracle(col, cut)) / (1.0 - oracle(col, self.cutoff))
return 1.0 - pless
def georacle(col, cut):
if col != self.column:
return oracle(col, cut)
if cut < self.cutoff:
return 0.0
else:
origp = oracle(col, self.cutoff)
newp = oracle(col, cut)
assert origp >= newp
lessorig = 1.0 - origp
lessnew = 1.0 - newp
less = (lessnew - lessorig) / (1.0 - lessorig)
return 1.0 - less
# generate trees
self._ge_tree = self.__class__(geX, georacle, target, maxdepth-1, g=self.g, eprobs=ge_eprobs, bias=bias)
self._less_tree = self.__class__(lX, loracle, target, maxdepth-1, g=self.g, eprobs=l_eprobs, bias=bias)
less_edge = self.__symbolic_column_name__(self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
def __symbolic_column_name__(self, colnum):
x = symbols('x')
return x(colnum)
def __str__(self):
return str(id(self)) + "\n" + str(numpy.vstack([self.probs, self.eprobs]))
def __repr__(self):
return str(self)
class OneClassTree(object):
def __init__(self,
X,
oracle,
target=0.6,
maxdepth=5,
g=None,
eprobs=None,
bias=0.0):
if g == None:
g = DirectedGraph()
if eprobs == None:
eprobs = numpy.array([1.0, 1.0])
self.g = g
self.eprobs = eprobs
self.IS_LEAF = False
self.column = None
self.cutoff = None
self._less_tree = None
self._ge_tree = None
probs = 0.5 * self.eprobs
probs = probs / numpy.sum(probs)
self.probs = probs
met_target = numpy.all(
[p <= 1.0 - target or p >= target for p in probs])
self.g.add_node(self)
if met_target:
self._make_leaf()
elif X.shape[0] < 10:
self._make_leaf()
elif maxdepth <= 1:
self._make_leaf()
else:
self._make_split(X, oracle, target, maxdepth, bias)
def selected_columns(self):
if self.IS_LEAF:
return set([])
else:
left = self._less_tree.selected_columns()
right = self._ge_tree.selected_columns()
middle = set([self.column])
return middle.union(left).union(right)
def __getstate__(self):
return (self.eprobs, self.IS_LEAF, self.column, self.cutoff,
self._less_tree, self._ge_tree)
def __setstate__(self, state):
self.eprobs = state[0]
self.IS_LEAF = state[1]
self.column = state[2]
self.cutoff = state[3]
self._less_tree = state[4]
self._ge_tree = state[5]
self.g = DirectedGraph()
self.g.add_node(self)
self.probs = 0.5 * self.eprobs
self.probs = self.probs / numpy.sum(self.probs)
if self._less_tree != None:
assert self._ge_tree != None
self.g.union(self._less_tree.g)
self.g.union(self._ge_tree.g)
self._less_tree.g = self.g
self._ge_tree.g = self.g
less_edge = self.__symbolic_column_name__(
self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
def classify(self, x):
assert x.shape[0] > 0
if x.shape[0] > 1:
rv = numpy.vstack([self.classify(x[i]) for i in range(x.shape[0])])
elif self.IS_LEAF:
rv = self.eprobs
else:
rv = self._less_tree.classify(x) if x[
0, self.column] < self.cutoff else self._ge_tree.classify(x)
if len(rv.shape) == 1:
rv = numpy.array([rv])
return rv
def _make_leaf(self):
self.IS_LEAF = True
def _make_split(self, X, oracle, target, maxdepth, bias):
self.IS_LEAF = False
try:
best = select_best_column(X, oracle, bias=bias)
except NoBestColumn:
self._make_leaf()
return
self.column = int(best[0])
self.cutoff = float(best[1])
# get indexes for >= and <
columndata = _toarray(X[:, self.column].T)[0]
geidx = columndata >= self.cutoff
lidx = geidx == False
lX = X[lidx.nonzero()[0], :] if not numpy.all(
lidx == False) else numpy.matrix((0, X.shape[1]))
geX = X[geidx.nonzero()[0], :] if not numpy.all(
geidx == False) else numpy.matrix((0, X.shape[1]))
ge_eprobs = numpy.array([
oracle(self.column, self.cutoff),
float(geX.shape[0]) / X.shape[0]
],
dtype=numpy.double)
l_eprobs = 1.0 - ge_eprobs
ge_eprobs = ge_eprobs * self.eprobs
l_eprobs = l_eprobs * self.eprobs
# need to rebuild oracles to account for this split
def loracle(col, cut):
if col != self.column:
return oracle(col, cut)
if cut >= self.cutoff:
return 0.0
else:
pless = (1.0 - oracle(col, cut)) / (1.0 -
oracle(col, self.cutoff))
return 1.0 - pless
def georacle(col, cut):
if col != self.column:
return oracle(col, cut)
if cut < self.cutoff:
return 0.0
else:
origp = oracle(col, self.cutoff)
newp = oracle(col, cut)
assert origp >= newp
lessorig = 1.0 - origp
lessnew = 1.0 - newp
less = (lessnew - lessorig) / (1.0 - lessorig)
return 1.0 - less
# generate trees
self._ge_tree = self.__class__(geX,
georacle,
target,
maxdepth - 1,
g=self.g,
eprobs=ge_eprobs,
bias=bias)
self._less_tree = self.__class__(lX,
loracle,
target,
maxdepth - 1,
g=self.g,
eprobs=l_eprobs,
bias=bias)
less_edge = self.__symbolic_column_name__(self.column) < self.cutoff
ge_edge = self.__symbolic_column_name__(self.column) >= self.cutoff
self.g.connect(self, self._ge_tree, edgeValue=str(ge_edge))
self.g.connect(self, self._less_tree, edgeValue=str(less_edge))
def __symbolic_column_name__(self, colnum):
x = symbols('x')
return x(colnum)
def __str__(self):
return str(id(self)) + "\n" + str(
numpy.vstack([self.probs, self.eprobs]))
def __repr__(self):
return str(self)
def __init__(self):
self.graph = DirectedGraph()