def log_fib(n):
A = [[1, 1], [1, 0]]
init_vector = [[0, 1]]
result = dot(A, init_vector)
for i in range(n - 2):
result = dot(A, result)
return result[0]
def gen_interf(fname):
p = IRParser(open(datadir + fname))
mod = p.parse()
f = mod[0]
l = Liveness(f)
ig = InterferenceGraph(f, l)
dot.dot(ig, open(fname + ".dot", "w"))
return ig
def test_size_mismatch():
"""Make sure that ValueError is raised on size mismatch
Matrix multiplication is only possible if the horizontal dimension
of the first matrix is equal to the vertical dimension of the other
one.
"""
with pytest.raises(ValueError):
dot.dot(np.zeros((3, 2)), np.zeros((3, 2)))
def gendots():
cb = dot('Caustic Bite', 40, 30)
sb = dot('Stormbite', 50, 30)
table = [cb, sb]
dict = {}
for i in table:
dict[i.name] = i
return dict
def pair_up(p,q,r):
"returns the point in segment q-p that is closest to r; accepts p==q"
segment = (q[0] - p[0], q[1] - p[1])
segsq = dot(segment,segment)
if segsq == 0:
"p==q"
return p, 0
vnew = (r[0] - p[0], r[1] - p[1])
alpha = dot(vnew,segment)*1.0 / segsq # we know segsq != 0
if alpha < 0: alpha = 0
if alpha > 1: alpha = 1
return (alpha*q[0] + (1-alpha)*p[0], alpha*q[1] + (1-alpha)*p[1]), alpha
def dot(self):
d = dot.dot("LR0")
for idx in range(len(self.states)):
label = "State %d" % idx
for itnum in self.states[idx]:
label += "\\n" + self.itemstr(itnum)
d.add(" %d [label=\"%s\",shape=box];" % (idx, label))
# make names for transitions to each next state
name = ["" for st in range(len(self.states))]
for (st, tr), ns in self.goto.items():
if st != idx:
continue
if name[ns] != "":
name[ns] += ", "
if isinstance(tr, str):
name[ns] += printable(tr, 1)
else:
name[ns] += self.prodstr(tr, 1)
# print all transitions that have names
for ns in range(len(name)):
if name[ns] != "":
d.add(" %d -> %d [label=\"%s\"];" % (idx, ns, name[ns]))
d.end()
d.show()
def vec_linear_mse(x, y, theta):
y = y.reshape(y.shape[0], 1)
if x.shape[0] != y.shape[0] or x.shape[1] != theta.shape[0] or len(
x) == 0 or len(y) == 0 or len(theta) == 0:
return None
return dot(((mat_vec_prod(x, theta) - y) / len(y)),
(mat_vec_prod(x, theta) - y))[0]
# X = np.array([
# [ -6, -7, -9],
# [ 13, -2, 14],
# [ -7, 14, -1],
# [ -8, -4, 6],
# [ -5, -9, 6],
# [ 1, -5, 11],
# [ 9, -11, 8]])
# Y = np.array([2, 14, -13, 5, 12, 4, -19])
# Z = np.array([3,0.5,-6])
# print("1 -", vec_linear_mse(X, Y, Z))
# # 2641.0
# W = np.array([0,0,0])
# print("2 -", vec_linear_mse(X, Y, W))
# 130.71428571
def gradient(x, y, theta):
if (len(x) == 0 or len(x) != len(y) or len(theta) == 0
or len(theta) != x.shape[1]):
return (None)
res = 0
for i in range(len(x)):
tmp = np.zeros(x.shape[1])
for j in range(x.shape[1]):
tmp[j] = (dot(theta, x[i]) - y[i]) * x[i][j]
res += tmp
return res / len(x)
# X = np.array([
# [ -6, -7, -9],
# [ 13, -2, 14],
# [ -7, 14, -1],
# [ -8, -4, 6],
# [ -5, -9, 6],
# [ 1, -5, 11],
# [ 9, -11, 8]])
# Y = np.array([2, 14, -13, 5, 12, 4, -19])
# Z = np.array([3,0.5,-6])
# print(gradient(X, Y, Z))
# # array([ -37.35714286, 183.14285714, -393. ])
# W = np.array([0,0,0])
# print(gradient(X, Y, W))
# # array([ 0.85714286, 23.28571429, -26.42857143])
# print(gradient(X, X.dot(Z), Z))
# # print(grad(X, X.dot(Z), Z))
# # array([0., 0., 0.])
def dot(self) :
d = dot.dot("LR0")
for idx in range(len(self.states)) :
label = "State %d" % idx
for itnum in self.states[idx] :
label += "\\n" + self.itemstr(itnum)
d.add(" %d [label=\"%s\",shape=box];" % (idx, label))
# make names for transitions to each next state
name = ["" for st in range(len(self.states))]
for (st,tr),ns in self.goto.items() :
if st != idx :
continue
if name[ns] != "" :
name[ns] += ", "
if isinstance(tr, str) :
name[ns] += printable(tr, 1)
else :
name[ns] += self.prodstr(tr, 1)
# print all transitions that have names
for ns in range(len(name)) :
if name[ns] != "" :
d.add(" %d -> %d [label=\"%s\"];" % (idx, ns, name[ns]))
d.end()
d.show()
def mat_mat_product(x, y): if x.shape[1] != y.shape[0] or x.size == 0 or y.size == 0: return (None) l = [] for i in range (x.shape[0]): l.append([dot(x[i, :],y[:, j]) for j in range (y.shape[1])]) return(np.array(l))
def dotstack(heads):
"print out a tomita stack graphically"
d = dot.dot("ParseStacks")
visited = dict()
for h in heads:
dotstack_rec(d, h, visited, heads)
d.end()
d.show()
def rand_hplane(w,h):
w = w/4
h = h/4
pp = (randrange(w,3*w),randrange(h,3*h)) # chosen in the central area
vv = (randrange(-w,w),randrange(-h,h))
v_len = sqrt(dot(vv,vv))
vv = (vv[0]/v_len,vv[1]/v_len)
return pp, vv
def linear_mse(x, y, theta):
if x.shape[0] != y.shape[0] or x.shape[1] != theta.shape[0] or len(
x) == 0 or len(y) == 0 or len(theta) == 0:
return None
tmp = 0.0
for i in range(len(y)):
tmp += (dot(theta, x[i]) - y[i])**2
return tmp / len(y)
def dotstack(heads) :
"print out a tomita stack graphically"
d = dot.dot("ParseStacks")
visited = dict()
for h in heads :
dotstack_rec(d, h, visited, heads)
d.end()
d.show()
def gradient(x, y, theta):
res = 0.0
for i in range(y.shape[0]):
tmp = np.zeros(x.shape[1])
for j in range(x.shape[1]):
tmp[j] = (dot(theta, x[i]) - y[i]) * x[i][j]
res += tmp
return res / len(x)
def vec_mse(y, y_hat):
if y.size == 0 or isinstance(y, (np.ndarray, np.generic)) == False:
return None
if y_hat.size == 0 or isinstance(y_hat, (np.ndarray, np.generic)) == False:
return None
if y.shape != y_hat.shape:
return None
return dot(y_hat - y, y_hat - y) / y.size
def draw_hplane(p,nv,pc,nc):
"pc: color for pos side (single pt), nc: color for neg side (segment)"
for i in range(width):
for j in range(height):
if dot((i-p[0],j-p[1]), nv) >= 0:
"positive halfspace, including hyperplane itself"
myspace[i][j] = pc
else:
myspace[i][j] = nc
def matrix_vector_mult(A,x,y):
# check that all inputs are 2x2 arrays
if len(A.shape) != 2 or len(x.shape) != 2 or len(y.shape) != 2:
return "FAILED"
# check that x and y are both column vectors
if y.shape[1] != 1 or x.shape[1] != 1:
return "FAILED"
# check that inputs have compatible dimensions
if y.shape[0] != x.shape[0] or A.shape[1] != x.shape[0]:
return "FAILED"
# check that A is a square matrix
if A.shape[0] != A.shape[1]:
return "FAILED"
for i in range(A.shape[0]):
y[i,0] += dot(np.array([A[i,i:]]), x[i:]) + dot(np.array([A[:i,i]]), x[:i])
return y
def vec_mse(y, y_hat): if (len(y) == 0 or len(y) != len(y_hat)): return (None) return dot((y - y_hat) / len(y), (y - y_hat)) # X = np.array([0, 15, -9, 7, 12, 3, -21]) # Y = np.array([2, 14, -13, 5, 12, 4, -19]) # print(vec_mse(X, Y)) # print(vec_mse(X, X))
def gradient(x, y, theta): nabla_j = [] for j in range(0, x.shape[1]): res = 0 dot_l = [] for i in range(x.shape[0]): res += dot(theta, x[i]) - y[i] * x[i] nabla_j.append(res[j] / x.shape[0]) print(np.array(nabla_j))
def mat_vec_prod(x, y):
if x.shape[1] != y.shape[0]:
print(y.shape)
if type(x) != np.ndarray or type(y) != np.ndarray or len(x.shape) != 2 or \
x.shape[1] != y.shape[0] or y.shape[1] != 1:
return None
res = []
for row in x:
res.append(dot(row, y))
return np.array(res).reshape(len(res), 1)
def mat_mat_prod(x, y):
if type(x) != np.ndarray or type(y) != np.ndarray:
return None
if x.shape[1] != y.shape[0]:
return None
ret = np.zeros((x.shape[0], y.shape[1]))
for i in range(x.shape[0]):
for j in range(y.shape[1]):
ret[i][j] = dot(x[i], y[:,j])
return ret
def vec_mse(y, y_hat):
if y.shape != y_hat.shape or len(y) == 0:
return None
return dot((y_hat - y) / len(y), (y_hat - y))
# X = np.array([0, 15, -9, 7, 12, 3, -21])
# Y = np.array([2, 14, -13, 5, 12, 4, -19])
# print("1 -", vec_mse(X, Y))
# print("2 -", vec_mse(X, X))
def test_transposed():
"""Make sure that multiplication of transposed matrices
x × y = (y.T × x.T).T
(where .T means that the matrix is transposed, i.e. flipped
around the diagonal.)
Create two sample matrices using numpy.random.random, and
verify that the transposed result of multiplication of the
tranposed matrices gives the same result as the original
multiplication.
"""
x = np.array([[1, 4, 3], [2, -1, 12]])
y = np.array([[32, 12, 8], [0, 5, 8], [-3, -2, 4]])
z1 = dot.dot(x, y)
z2 = dot.dot(y.T, x.T).T
assert np.allclose(z1, z2)
def fit_(self, X, Y, alpha, n_cycle):
M = len(X)
N = len(X[0])
coef = alpha / M
new_theta = np.zeros((N + 1, 1))
for cycle in range(n_cycle):
err_pred = self.predict_(X) - Y
new_theta[0] = self.theta[0] - (np.sum(err_pred) * coef)
for feature in range(N):
tmp_theta = np.sum(dot(err_pred, X.T[feature])) * coef
new_theta[feature + 1] = self.theta[feature + 1] - tmp_theta
self.theta = new_theta
def draw_hplane(p, nv, pc, nc):
"pc: color for pos side (single pt), nc: color for neg side (segment)"
myspace = pygame.surfarray.pixels2d(screen)
screen.lock()
for i in range(width):
for j in range(height):
if dot((i - p[0], j - p[1]), nv) >= 0:
"positive halfspace, including hyperplane itself"
myspace[i][j] = pc
else:
myspace[i][j] = nc
screen.unlock()
def dottree(tree, printcover=0):
"""
print out a tree (rooted with a symnode) graphically
if printcover is true, print the cover of each node.
"""
if tree == None:
return
d = dot.dot("ParseTree")
d.add(" start -> sym_%s;" % hash(tree))
dottree_rec(d, tree, dict(), dict(), printcover)
d.end()
d.show()
def dottree(tree, printcover = 0) :
"""
print out a tree (rooted with a symnode) graphically
if printcover is true, print the cover of each node.
"""
if tree == None :
return
d = dot.dot("ParseTree")
d.add(" start -> sym_%s;" % hash(tree))
dottree_rec(d, tree, dict(), dict(), printcover)
d.end()
d.show()
def mat_vec_prod(x, y):
if x.size == 0 or y.size == 0:
return None
#if x.shape[1] != y.shape[0]:
# print(x.shape, y.shape)
# return None
array = []
#print("here")
#res_n = np.zeros((x.shape[0], 1))
for i in range(0, x.shape[0]):
array.append(dot(x[i], y))
return np.array(array)
def trifib(n):
S = [[1,1,1],[1,0,0],[0,1,0]]
V = [1,0,0]
temp = []
result = []
if n <= 3:
return V[n-1]
temp = m_power(S,n-4)
result = dot(temp,V)
return (result[0],temp)
def __main__():
argp = argparse.ArgumentParser(
description="Parse and dump PseudoC program")
argp.add_argument("file", help="Input file in PseudoC format")
argp.add_argument("--repr",
action="store_true",
help="Dump __repr__ format of instructions")
argp.add_argument("--roundtrip",
action="store_true",
help="Dump PseudoC asm")
argp.add_argument("--addr-width",
type=int,
default=8,
help="Width of address field (%(default)d)")
argp.add_argument("--inst-indent",
type=int,
default=4,
help="Indent of instructions (%(default)d)")
args = argp.parse_args()
p = Parser(args.file)
cfg = p.parse()
cfg.parser = p
if args.roundtrip:
p = AsmPrinter(cfg)
p.addr_width = args.addr_width
p.inst_indent = args.inst_indent
p.print()
sys.exit()
print("Labels:", p.labels)
if args.repr:
core.SimpleExpr.simple_repr = False
print("Basic blocks:")
dump_bblocks(cfg, printer=repr if args.repr else str)
with open(sys.argv[1] + ".0.dot", "w") as f:
dot.dot(cfg, f)
def best_margin(ng_pts,s_n_ids,ps_pts,s_p_ids):
"admittedly some repeated work here - does not work for ns > 3"
m_sq_best = float("inf")
# cannot loop just on s_p_ids as needs the "next" for the segment
for i in range(len(s_n_ids)):
for j in range(len(s_p_ids)):
p1 = ps_pts.coords[s_p_ids[j]]
p2 = ps_pts.coords[s_p_ids[(j+1)%len(s_p_ids)]]
n1 = ng_pts.coords[s_n_ids[i]]
n2 = ng_pts.coords[s_n_ids[(i+1)%len(s_n_ids)]]
c_p, alpha = pair_up(p1,p2,n1)
# v must point from neg to pos
v = (c_p[0] - n1[0], c_p[1] - n1[1])
m_sq = dot(v,v)
if m_sq < m_sq_best:
m_sq_best = m_sq
v_best = v
c_p_best = c_p
ref = (0.5*c_p[0] + 0.5*n1[0], 0.5*c_p[1] + 0.5*n1[1])
s_v_p = [ ]
if alpha > 0:
s_v_p.append(s_p_ids[(j+1)%len(s_p_ids)])
if alpha < 1:
s_v_p.append(s_p_ids[j])
s_v_n = [ s_n_ids[i] ]
c_p, alpha = pair_up(n1,n2,p1)
v = (p1[0] - c_p[0], p1[1] - c_p[1])
m_sq = dot(v,v)
if m_sq < m_sq_best:
m_sq_best = m_sq
v_best = v
c_p_best = c_p
ref = (0.5*c_p[0] + 0.5*p1[0], 0.5*c_p[1] + 0.5*p1[1])
s_v_n = [ ]
if alpha > 0:
s_v_n.append(s_n_ids[(i+1)%len(s_n_ids)])
if alpha < 1:
s_v_n.append(s_n_ids[i])
s_v_p = [ s_p_ids[j] ]
return ref, v_best, c_p_best, s_v_p, s_v_n
def test_f2py():
ok=True
print(' Testing numpy f2py:')
#removing old , if exists
dots=('dot.so','dot.f','dot.f.log')
for ff in dots:
if os.path.isfile(ff):
os.remove(ff)
source=[
' subroutine dot(x,y,z,dx1,dxy,dy2)\n',
'cf2py intent(in) :: x,y\n',
'cf2py intent(out) :: z\n',
'cf2py intent(hide) :: dx1,dxy,dy2\n',
'cf2py double :: x(dx1,dxy),y(dxy,dy2),z(dx1,dy2)\n',
' implicit none\n',
' integer dx1,dxy,dy2\n',
' integer ix1,ixy,iy2\n',
' double precision x(dx1,dxy),y(dxy,dy2),z(dx1,dy2)\n',
' do 100 ix1=1,dx1\n',
' do 200 iy2=1,dy2\n',
' z(ix1,iy2)=0.d0\n',
' do 300 ixy=1,dxy\n',
' z(ix1,iy2)=z(ix1,iy2)+ x(ix1,ixy)*y(ixy,iy2)\n',
'300 enddo \n',
'200 enddo\n',
'100 enddo\n',
' end\n']
with open('dot.f','w') as f:
f.writelines(source)
aa=os.system('f2py -m dot -c dot.f &>dot.f.log')
if aa != 0:
ok= False
redprint(' Error compiling test function. Check dot.f.log')
else:
if not os.path.isfile('dot.so'):
redprint(' Module dot.so does not exist. Is the gfortran compiler installed?')
ok=False
try:
sys.path.append(os.getcwd())
import numpy
import dot
a=numpy.arange(12.).reshape(1,12)
b=numpy.arange(12.).reshape(12,1)
if a.dot(b)[0,0] != dot.dot(a,b)[0,0]:
redprint(' Module dot does not work as expected. Hm... ask Rene?')
ok=False
except Exception, e:
print('')
print(' '+e)
redprint(' Module dot could not imported. Is the gfortran compiler installed?')
ok=False
def test_f2py():
ok = True
print(' Testing numpy f2py:')
#removing old , if exists
dots = ('dot.so', 'dot.f', 'dot.f.log')
for ff in dots:
if os.path.isfile(ff):
os.remove(ff)
source = [
' subroutine dot(x,y,z,dx1,dxy,dy2)\n',
'cf2py intent(in) :: x,y\n', 'cf2py intent(out) :: z\n',
'cf2py intent(hide) :: dx1,dxy,dy2\n',
'cf2py double :: x(dx1,dxy),y(dxy,dy2),z(dx1,dy2)\n',
' implicit none\n', ' integer dx1,dxy,dy2\n',
' integer ix1,ixy,iy2\n',
' double precision x(dx1,dxy),y(dxy,dy2),z(dx1,dy2)\n',
' do 100 ix1=1,dx1\n', ' do 200 iy2=1,dy2\n',
' z(ix1,iy2)=0.d0\n', ' do 300 ixy=1,dxy\n',
' z(ix1,iy2)=z(ix1,iy2)+ x(ix1,ixy)*y(ixy,iy2)\n',
'300 enddo \n', '200 enddo\n', '100 enddo\n', ' end\n'
]
with open('dot.f', 'w') as f:
f.writelines(source)
aa = os.system('f2py -m dot -c dot.f &>dot.f.log')
if aa != 0:
ok = False
redprint(' Error compiling test function. Check dot.f.log')
else:
if not os.path.isfile('dot.so'):
redprint(
' Module dot.so does not exist. Is the gfortran compiler installed?'
)
ok = False
try:
sys.path.append(os.getcwd())
import numpy
import dot
a = numpy.arange(12.).reshape(1, 12)
b = numpy.arange(12.).reshape(12, 1)
if a.dot(b)[0, 0] != dot.dot(a, b)[0, 0]:
redprint(
' Module dot does not work as expected. Hm... ask Rene?'
)
ok = False
except Exception, e:
print('')
print(' ' + e)
redprint(
' Module dot could not imported. Is the gfortran compiler installed?'
)
ok = False
def fit_(theta, X, Y, alpha, n_cycle):
M = len(X)
N = len(X[0])
new_theta = np.zeros((N + 1, 1))
coef = alpha / M
for cycle in range(n_cycle):
err_pred = predict_(theta, X) - Y
new_theta[0] = theta[0] - (sum(err_pred) * coef)
for feat in range(N):
tmp_theta = np.sum(dot(err_pred, X.T[feat])) * coef
new_theta[feat + 1] = theta[feat + 1] - tmp_theta
theta = new_theta
return new_theta
def dot(self, mach) :
d = dot.dot("nfa")
d.add(" rankdir = LR;");
for idx in range(mach.min, mach.max + 1) :
trset, tr1, tr2, acc = self.states[idx]
name = "%d" % idx
if acc != NOACCEPT :
name = "accept %d" % acc
extra = ""
if idx == mach.start :
extra = ",color=red"
d.add(" n%d [label=\"%s\"%s];" % (idx, name, extra))
name = self.chsetname(trset, 1)
if tr1 != NOSTATE :
d.add(" n%d -> n%d [label=\"%s\"];" % (idx, tr1, name))
if tr2 != NOSTATE :
d.add(" n%d -> n%d [label=\"%s\"];" % (idx, tr2, name))
d.end()
d.show()
import parser
import dot
adj = parser.parseAdj('input')
start = 1
l = len(adj)
v = set()
v.add(start)
inf = float('inf')
parents = [-1]*l
while len(v) < l:
next_min = inf
for i in v:
E = adj[i]
for j in range(l):
if E[j] < next_min and E[j] != 0 and j not in v:
next_min = E[j]
nextV = j
parent = i
parents[nextV] = parent
v.add(nextV)
edges = []
weights = []
for i in range(l):
if parents[i] != -1:
edges.append((i,parents[i]))
weights.append(adj[i][parents[i]])
dot.dot(edges,weights)
def findSimilarTrees(
tmDataList,
columnListNames,
sizeColName,
sizeMin=-1.0,
sizeMax=10000000000,
outputEachPair=False,
justKeepBest=False,
):
"""does munkreskuhn matching over pocket-pocket shapes to get a score
per tree, returns table of these"""
columnList = tmDataList[0].titlesToColumns(columnListNames)
sizeCol = tmDataList[0].titleToColumn(sizeColName)
colToMean, colToStddev = calcColumnsMeanStddev(columnList, tmDataList)
totalMatrix = {}
for tmDataCount1, tmData1 in enumerate(tmDataList):
totalMatrix[tmData1] = {}
for tmDataCount2, tmData2 in enumerate(tmDataList):
if tmDataCount2 > tmDataCount1:
dotData = dot.dot([tmData1, tmData2])
rowNames, colNames, matchMatrix, tooBig = dotData.computeSearchConnections(
1e10000000,
columnList,
colToMean,
colToStddev,
False,
sizeCol,
False,
False,
sizeMin=sizeMin,
sizeMax=sizeMax,
doSelfScore=False,
returnMatrix=True,
)
if not justKeepBest:
matches = munkreskuhn.assignAndReturnMatches(matchMatrix)
sumScore = 0
for match in matches:
sumScore += match[2]
totalMatrix[tmData1][tmData2] = sumScore / float(len(matches))
if outputEachPair: # output a gdl for each pair of the munkres match
newMatches = []
justNodes = tooBig
for match in matches:
node1 = rowNames[match[0]]
node2 = colNames[match[1]]
justNodes.append(node1)
justNodes.append(node2)
newMatches.append([tmData1, tmData2, node1, node2, match[2]])
dotData.matchList = newMatches
dotData.addSearchConnections(1e1000000, remove=True)
dotData.writeGdl(
tmData1.inputFileName + "_" + tmData2.inputFileName + ".gdl",
justNodes=justNodes,
edges=True,
force=True,
)
else: # just find the best match
minMatchMatrix = 1e10000000
for row in matchMatrix:
for entry in row:
if entry < minMatchMatrix:
minMatchMatrix = entry
totalMatrix[tmData1][tmData2] = minMatchMatrix
return totalMatrix
for name, cfg in cfg_layer.items():
save_cfg(cfg, suffix)
progdb.load_funcdb(sys.argv[1] + "/funcdb.yaml")
# Load binary data
import bindata
bindata.init(sys.argv[1])
# Load symtab
if os.path.exists(sys.argv[1] + "/symtab.txt"):
progdb.load_symtab(sys.argv[1] + "/symtab.txt")
callgraph = xform_inter.build_callgraph()
with open("cg-current.dot", "w") as out:
dot.dot(callgraph, out, is_cfg=False)
#for func in callgraph.iter_rev_postorder():
# print(func, callgraph[func])
CFG_MAP = collections.defaultdict(dict)
import script_i_prepare
for full_name in glob.glob(sys.argv[1] + "/*.lst"):
p = Parser(full_name)
cfg = p.parse()
cfg.parser = p
#print("Loading:", cfg.props["name"])
CFG_MAP["org"][cfg.props["name"]] = cfg
def create_event(self, pname, name, id, value):
"""create an event
- **parameters**, **types**, **return** and **return types**::
:param arg1: parent name
:param arg2: function name
:param arg3: Event ID
:param arg4: value for event ID
"""
if id == "Start":
self.stack.append(name)
if id == "End":
self.stack.pop()
try:
nametest = self.stack[-1]
except:
nametest = "main"
tmp = (
"EVENT "
+ strftime("%Y-%m-%d %H:%M:%S", gmtime())
+ " "
+ str(id)
+ " "
+ str(pname)
+ " "
+ str(name)
+ " "
+ str(value)
+ "\n"
)
self.fd.write(tmp)
self.event_list.append(tmp)
if id == "BoardnameEnd":
self.event_flush()
if self.tb.config.create_webpatch == "yes":
self.webpatch = web_patchwork(self.tb, "webpatch.html")
if self.tb.config.create_dot == "yes":
self.ignoretclist = [
"tc_workfd_check_cmd_success.py",
"tc_lab_cp_file.py",
"tc_workfd_check_if_file_exist.py",
"tc_workfd_rm_file.py",
]
self.dot = dot(self.tb, "tc.dot", self.ignoretclist)
if self.tb.config.create_statistic == "yes":
self.statistic = statistic_plot_backend(self.tb, "stat.dat", self.ignoretclist)
if self.tb.config.create_html_log == "yes":
self.html_log = html_log(self.tb, "log/html_log.html")
if self.tb.config.create_documentation == "yes":
self.ignoretclist = ["tc_workfd_check_cmd_success.py"]
self.doc = doc_backend(self.tb, self.ignoretclist)
if self.tb.config.create_dashboard == "yes":
from dashboard import dashboard
self.dashboard = dashboard(self.tb, "localhost", "tbot", "tbot", "tbot_root", "tbot_results")
# execute the event backends
if self.tb.config.create_webpatch == "yes":
self.webpatch.create_webfile()
if self.tb.config.create_dot == "yes":
self.dot.create_dotfile()
if self.tb.config.create_statistic == "yes":
self.statistic.create_statfile()
if self.tb.config.create_html_log == "yes":
self.html_log.create_htmlfile()
if self.tb.config.create_dashboard == "yes":
self.dashboard.insert_test_into_db()
if self.tb.config.create_documentation == "yes":
self.doc.create_docfiles()
def findSimilarNodes(
tmDataList,
columnListNames,
maxThr,
maxConnectionCount,
skipConnCount,
resCols,
sizeColName,
sameStruct=False,
sizeMin=-1.0,
sizeMax=10000000000,
mst=False,
selfColListNames=None,
doSelfScore=True,
justNodes=None,
lineMst=False,
lineMstEnds=False,
refinePockets=False,
possNodes=None,
dotResidues=None,
printThings=True,
calcColMeansStd=True,
alpha=1.0,
clusterOutput=False,
):
"""does all against all comparison of nodes looking for similar ones
does lots of other stuff (need to update this documentation)"""
justNodesValues = None
if justNodes is not None:
justNodesValues = justNodes.values()
if selfColListNames is None:
selfColListNames = columnListNames
columnList = tmDataList[0].titlesToColumns(columnListNames)
selfColList = tmDataList[0].titlesToColumns(selfColListNames)
resColNums = tmDataList[0].titlesToColumns(resCols)
sizeCol = tmDataList[0].titleToColumn(sizeColName)
if calcColMeansStd:
colToMean, colToStddev = calcColumnsMeanStddev(columnList, tmDataList)
else:
colToMean, colToStddev = getStandardColumnsMeanStddev()
selfScores = {} # make combined list
if doSelfScore:
for tmData in tmDataList:
selfSc = findSelfScore(tmData, selfColList, resColNums, colToMean, colToStddev)
selfScores.update(selfSc)
# print selfScores.values()
if len(tmDataList) == 1: # can't compare if only 1 struct
# want to output nodes + self scores in sorted order for debugging of
# self scoring system
treeName = tmDataList[0].inputFileName # only one
selfScoreList = list(selfScores.iteritems())
selfScoreList.sort(key=operator.itemgetter(1)) # sort by score
if printThings:
for node, selfScore in selfScoreList:
if node.attributes[sizeCol] > sizeMin and node.attributes[sizeCol] < sizeMax:
print dot.outputDrawStr(treeName, node) + "; " + str(selfScore)
return None, none, none # no matches to return
else: # more than 1 tree, do comparisons
dotData = dot.dot(tmDataList)
matchList = dotData.computeSearchConnections(
maxThr,
columnList,
colToMean,
colToStddev,
resColNums,
sizeCol,
selfScores,
sameStruct,
sizeMin=sizeMin,
sizeMax=sizeMax,
doSelfScore=doSelfScore,
justNodes=justNodesValues,
alpha=alpha,
)
if refinePockets:
# iteration done here
notDone = True
eliminatedNodes = []
while notDone:
changeNode, examinedNode = dotData.refinePockets(
columnList,
colToMean,
colToStddev,
resColNums,
sizeCol,
selfScores,
sizeMin=sizeMin,
sizeMax=sizeMax,
doSelfScore=doSelfScore,
justNodes=justNodes,
matchList=matchList,
possNodes=possNodes,
notTheseNodes=eliminatedNodes,
)
if changeNode: # only recompute if there was a change
if justNodes is not None:
justNodesValues = justNodes.values()
matchList = dotData.computeSearchConnections(
maxThr,
columnList,
colToMean,
colToStddev,
resColNums,
sizeCol,
selfScores,
sameStruct,
sizeMin=sizeMin,
sizeMax=sizeMax,
doSelfScore=doSelfScore,
justNodes=justNodesValues,
alpha=alpha,
)
eliminatedNodes = [] # reset since there was a change
else: # worst wasn't changed this round
eliminatedNodes.append(examinedNode)
if examinedNode is None:
notDone = False
if dotResidues is not None:
dotResidues.setBestNodes(justNodes)
connections = 0
while connections <= maxConnectionCount:
dotData.addSearchConnections(maxThr, maxConnCount=connections, mst=mst)
# clusters = dotData.getClustersConnections()
# print len(clusters)
if not mst:
if printThings:
dotData.tempRemoveKeepersWriteGdl(
"search." + string.zfill(connections, 6) + ".gdl", justNodes=justNodesValues
)
elif mst: # change filenames, don't write edges
if printThings:
dotData.tempRemoveKeepersWriteGdl(
"mst." + string.zfill(connections, 4) + ".gdl", edges=False, justNodes=justNodesValues
)
if connections + skipConnCount > maxConnectionCount: # last time only
if clusterOutput: # do but record the num of clusters as they are added
dotData.addSearchConnections(
maxThr, maxConnCount=connections * 25, mst=mst, clusterOutput=clusterOutput
)
if lineMst: # do all again
dotData.addSearchConnections(maxThr, maxConnCount=connections, lineMst=True)
if printThings:
dotData.tempRemoveKeepersWriteGdl(
"linemst." + string.zfill(connections, 4) + ".gdl", edges=False, justNodes=justNodesValues
)
mslList = getLineMst(dotData.connections)
mslScore = statistics.listOrderCorrectness(mslList)
if dotResidues is not None:
dotResidues.mslScore = mslScore
if printThings:
print "linemst",
for item in mslList:
print item,
print " "
print "linemstscore, ", mslScore
if lineMstEnds: # do one more time
# want to do same as linemst but give hints as to the endpoints
startNode = justNodes[tmDataList[0]]
endNode = justNodes[tmDataList[-1]]
dotData.addSearchConnections(
maxThr, maxConnCount=connections, lineMst=True, startNode=startNode, endNode=endNode
)
if printThings:
dotData.tempRemoveKeepersWriteGdl(
"linemstends." + string.zfill(connections, 4) + ".gdl",
edges=False,
justNodes=justNodesValues,
)
mslList = getLineMst(dotData.connections)
if printThings:
print "linemstends",
for item in mslList:
print item,
print " "
connections += skipConnCount
matchList.sort(lambda xVal, yVal: cmp(xVal[4], yVal[4]))
# sort by score. best first
if printThings:
for match in matchList:
print "%5.2f ;" % match[4],
print dot.outputDrawStr(match[0].inputFileName, match[2]) + ";",
print dot.outputDrawStr(match[1].inputFileName, match[3]) + ";"
# print " "
return matchList
#!/usr/bin/env python3
import sys
from parser import *
import dot
p = Parser(sys.argv[1])
cfg = p.parse()
print("Labels:", p.labels)
print("Basic blocks:")
dump_bblocks(cfg)
with open(sys.argv[1] + ".0.dot", "w") as f:
dot.dot(cfg, f)
sys.exit('Unable to locate `fdp`')
except OSError as e:
sys.exit('Execution failed: {0}'.format(e))
if not os.path.isdir('_data'):
os.mkdir('_data')
data = parse.parse(*args.packages)
if args.exclude_externals:
for k, v_list in data.items():
new_v_list = []
for v in v_list:
if v in data.keys():
new_v_list.append(v)
data[k] = new_v_list
dot_str = dot.dot(data)
f_name = datetime.now().strftime('%Y-%m-%d_%H%S')
with open('_data/{0}.dot'.format(f_name), 'w', encoding='utf-8') as dot_file:
dot_file.write(dot_str)
cmd = 'fdp -Tsvg _data/{fn}.dot > _data/{fn}.svg'
try:
retcode = call(cmd.format(fn=f_name), shell=True)
if retcode != 0:
sys.exit('fdp fails')
except OSError as e:
sys.exit('Execution failed: {0}'.format(e))
from asmprinter import AsmPrinter
argp = argparse.ArgumentParser(description="Parse and dump PseudoC program")
argp.add_argument("file", help="Input file in PseudoC format")
argp.add_argument("--repr", action="store_true", help="Dump __repr__ format of instructions")
argp.add_argument("--roundtrip", action="store_true", help="Dump PseudoC asm")
argp.add_argument("--addr-width", type=int, default=8, help="Width of address field (%(default)d)")
argp.add_argument("--inst-indent", type=int, default=4, help="Indent of instructions (%(default)d)")
args = argp.parse_args()
p = Parser(args.file)
cfg = p.parse()
cfg.parser = p
if args.roundtrip:
p = AsmPrinter(cfg)
p.addr_width = args.addr_width
p.inst_indent = args.inst_indent
p.print()
sys.exit()
print("Labels:", p.labels)
if args.repr:
core.SimpleExpr.simple_repr = False
print("Basic blocks:")
dump_bblocks(cfg, printer=repr if args.repr else str)
with open(sys.argv[1] + ".0.dot", "w") as f: dot.dot(cfg, f)
def handle_file_unprotected(args):
p = Parser(args.file)
cfg = p.parse()
cfg.parser = p
# If we want to get asm back, i.e. stay close to the input, don't remove
# trailing jumps. This will work OK for data flow algos, but will produce
# broken or confusing output for control flow algos (for which asm output
# shouldn't be used of course).
# Update: it's unsafe to use this during dataflow analysis
#if args.format != "asm":
# foreach_bblock(cfg, remove_trailing_jumps)
if args.debug:
with open(args.file + ".0.bb", "w") as f:
dump_bblocks(cfg, f, no_graph_header=args.no_graph_header)
with open(args.file + ".0.dot", "w") as f:
dot.dot(cfg, f)
if args.script:
for s in args.script:
mod = __import__(s)
mod.apply(cfg)
elif hasattr(p, "script"):
for op_type, op_name in p.script:
if op_type == "xform:":
func = globals()[op_name]
func(cfg)
elif op_type == "xform_bblock:":
func = globals()[op_name]
foreach_bblock(cfg, func)
elif op_type == "xform_inst:":
func = globals()[op_name]
foreach_inst(cfg, func)
elif op_type == "script:":
mod = __import__(op_name)
mod.apply(cfg)
else:
assert 0
if args.debug:
with open(args.file + ".out.bb", "w") as f:
dump_bblocks(cfg, f, no_graph_header=args.no_graph_header)
with open(args.file + ".out.dot", "w") as f:
dot.dot(cfg, f)
if args.output and args.format != "none":
out = open(args.output, "w")
else:
out = sys.stdout
if args.no_comments:
Inst.show_comments = False
if args.format == "bblocks":
p = CFGPrinter(cfg, out)
if args.no_graph_header:
p.print_graph_header = lambda: None
p.inst_printer = repr if args.repr else str
p.no_dead = args.no_dead
p.print()
elif args.format == "asm":
p = AsmPrinter(cfg, out)
p.no_dead = args.no_dead
p.print()
elif args.format == "c":
#foreach_bblock(cfg, remove_trailing_jumps)
cfg.number_postorder()
Inst.trail = ";"
cprinter.no_dead = args.no_dead
cprinter.dump_c(cfg, out)
if out is not sys.stdout:
out.close()
progdb.update_funcdb(cfg)
return cfg
def dot(self, shownfastates = 0, showccls = 0) :
if showccls :
# make names for the character classes so we dont repeat this often
cclname = ["" for idx in range(len(self.uccls))]
ch = 0
while ch < self.maxchar + 1 :
minch = ch
curccl = self.chr2uccl[chr(ch)]
while ch < self.maxchar + 1 and self.chr2uccl[chr(ch)] == curccl :
ch += 1
maxch = ch - 1
if cclname[curccl] != "" :
cclname[curccl] += ", "
cclname[curccl] += printable(chr(minch), 1)
if minch < maxch :
cclname[curccl] += "-" + printable(chr(maxch), 1)
d = dot.dot("dfa")
for idx in range(1, len(self.rows)) :
xtra = ""
name = "%d" % idx
if shownfastates :
name += ": %s" % self.dfastate[idx][0]
if len(self.acc[idx]) > 0 :
name += " acc%s" % self.acc[idx]
xtra += ", color=red"
cnt = 0
for s1,s2 in self.starts :
if idx == s1 or idx == s2 :
xtra += ", color=red"
if idx == s1 :
name += "\\nstart %d" % cnt
if idx == s2 :
name += "\\nstart ^%d" % cnt
cnt += 1
d.add(" n%d [label=\"%s\"%s];" % (idx, name, xtra))
# make one arc to each reachable next state
# this graph most accurately portrays the table we build, but
# is sometimes hard to read. perhaps we should have an option
# for collecting the character classes for the edge into a single
# character class and then converting that to a string.
for ns in range(1, len(self.rows)) :
ccls = []
for ccl in range(len(self.rows[idx])) :
if self.rows[idx][ccl] == ns :
ccls.append(ccl)
if ccls == [] :
continue
name = ""
if showccls : # build up name out of each ccl
for ccl in ccls :
if name != "" :
name += "\\n"
name += "%d: %s" % (ccl, cclname[ccl])
else : # build up name out of the combined ccl
ch = 0
while ch < self.maxchar + 1 :
minch = ch
while ch < self.maxchar + 1 and self.chr2uccl[chr(ch)] in ccls :
ch += 1
maxch = ch - 1
if maxch < minch :
ch += 1
continue
if name != "" :
name += ", "
name += printable(chr(minch), 1)
if maxch > minch :
name += "-" + printable(chr(maxch), 1)
d.add(" n%d -> n%d [label=\"%s\"];" % (idx, ns, name))
d.show()
