def negSamplingCostAndGradient(y1, target, w2, K=10):
""" Negative sampling cost function for word2vec models """
###################################################################
# Implement the cost and gradients for one predicted word vector #
# and one target word vector as a building block for word2vec #
# models, using the negative sampling technique. K is the sample #
# size. You might want to use dataset.sampleTokenIdx() to sample #
# a random word index. #
# Input/Output Specifications: same as softmaxCostAndGradient #
# We will not provide starter code for this function, but feel #
# free to reference the code you previously wrote for this #
# assignment! #
###################################################################
indexes = list(set([dataset.sampleTokenIdx() for i in range(K)]))
if(target in indexes):
indexes.remove(target)
K1 = len(indexes)
w2sampled = np.zeros((K1+1, np.shape(w2)[1]))
w2sampled[K1] = w2[target]
for i in range(K1):
w2sampled[i] = w2[indexes[i]]
out = np.dot(w2sampled,y1)
cost = -math.log(sigmoid(out[K1]))-sum(np.log(sigmoid(-out)))+np.log(sigmoid(-out[K1]))
d2Sampled = sigmoid(out)
d2Sampled[K1] -= 1.0
grad2 = np.dot(np.row_stack(d2Sampled),np.column_stack(y1))
d1 = np.dot(np.transpose(w2sampled), np.row_stack(d2Sampled))
grad2A = np.zeros((np.shape(w2)[0], np.shape(w2)[1]))
for i in range(K1):
grad2A[indexes[i]] += grad2[i]
grad2A[target] = grad2[K1]
return cost, d1, grad2A
def plot_attention_loss(train_plot, test_plot):
testing_freq = (test_plot["iter"][1] - test_plot["iter"][0]) / (train_plot["iter"][1] - train_plot["iter"][0])
loss_y = train_plot["iter"]
pl.figure(1)
test_loss = test_plot["dir_loss"]
train_loss = train_plot["dir_loss"]
test_loss = numpy.row_stack(test_loss)
test_loss = numpy.tile(test_loss, (1, testing_freq))
test_loss = list(test_loss.flatten())
test_loss += [test_loss[-1]] * max(0, len(train_loss) - len(test_loss))
test_loss = test_loss[: len(train_loss)]
pl.plot(loss_y, train_loss, "k-", label="Train", linewidth=0.75, marker="x")
pl.plot(loss_y, test_loss, "r-", label="Test", linewidth=0.75)
pl.legend(loc="best")
pl.xlabel("Iter")
pl.title("Direction Loss")
pl.figure(2)
test_loss = test_plot["cls_loss"]
train_loss = train_plot["cls_loss"]
test_loss = numpy.row_stack(test_loss)
test_loss = numpy.tile(test_loss, (1, testing_freq))
test_loss = list(test_loss.flatten())
test_loss += [test_loss[-1]] * max(0, len(train_loss) - len(test_loss))
test_loss = test_loss[: len(train_loss)]
pl.plot(loss_y, train_loss, "k-", label="Train", linewidth=0.75, marker="x")
pl.plot(loss_y, test_loss, "r-", label="Test", linewidth=0.75)
pl.legend(loc="best")
pl.xlabel("Iter")
pl.title("Classification Loss")
pl.show()
def mirrorpad(img, width):
"""Return the image resulting from padding width amount of pixels on each
sides of the image img. The padded values are mirror image with respect to
the borders of img. Width can be an integer or a tuple of the form (north,
south, east, west).
"""
n, s, e, w = _get4widths(width)
row = N.shape(img)[0]
col = N.shape(img)[1]
if n != 0:
north = img[:n,:]
img = N.row_stack((north[::-1,:], img))
if s != 0:
south = img[-s:,:]
img = N.row_stack((img, south[::-1,:]))
if e != 0:
east = img[:,-e:]
img = N.column_stack((img, east[:,::-1]))
if w != 0:
west = img[:,:w]
img = N.column_stack((west[:,::-1], img))
return img
def solveMartix(self):
MNA1 = np.column_stack((self.C, self.D))
MNA2 = np.column_stack((self.G, self.B))
MNA = np.row_stack((MNA2, MNA1))
RHS = np.row_stack((self.U, self.I))
x = np.linalg.lstsq(MNA, RHS)
self.x = x[0]
def plot_attention_loss(train_plot, test_plot) :
testing_freq = (test_plot['iter'][1]-test_plot['iter'][0]) / (train_plot['iter'][1]-train_plot['iter'][0])
loss_y = train_plot['iter']
pl.figure(1)
test_loss = test_plot['dir_loss']
train_loss = train_plot['dir_loss']
test_loss = numpy.row_stack(test_loss)
test_loss = numpy.tile(test_loss, (1, testing_freq))
test_loss = list(test_loss.flatten())
test_loss += [test_loss[-1]] * max(0,len(train_loss) - len(test_loss))
test_loss = test_loss[:len(train_loss)]
pl.plot(loss_y, train_loss, 'k-', label='Train', linewidth=0.75, marker='x')
pl.plot(loss_y, test_loss, 'r-', label='Test' , linewidth=0.75)
pl.legend(loc='best')
pl.xlabel('Iter')
pl.title('Direction Loss')
pl.figure(2)
test_loss = test_plot['cls_loss']
train_loss = train_plot['cls_loss']
test_loss = numpy.row_stack(test_loss)
test_loss = numpy.tile(test_loss, (1, testing_freq))
test_loss = list(test_loss.flatten())
test_loss += [test_loss[-1]] * max(0,len(train_loss) - len(test_loss))
test_loss = test_loss[:len(train_loss)]
pl.plot(loss_y, train_loss, 'k-', label='Train', linewidth=0.75, marker='x')
pl.plot(loss_y, test_loss, 'r-', label='Test' , linewidth=0.75)
pl.legend(loc='best')
pl.xlabel('Iter')
pl.title('Classification Loss')
pl.show()
def __call__(self, event):
if event.button == 1:
print 'You press button 1 to zomm, to mask use right button'
if event.button >=2:
if event.inaxes!=self.graph.axes: return
# self.bt.append(event.button)
self.xs.append(event.xdata)
if len(self.xs)%2 == 0: # only odd numbers
if event.button == 3:
self.wei.append(0)
print 'You press button 3 to mask, maksed regions is', self.xs[-2],self.xs[-1],self.wei[-1]
self.tmp=np.column_stack((self.xs[-2],self.xs[-1],self.wei[-1]))
if (self.masked[0][0])==0:
self.masked=np.row_stack((self.tmp))
else:
self.masked=np.row_stack((self.masked,self.tmp))
plot(self.xso[(self.xso>=self.xs[-2]) & (self.xso<=self.xs[-1])],self.yso[(self.xso>=self.xs[-2]) & (self.xso<=self.xs[-1])],color='red',lw='2')
self.graph.figure.canvas.draw()
if event.button == 2:
self.wei.append(2)
print 'You press button 2 attributed weight 2 for this region', self.xs[-2],self.xs[-1],self.wei[-1]
self.tmp=np.column_stack((self.xs[-2],self.xs[-1],self.wei[-1]))
if (self.masked[0][0])==0:
self.masked=np.row_stack((self.tmp))
else:
self.masked=np.row_stack((self.masked,self.tmp))
plot(self.xso[(self.xso>=self.xs[-2]) & (self.xso<=self.xs[-1])],self.yso[(self.xso>=self.xs[-2]) & (self.xso<=self.xs[-1])],color='green',lw='2')
self.graph.figure.canvas.draw()
def getstats_base(X, linds):
sval = {}
for l_ind in linds:
print(l_ind, X['model_state']['layers'][l_ind]['name'])
layer = X['model_state']['layers'][l_ind]
w = layer['weights'][0]
karray = stats.kurtosis(w)
kall = stats.kurtosis(w.ravel())
cf0 = np.corrcoef(w)
cf0t = np.corrcoef(w.T)
wmean = w.mean(1)
w2mean = (w**2).mean(1)
lname = X['model_state']['layers'][l_ind]['name']
sval[lname] = {'karray': karray, 'kall': kall, 'corr0': cf0, 'corr0_t': cf0t,
'wmean': wmean, 'w2mean': w2mean}
if 'filterSize' in X['model_state']['layers'][l_ind]:
fs = X['model_state']['layers'][l_ind]['filterSize'][0]
ws = w.shape
w = w.reshape((ws[0] / (fs**2), fs, fs, ws[1]))
mat = np.row_stack([np.row_stack([w[i, j, :, :] for i in range(w.shape[0])]).T for j in range(w.shape[1])] )
cf = np.corrcoef(mat.T)
cft = np.corrcoef(mat)
mat2 = np.row_stack([np.row_stack([w[i, :, :, j] for i in range(w.shape[0])]).T for j in range(w.shape[3])] )
cf2 = np.corrcoef(mat2.T)
cf2t = np.corrcoef(mat2)
sval[lname].update({'corr': cf, 'corr2': cf2, 'corr_t': cft, 'corr2_t': cf2t})
return sval
def train(self, training_data_array):
data = training_data_array[0] #dict
# Step 2: Forward propagation
a1 = np.mat(data['y0']).T
# 400*1 matrix
z2 = np.dot(self.theta1, np.row_stack((1, a1)))
# num_hidden_nodes*1 matrix
a2 = self.sigmoid(z2)
z3 = np.dot(self.theta2, np.row_stack((1, a2)))
# 10*1 matrix
a3 = self.sigmoid(z3)
# Step 3: Back propagation
y = [0] * 10 # y is a python list for easy initialization and is later turned into an np matrix (2 lines down).
y[data['label']] = 1
# 1*10 list
delta3 = a3 - np.mat(y).T
# 10*1 matrix
z2plus = np.row_stack((0, z2))
# (num_hidden_nodes+1)*1 matrix
delta2 = np.multiply(np.dot(self.theta2.T, delta3), self.sigmoid_prime(z2plus))
# (num_hidden_nodes+1)*1 matrix
delta2 = delta2[1:,0]
# num_hidden_nodes*1 matrix
# Step 4: Update weights
self.theta1 -= self.LEARNING_RATE * np.dot(delta2, np.row_stack((1, a1)).T)
self.theta2 -= self.LEARNING_RATE * np.dot(delta3, np.row_stack((1, a2)).T)
def conf2yap(conf_fname, yap_filename):
print("Yap file : ", yap_filename)
positions, radii, meta = clff.read_conf_file(conf_fname)
positions[:, 0] -= float(meta['lx'])/2
positions[:, 1] -= float(meta['ly'])/2
positions[:, 2] -= float(meta['lz'])/2
if 'np_fixed' in meta:
# for conf with fixed particles
split_line = len(positions) - int(meta['np_fixed'])
pos_mobile, pos_fixed = np.split(positions, [split_line])
rad_mobile, rad_fixed = np.split(radii, [split_line])
yap_out = pyp.layer_switch(3)
yap_out = pyp.add_color_switch(yap_out, 3)
yap_out = np.row_stack((yap_out,
particles_yaparray(pos_mobile, rad_mobile)))
yap_out = pyp.add_layer_switch(yap_out, 4)
yap_out = pyp.add_color_switch(yap_out, 4)
yap_out = np.row_stack((yap_out,
particles_yaparray(pos_fixed, rad_fixed)))
else:
yap_out = pyp.layer_switch(3)
yap_out = pyp.add_color_switch(yap_out, 3)
yap_out = np.row_stack((yap_out,
particles_yaparray(positions, radii)))
pyp.savetxt(yap_filename, yap_out)
def find_goto_point_surface_1(grid,pt,display_list=None):
''' returns p_erratic,p_edge,surface_height
p_erratic - point where the erratic's origin should go to. (in thok0 frame)
p_edge - point on the edge closest to pt.
p_erratic,p_edge are 3x1 matrices.
surface_height - in thok0 coord frame.
'''
pt_thok = pt
# everything is happening in the thok coord frame.
close_pt,approach_vector = find_approach_direction(grid,pt_thok,display_list)
if close_pt == None:
return None,None,None
# move perpendicular to approach direction.
# lam = -(close_pt[0:2,0].T*approach_vector)[0,0]
# lam = min(lam,-0.3) # atleast 0.3m from the edge.
lam = -0.4
goto_pt = close_pt[0:2,0] + lam*approach_vector # this is where I want the utm
# to be
err_to_thok = tr.erraticTglobal(tr.globalTthok0(np.matrix([0,0,0]).T))
goto_pt_erratic = -err_to_thok[0,0]*approach_vector + goto_pt # this is NOT
# general. It uses info about the two frames. If frames move, bad
# things can happen.
if display_list != None:
display_list.append(pu.CubeCloud(np.row_stack((goto_pt,close_pt[2,0])),color=(0,250,250), size=(0.012,0.012,0.012)))
display_list.append(pu.Line(np.row_stack((goto_pt,close_pt[2,0])).A1,close_pt.A1,color=(255,20,0)))
p_erratic = np.row_stack((goto_pt_erratic,np.matrix((close_pt[2,0]))))
print 'p_erratic in thok0:', p_erratic.T
return p_erratic,close_pt,close_pt[2,0]
def _nnCostFunction(self, the_thetas, *args):
the_thetas = np.mat(the_thetas)
theta1 = np.reshape(the_thetas[0,0:self.num_hidden_nodes*401], (400+1, self.num_hidden_nodes)).T
theta2 = np.reshape(the_thetas[0,self.num_hidden_nodes*401:], (self.num_hidden_nodes+1, 10)).T
training_data_array = args
J=0
for data in training_data_array:
a1 = np.mat(data.fig).T
# 400*1 matrix
z2 = np.dot(theta1, np.row_stack((1, a1)))
# num_hidden_nodes*1 matrix
a2 = self.sigmoid(z2)
z3 = np.dot(theta2, np.row_stack((1, a2)))
# 10*1 matrix
a3 = self.sigmoid(z3)
y = [0] * 10 # y is a python list for easy initialization and is later turned into an np matrix (3 lines down).
y[data.label] = 1
# 1*10 list
for j in range(10):
J = J + np.mat(y).T[j,0]*math.log(a3[j,0])+(1-np.mat(y).T[j,0])*math.log(1-a3[j,0])
# numerically a3[j,0] could be smaller than 0 or larger than 1 a bit
J = -J/self.sample_num + self.LAMBDA/(2*self.sample_num)*(np.multiply(theta1[:,1:], theta1[:,1:]).sum()+np.multiply(theta2[:,1:], theta2[:,1:]).sum())
# print J
return J
def __init__(self,d,l, dtype=None):
self.d = d
self.l = l
[self.smolyak_points, self.smolyak_indices] = smolyak_grids(d,l)
self.u_grid = array( self.smolyak_points.T, order='C')
self.isup = max(max(self.smolyak_indices))
self.n_points = len(self.smolyak_points)
#self.grid = self.real_gri
Ts = chebychev( self.smolyak_points.T, self.n_points - 1 )
C = []
for comb in self.smolyak_indices:
p = reduce( mul, [Ts[comb[i],i,:] for i in range(self.d)] )
C.append(p)
C = numpy.row_stack( C )
# C is such that : X = theta * C
self.__C__ = C
self.__C_inv__ = numpy.linalg.inv(C) # would be better to store preconditioned matrix
self.bounds = numpy.row_stack([(0,1)]*d)
def test(self, ratings):
L = ratings.kv_dict.items()
R = np.matrix([I[1] for I in L]).T
sqr_err = list()
au = len(ratings.rows) - len(self.user_factor)
self.user_factor = np.row_stack([self.user_factor, np.matrix(np.zeros([au, self.factor]))])
self.user_bias = np.row_stack([self.user_bias, np.matrix(np.zeros([au, 1]))])
ai = len(ratings.cols) - len(self.item_factor)
self.item_factor = np.row_stack([self.item_factor, np.matrix(np.zeros([ai, self.factor]))])
self.item_bias = np.row_stack([self.item_bias, np.matrix(np.zeros([ai, 1]))])
for s in range(0, len(L), self.batch_size):
mini_batch = L[s : s + self.batch_size]
r = R[s : s + self.batch_size]
uid = [I[0][0] for I in mini_batch]
iid = [I[0][1] for I in mini_batch]
base_line = self.mu + self.user_bias[uid] + self.item_bias[iid]
r_pred = base_line + np.sum(np.multiply(self.user_factor[uid], self.item_factor[iid]), 1)
err = r - r_pred
sqr_err.append(float(err.T * err) / self.batch_size)
rmse = math.sqrt(np.mean(np.matrix(sqr_err)))
# sys.stderr.write('RMSE: %f\n' % (rmse))
return rmse
def save_Ian_E_H_Yen(self,folder):
if not self.B_lower is None:
print ('self.B_lower is not None, you should convert your problem with convertToOnesideInequalitySystem first')
raise
if not np.all(self.lowerbounds==0):
print ('lower bound constraint on variables should at 0')
raise
import os
Aeq=self.Aequalities.tocoo()
tmp=np.row_stack(([Aeq.shape[0],Aeq.shape[1],0.0],np.column_stack((Aeq.row+1,Aeq.col+1,Aeq.data))))
np.savetxt(os.path.join(folder,'Aeq'),tmp,fmt='%d %d %f')
np.savetxt(os.path.join(folder,'beq'),self.Bequalities,fmt='%f')
np.savetxt(os.path.join(folder,'c'),self.costsvector,fmt='%f')
nbvariables=self.costsvector.size
upperbounded=np.nonzero(~np.isinf(self.upperbounds))[0]
nbupperbounded=len(upperbounded)
Aineq2=sparse.coo_matrix((np.ones(nbupperbounded),(np.arange(nbupperbounded),upperbounded)),(nbupperbounded,nbvariables))
Aineq=scipy.sparse.vstack((self.Ainequalities,Aineq2)).tocoo()
bupper=np.hstack((self.B_upper,self.upperbounds[upperbounded]))
tmp=np.row_stack(([Aineq.shape[0],Aineq.shape[1],0.0],np.column_stack((Aineq.row+1,Aineq.col+1,Aineq.data))))
np.savetxt(os.path.join(folder,'A'),tmp,fmt='%d %d %f')
np.savetxt(os.path.join(folder,'b'),bupper,fmt='%f')
with open(os.path.join(folder,'meta'), 'w') as f:
f.write('nb %d\n'%nbvariables)
f.write('nf %d\n'%0)
f.write('mI %d\n'%Aineq.shape[0])
f.write('mE %d\n'%Aeq.shape[0])
def softmaxCostAndGradient(y1, target, w2):
""" Softmax cost function for word2vec models """
###################################################################
# Implement the cost and gradients for one predicted word vector #
# and one target word vector as a building block for word2vec #
# models, assuming the softmax prediction function and cross #
# entropy loss. #
# Inputs: #
# - predicted: numpy ndarray, predicted word vector (\hat{r} in #
# the written component) #
# - target: integer, the index of the target word #
# - outputVectors: "output" vectors for all tokens #
# Outputs: #
# - cost: cross entropy cost for the softmax word prediction #
# - gradPred: the gradient with respect to the predicted word #
# vector #
# - grad: the gradient with respect to all the other word #
# vectors #
# We will not provide starter code for this function, but feel #
# free to reference the code you previously wrote for this #
# assignment! #
###################################################################
out = np.dot(w2,y1)
y2 = softmaxactual(out)
cost = -math.log(y2[target])
y2[target] -= 1.0
d2= y2
grad2 = np.dot(np.row_stack(d2),np.column_stack(y1))
d1 = np.dot(np.transpose(w2), np.row_stack(d2))
return cost, d1, grad2
def add_margins(self, matrix, regions=2):
nrow, ncol = np.shape(matrix)
assert regions<=2
for i in range(regions):
nc = ncol/regions
nr = nrow
if i==0:
m = matrix[:, 0:nc]
else:
m = matrix[:, nc:]
left_col = np.ones((nr+2,1))
bottom_row = np.ones((1, nc))
top_row = np.reshape([[0,1] for _ in range(nc/2)], (1, nc))
right_row = np.reshape([[0,1] for _ in range(nr/2+1)], (nr+2, 1))
m = np.row_stack((top_row, m))
m = np.row_stack((m, bottom_row))
m = np.column_stack((left_col, m))
m = np.column_stack((m, right_row))
if i == 0:
ans = m
else:
ans = np.column_stack((ans, m))
return ans
def multi_scale_test(image, max_im_shrink):
# shrink detecting and shrink only detect big face
st = 0.5 if max_im_shrink >= 0.75 else 0.5 * max_im_shrink
det_s = detect_face(image, st)
if max_im_shrink > 0.75:
det_s = np.row_stack((det_s,detect_face(image,0.75)))
index = np.where(np.maximum(det_s[:, 2] - det_s[:, 0] + 1, det_s[:, 3] - det_s[:, 1] + 1) > 30)[0]
det_s = det_s[index, :]
# enlarge one times
bt = min(2, max_im_shrink) if max_im_shrink > 1 else (st + max_im_shrink) / 2
det_b = detect_face(image, bt)
# enlarge small iamge x times for small face
if max_im_shrink > 1.5:
det_b = np.row_stack((det_b,detect_face(image,1.5)))
if max_im_shrink > 2:
bt *= 2
while bt < max_im_shrink: # and bt <= 2:
det_b = np.row_stack((det_b, detect_face(image, bt)))
bt *= 2
det_b = np.row_stack((det_b, detect_face(image, max_im_shrink)))
# enlarge only detect small face
if bt > 1:
index = np.where(np.minimum(det_b[:, 2] - det_b[:, 0] + 1, det_b[:, 3] - det_b[:, 1] + 1) < 100)[0]
det_b = det_b[index, :]
else:
index = np.where(np.maximum(det_b[:, 2] - det_b[:, 0] + 1, det_b[:, 3] - det_b[:, 1] + 1) > 30)[0]
det_b = det_b[index, :]
return det_s, det_b
def _build_series(series, dim_names, comment, delta_name, delta_unit):
from glue.ligolw import array as ligolw_array
Attributes = ligolw.sax.xmlreader.AttributesImpl
elem = ligolw.LIGO_LW(
Attributes({u"Name": unicode(series.__class__.__name__)}))
if comment is not None:
elem.appendChild(ligolw.Comment()).pcdata = comment
elem.appendChild(ligolw.Time.from_gps(series.epoch, u"epoch"))
elem.appendChild(ligolw_param.Param.from_pyvalue(u"f0", series.f0,
unit=u"s^-1"))
delta = getattr(series, delta_name)
if numpy.iscomplexobj(series.data.data):
data = numpy.row_stack((numpy.arange(len(series.data.data)) * delta,
series.data.data.real, series.data.data.imag))
else:
data = numpy.row_stack((numpy.arange(len(series.data.data)) * delta,
series.data.data))
a = ligolw_array.Array.build(series.name, data, dim_names=dim_names)
a.Unit = str(series.sampleUnits)
dim0 = a.getElementsByTagName(ligolw.Dim.tagName)[0]
dim0.Unit = delta_unit
dim0.Start = series.f0
dim0.Scale = delta
elem.appendChild(a)
return elem
def setup_data(db,h,w):
corners = np.array([[0,0],[0,h-1],[w-1,0],[w-1,h-1]])
im1_pts = np.array(db_unpack(db)[0])
im1_pts = np.row_stack([im1_pts,corners])
im2_pts = np.array(db_unpack(db)[1])
im2_pts = np.row_stack([im2_pts,corners])
return im1_pts,im2_pts
def predict_test(batch, model, params, **kwparams):
""" some code duplication here with forward pass, but I think we want the freedom in future """
F = np.row_stack(x['image']['feat'] for x in batch)
lda_enabled = params.get('lda',0)
L = np.zeros((params.get('image_encoding_size',128),lda_enabled))
if lda_enabled:
L = np.row_stack(x['image']['topics'] for x in batch)
We = model['We']
if lda_enabled:
Wlda = model['Wlda']
lda = L.dot(Wlda)
be = model['be']
Xe = F.dot(We) + be # Xe becomes N x image_encoding_size
#print('L shape', L.shape)
#print('Wlda shape', Wlda.shape)
generator_str = params['generator']
Generator = decodeGenerator(generator_str)
Ys = []
guide_input = params.get('guide',None)
for i,x in enumerate(batch):
Xi = Xe[i,:]
if guide_input=='cca':
guide = get_guide(guide_input,F[i,:],kwparams.get('ccaweights'))
else:
guide = get_guide(guide_input,F[i,:],L=L[i,:])
if (lda_enabled and guide_input=="image") or (lda_enabled and not guide_input):
guide = lda[i,:]
print 'guide = lda'
gen_Y = Generator.predict(Xi, guide, model, model['Ws'], params, **kwparams)
Ys.append(gen_Y)
return Ys
def printMartix(self):
MNA1 = np.column_stack((self.C, self.D))
MNA2 = np.column_stack((self.G, self.B))
MNA = np.row_stack((MNA2, MNA1))
RHS = np.row_stack((self.U, self.I))
print "MNA", MNA
print "RHS", RHS
def get_scan(self, avoid_duplicate=False, avg=1, remove_graze=True):
''' avoid_duplicate - prevent duplicate scans which will happen if get_scan is
called at a rate faster than the scanning rate of the hokuyo.
avoid_duplicate == True, get_scan will block till new scan is received.
(~.2s for urg and 0.05s for utm)
'''
l = []
l2 = []
for i in range(avg):
hscan = self.hokuyo.get_scan(avoid_duplicate)
l.append(hscan.ranges)
l2.append(hscan.intensities)
ranges_mat = np.row_stack(l)
ranges_mat[np.where(ranges_mat==0)] = -1000. # make zero pointvery negative
ranges_avg = (ranges_mat.sum(0)/avg)
if self.flip:
ranges_avg = np.fliplr(ranges_avg)
hscan.ranges = ranges_avg
intensities_mat = np.row_stack(l2)
if self.hokuyo_type == 'utm':
hscan.intensities = (intensities_mat.sum(0)/avg)
if remove_graze:
if self.hokuyo_type=='utm':
hp.remove_graze_effect_scan(hscan)
else:
print 'hokuyo_scan.Hokuyo.get_scan: hokuyo type is urg, but remove_graze is True'
return hscan
def test_smolyak(self):
import numpy
f = lambda x: numpy.row_stack([
x[0,:] * x[1,:]**0.5,
x[1,:] * x[1,:] - x[0,:] * x[0,:]
])
bounds = numpy.row_stack([
[0.5,0.1],
[2,3]
])
from dolo.numeric.smolyak import SmolyakGrid
sg = SmolyakGrid(bounds,3)
values = f(sg.grid)
sg.fit_values(values)
theta_0 = sg.theta.copy()
def fobj(theta):
sg.theta = theta
return sg(sg.grid)
fobj(theta_0)
def __init__(self, grid, start, orientation='up', conf='white'):
"""
Initialize parameters
"""
self.rows = grid[0]
self.cols = grid[1]
self.orientation = orientation
self.conf = conf
self.current = start
if conf == 'white':
self.grid = np.zeros((self.rows, self.cols), dtype=int)
elif conf == 'black':
self.grid = np.ones((self.rows, self.cols), dtype=int)
elif conf == 'stripes':
re = np.r_[ 41 * [0, 0] ]
ro = np.r_[ 41 * [1, 1] ]
self.grid = np.row_stack(51 * (re, ro))[:-1]
elif conf == 'checker':
re = np.r_[ 41 * [0, 1] ]
ro = np.r_[ 41 * [1, 0] ]
self.grid = np.row_stack(51 * (re, ro))[:-1]
elif conf == 'random':
a = np.random.random((self.rows, self.cols)) + 0.5
self.grid = a.astype(int)
def array_to_ij(self, array):
assert array.shape == (self._iilen, self._jjlen), "{} != {}, {}".format(array.shape, self._iilen, self._jjlen)
#arange in the same order (and starting pos) as gchem grid
array = np.roll(array, 6, 0)
lons_regridded = [regrid(array[:,j], sum, 10) for j in range(self._jjlen)]
cols = map(lambda a: regrid_lats(np.array(a)), zip(*lons_regridded))
regridded = np.row_stack(cols)
return np.row_stack(((i+1,j+1,regridded[i,j]) for i,j in it.product(range(72),range(46))))
def absbound(mat): mat1=np.zeros((2,N)) mat=np.row_stack((mat1,mat)) mat=np.row_stack((mat,mat1)) mat2=np.zeros((N+4,2)) mat=np.column_stack((mat2,mat)) mat=np.column_stack((mat,mat2)) return mat
def prepare_data( batch, wordtoix, maxlen=None,sentTagMap=None,ixw = None):
"""Create the matrices from the datasets.
This pad each sequence to the same length: the lenght of the
longest sequence or maxlen.
if maxlen is set, we will cut all sequence to this maximum
length.
This swap the axis!
"""
seqs = []
xI = np.row_stack(x['image']['feat'] for x in batch)
for ix,x in enumerate(batch):
seqs.append([0] + [wordtoix[w] for w in x['sentence']['tokens'] if w in wordtoix] + [0])
# x: a list of sentences
lengths = [len(s) for s in seqs]
if maxlen is not None:
new_seqs = []
new_labels = []
new_lengths = []
for l, s, y in zip(lengths, seqs, labels):
if l < maxlen:
new_seqs.append(s)
new_labels.append(y)
new_lengths.append(l)
lengths = new_lengths
labels = new_labels
seqs = new_seqs
if len(lengths) < 1:
return None, None, None
n_samples = len(seqs)
maxlen = np.max(lengths)
xW = np.zeros((maxlen, n_samples)).astype('int64')
x_mask = np.zeros((maxlen, n_samples)).astype(theano.config.floatX)
for idx, s in enumerate(seqs):
xW[:lengths[idx], idx] = s
x_mask[:lengths[idx], idx] = 1.
if sentTagMap != None:
for i,sw in enumerate(s):
if sentTagMap[batch[idx]['sentence']['sentid']].get(ixw[sw],'') == 'JJ':
x_mask[i,idx] = 2
inp_list = [xW, xI, x_mask]
if 'aux_inp' in batch[0]['image']:
xAux = np.row_stack(x['image']['aux_inp'] for x in batch)
#xAux = np.tile(xAux,[maxlen,1,1])
inp_list.append(xAux)
return inp_list, (np.sum(lengths) - n_samples)
def plot_srhel():
print(" SR Helicity")
# Set Figure Size (1000 x 800)
plt.figure(figsize=(width,height),frameon=False)
P = nc.variables['P']
PB = nc.variables['PB']
UU = nc.variables['U']
VV = nc.variables['V']
PH = nc.variables['PH']
PHB = nc.variables['PHB']
# Need pressures, temps and mixing ratios
PR = P[time] + PB[time]
PHT = np.add(PH[time],PHB[time])
ZH = np.divide(PHT, 9.81)
U = UU[time]
V = VV[time]
for j in range(len(U[1,:,1])):
curcol_c = []
curcol_Umo = []
curcol_Vmo = []
for i in range(len(V[1,1,:])):
sparms = severe.SRHEL_CALC(U[:,j,i], V[:,j,i], ZH[:,j,i], PR[:,j,i])
curcol_c.append(sparms[0])
curcol_Umo.append(sparms[1])
curcol_Vmo.append(sparms[2])
np_curcol_c = np.array(curcol_c)
np_curcol_Umo = np.array(curcol_Umo)
np_curcol_Vmo = np.array(curcol_Vmo)
if j == 0:
srhel = np_curcol_c
U_srm = np_curcol_Umo
V_srm = np_curcol_Vmo
else:
srhel = np.row_stack((srhel, np_curcol_c))
U_srm = np.row_stack((U_srm, np_curcol_Umo))
V_srm = np.row_stack((V_srm, np_curcol_Vmo))
#print " SRHEL: ", np.shape(srhel)
# Now plot
SRHEL_LEVS = range(50,800,50)
srhel = np.nan_to_num(srhel)
SRHEL=plt.contourf(x,y,srhel,SRHEL_LEVS)
u_mo_kts = U_srm * 1.94384449
v_mo_kts = V_srm * 1.94384449
plt.barbs(x_th,y_th,u_mo_kts[::thin,::thin],\
v_mo_kts[::thin,::thin], length=5,\
sizes={'spacing':0.2},pivot='middle')
title = '0-3 km SRHelicity, Storm Motion (kt)'
prodid = 'hlcy'
units = "m" + u'\u00B2' + '/s' + u'\u00B2'
drawmap(SRHEL, title, prodid, units)
def _extract_grid_lines(LINES):
point_list = []
line_list = []
for (x, y, z) in LINES:
assert x.ndim == y.ndim == z.ndim
assert x.shape == y.shape, "Arrays x and y must have same shape."
assert y.shape == z.shape, "Arrays y and z must have same shape."
#
xyz = [x.ravel(), y.ravel(), z.ravel()]
points = np.column_stack(xyz).ravel()
points.shape = (-1, 3)
lines = np.zeros((0, 2), dtype='l')
#
grid = np.arange(x.size, dtype='l').reshape(x.shape)
if x.ndim == 1:
p0 = grid[:-1].ravel()
p1 = grid[+1:].ravel()
verts = [p0,p1]
lines = np.column_stack(verts).ravel()
lines.shape = (-1, 2)
elif x.ndim == 2:
p0 = grid[:-1, :-1].ravel()
p1 = grid[+1:, :-1].ravel()
p2 = grid[+1:, +1:].ravel()
p3 = grid[:-1, +1:].ravel()
verts = [p0,p1,p2,p3]
polys = np.column_stack(verts).ravel()
polys.shape = (-1, 4)
lines = polys[:,[0,1,1,2,2,3,3,0]].ravel()
lines.shape = (-1, 2)
elif x.ndim == 3:
p0 = grid[:-1, :-1, :-1].ravel()
p1 = grid[+1:, :-1, :-1].ravel()
p2 = grid[+1:, +1:, :-1].ravel()
p3 = grid[:-1, +1:, :-1].ravel()
p4 = grid[:-1, :-1, +1:].ravel()
p5 = grid[+1:, :-1, +1:].ravel()
p6 = grid[+1:, +1:, +1:].ravel()
p7 = grid[:-1, +1:, +1:].ravel()
verts = [p0,p1,p2,p3,
p4,p5,p6,p7,
p0,p1,p5,p4,
p2,p6,p7,p3,
p0,p3,p7,p4,
p1,p2,p6,p5]
polys = np.column_stack(verts).ravel()
polys.shape = (-1, 4)
lines = polys[:,[0,1,1,2,2,3,3,0]].ravel()
lines.shape = (-1, 2)
point_list.append(points)
line_list.append(lines)
offset = 0
for points, lines in zip(point_list, line_list):
lines += offset
offset += len(points)
return (np.row_stack(point_list),
np.row_stack(line_list))
def xyzToHomogenous(v, floating_vector=False):
""" convert 3XN matrix, to 4XN matrix in homogeneous coords
"""
# v = np.matrix(v)
# v = v.reshape(3,1)
if floating_vector == False:
return np.row_stack((v, np.ones(v.shape[1])))
else:
return np.row_stack((v, np.zeros(v.shape[1])))
def quick_color_check(target_matrix, source_matrix, num_chips):
""" Quickly plot target matrix values against source matrix values to determine
over saturated color chips or other issues.
Inputs:
source_matrix = an nrowsXncols matrix containing the avg red, green, and blue values for each color chip
of the source image
target_matrix = an nrowsXncols matrix containing the avg red, green, and blue values for each color chip
of the target image
num_chips = number of color card chips included in the matrices (integer)
:param source_matrix: numpy.ndarray
:param target_matrix: numpy.ndarray
:param num_chips: int
"""
# Imports
from plotnine import ggplot, geom_point, geom_smooth, theme_seaborn, facet_grid, geom_label, scale_x_continuous, \
scale_y_continuous, scale_color_manual, aes
import pandas as pd
# Extract and organize matrix info
tr = target_matrix[:num_chips, 1:2]
tg = target_matrix[:num_chips, 2:3]
tb = target_matrix[:num_chips, 3:4]
sr = source_matrix[:num_chips, 1:2]
sg = source_matrix[:num_chips, 2:3]
sb = source_matrix[:num_chips, 3:4]
# Create columns of color labels
red = []
blue = []
green = []
for i in range(num_chips):
red.append('red')
blue.append('blue')
green.append('green')
# Make a column of chip numbers
chip = np.arange(0, num_chips).reshape((num_chips, 1))
chips = np.row_stack((chip, chip, chip))
# Combine info
color_data_r = np.column_stack((sr, tr, red))
color_data_g = np.column_stack((sg, tg, green))
color_data_b = np.column_stack((sb, tb, blue))
all_color_data = np.row_stack((color_data_b, color_data_g, color_data_r))
# Create a dataframe with headers
dataset = pd.DataFrame({
'source': all_color_data[:, 0],
'target': all_color_data[:, 1],
'color': all_color_data[:, 2]
})
# Add chip numbers to the dataframe
dataset['chip'] = chips
dataset = dataset.astype({
'color': str,
'chip': str,
'target': float,
'source': float
})
# Make the plot
p1 = ggplot(dataset, aes(x='target', y='source', color='color', label='chip')) + \
geom_point(show_legend=False, size=2) + \
geom_smooth(method='lm', size=.5, show_legend=False) + \
theme_seaborn() + facet_grid('.~color') + \
geom_label(angle=15, size=7, nudge_y=-.25, nudge_x=.5, show_legend=False) + \
scale_x_continuous(limits=(-5, 270)) + scale_y_continuous(limits=(-5, 275)) + \
scale_color_manual(values=['blue', 'green', 'red'])
# Autoincrement the device counter
params.device += 1
# Reset debug
if params.debug is not None:
if params.debug == 'print':
p1.save(os.path.join(params.debug_outdir, 'color_quick_check.png'))
elif params.debug == 'plot':
print(p1)
def train_controller(Controller, Controller_optim, rnd_fix, rnd_learn,
rnd_learn_optim, trainloader, testloader, model,
optimizer, criterion, hyperparams):
"""
The controller is updated with a score function gradient estimator
(i.e., REINFORCE), with the reward being c/valid_ppl, where valid_ppl
is computed on a minibatch of validation data.
A moving average baseline is used.
The controller is trained for 2000 steps per epoch.
"""
logger = Logger(os.path.join(hyperparams['checkpoint'], 'search_log.txt'),
title='')
logger.set_names([
'Loss', 'Baseline', 'Reward', 'Action', 'Binary', 'Valid Loss',
'Valid Acc.', 'Sparse', 'rnd_loss', 'p'
])
logger_all = Logger(os.path.join(hyperparams['checkpoint'],
'search_log_all_sampled.txt'),
title='')
logger_all.set_names([
'Loss', 'Baseline', 'Reward', 'Action', 'Binary', 'Valid Loss',
'Valid Acc.', 'Sparse'
])
model_fc = Controller
model_fc.train()
model_rnd_fix = rnd_fix
model_rnd_fix.eval()
model_rnd_learn = rnd_learn
model_rnd_learn.train()
baseline = None
total_loss = 0
buffer = ReplayBuffer(30)
buffer_sparse = ReplayBuffer(30)
update_mode = 'online'
for step in range(hyperparams['controller_max_step']):
print('************************* (' + str(step + 1) + '/' +
str(hyperparams['controller_max_step']) + ')******')
adjust_learning_rate(optimizer, step, args, hyperparams)
actions, log_probs = model_fc.sample(replay=None)
#sample N connection for val (for updating theta)
actions_validate, log_probs_validate = model_fc.sample(
batch_size=args.validate_size)
decimal_code_all = []
binary_code_all = []
# get the action code (binary to decimal)
actions_code_1, binary_code_1 = get_action_code(actions)
decimal_code_all.append(actions_code_1)
binary_code_all.append(binary_code_1)
for i in range(actions_validate[0].size(0)):
binary_code = ''
for action in actions_validate:
binary_code = binary_code + str(action[i].item())
decimal_code = int(binary_code, 2)
decimal_code_all.append(decimal_code)
binary_code_all.append(binary_code)
#get reward (train one "step")
rewards_org, val_acc, val_loss, sparse_portion = get_reward(
None, testloader, model, optimizer, criterion, actions, step,
hyperparams)
rewards_validate, val_acc_validate, val_loss_validate, sparse_portion_validate = get_reward(
None, testloader, model, optimizer, criterion, actions_validate,
step, hyperparams)
val_acc = np.row_stack((val_acc, val_acc_validate))
val_loss = np.row_stack((val_loss, val_loss_validate))
#buf_rewards
rewards = np.row_stack((rewards_org, rewards_validate))
# #buf_action
temp_action = torch.cat(actions).view(1, -1)
temp_actions_validate = torch.cat(actions_validate).view(
args.validate_size, -1)
buf_action = torch.cat((temp_action, temp_actions_validate), 0)
# #buf_prob
temp_prob = torch.exp(log_probs).detach()
temp_prob_val = torch.exp(log_probs_validate).detach()
buf_prob = torch.cat((temp_prob, temp_prob_val), 0)
# #store - buffer
buffer.add_new(buf_action, rewards, buf_prob, step)
#cal rnd
val_fix = model_rnd_fix(buf_action.float())
val_learn = model_rnd_learn(buf_action.float())
reward_rnd = ((val_fix - val_learn)**2).sum()
rewards = rewards + reward_rnd.detach().cpu().data.numpy()
# moving average baseline
if baseline is None:
baseline = rewards.mean()
else:
decay = hyperparams['ema_baseline_decay']
baseline = decay * baseline + (1 - decay) * rewards.mean()
adv = rewards - baseline
adv = adv
log_probs = torch.cat((log_probs, log_probs_validate), 0)
loss = -log_probs * tools.get_variable(adv, True, requires_grad=False)
loss = loss.mean(dim=0, keepdim=True)
loss = loss.sum()
# update
Controller_optim.zero_grad()
rnd_learn_optim.zero_grad()
loss.backward()
reward_rnd.mean().backward()
if hyperparams['controller_grad_clip'] > 0:
torch.nn.utils.clip_grad_norm(model_fc.parameters(),
hyperparams['controller_grad_clip'])
rnd_learn_optim.step()
Controller_optim.step()
log = 'Step: {step}| Loss: {loss:.4f}| Action: {act} |Baseline: {base:.4f}| ' \
'Reward {re:.4f}| Valid Acc {acc:.4f}'.format(loss=loss.item(), base=baseline, act = binary_code,
re=rewards[0].item(), acc=val_acc[0].item(), step=step)
print(log)
logger.append([loss.item(), baseline, rewards[0].item(), actions_code_1, binary_code_1, \
val_loss[0].item(), val_acc[0].item(), (binary_code.count('0')/len(binary_code)), reward_rnd.item(), torch.exp(log_probs).mean().item()])
for i in range(len(binary_code_all)):
logger_all.append([
loss.item(), baseline, rewards[i].item(), decimal_code_all[i],
binary_code_all[i], val_loss[i].item(), val_acc[i].item(),
(binary_code_all[i].count('0') / len(binary_code_all[i]))
])
save_checkpoint(
{
'iters': step + 1,
'state_dict': model_fc.state_dict(),
'optimizer': Controller_optim.state_dict(),
},
checkpoint=hyperparams['checkpoint'])
save_checkpoint(
{
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict()
},
checkpoint=hyperparams['checkpoint'],
filename='model.pth.tar')
if step >= 10:
# use old data (buff)
replay = buffer.sample(num=1)
log_probs, replay_rewards, prob_ratio = model_fc.sample(
replay=replay)
adv = replay_rewards - baseline
eps_clip = 0.1
temp_surr1 = (prob_ratio.detach() * log_probs).mm(
tools.get_variable(adv, True, requires_grad=False))
temp_surr2 = (
torch.clamp(prob_ratio.detach(), 1 - eps_clip, 1 + eps_clip) *
log_probs).mm(
tools.get_variable(adv, True, requires_grad=False))
surr1 = temp_surr1.sum() / (args.validate_size + 1)
surr2 = temp_surr2.sum() / (args.validate_size + 1)
loss = 0.1 * -torch.min(surr1, surr2)
# update
Controller_optim.zero_grad()
loss.backward()
if hyperparams['controller_grad_clip'] > 0:
torch.nn.utils.clip_grad_norm(
model_fc.parameters(), hyperparams['controller_grad_clip'])
Controller_optim.step()
def plot_decision_rule(model,
dr,
state,
plot_controls=None,
bounds=None,
n_steps=10,
s0=None,
**kwargs):
"""
Plots decision rule
Parameters:
-----------
model:
"fg" or "fga" model
dr:
decision rule
state:
state variable that is supposed to vary
plot_controls: string, list or None
- if None, return a pandas dataframe
- if a string denoting a control variable, plot this variable as a
function of the state
- if a list of strings, plot several variables
bounds: array_like
the state variable varies from bounds[0] to bounds[1]. By default,
boundaries are looked for in the the decision rule then in
the model.
n_steps: int
number of points to be plotted
s0: array_like or None
value for the state variables, that remain constant. Defaults to
`model.calibration['states']`
Returns:
--------
dataframe or plot, depending on the value of `plot_controls`
"""
states_names = model.symbols['states']
controls_names = model.symbols['controls']
index = states_names.index(str(state))
if bounds is None:
if hasattr(dr, 'a'):
bounds = [dr.a[index], dr.b[index]]
else:
approx = model.options['approximation_space']
bounds = [approx['a'][index], approx['b'][index]]
values = numpy.linspace(bounds[0], bounds[1], n_steps)
if s0 is None:
s0 = model.calibration['states']
svec = numpy.row_stack([s0] * n_steps)
svec[:, index] = values
xvec = dr(svec)
l = [svec, xvec]
series = model.symbols['states'] + model.symbols['controls']
if 'auxiliary' in model.functions:
p = model.calibration['parameters']
pp = numpy.row_stack([p] * n_steps)
avec = model.functions['auxiliary'](svec, xvec, pp)
l.append(avec)
series.extend(model.symbols['auxiliaries'])
tb = numpy.concatenate(l, axis=1)
df = pandas.DataFrame(tb, columns=series)
if plot_controls is None:
return df
else:
if isinstance(plot_controls, str):
cn = plot_controls
pyplot.plot(values, df[cn], **kwargs)
else:
for cn in plot_controls:
pyplot.plot(values, df[cn], label=cn, **kwargs)
pyplot.legend()
pyplot.xlabel('state = {}'.format(state))
def solve_risky_ss(model, X_bar, X_s, verbose=False):
import numpy
from dolo.compiler.compiling import compile_function
import time
from dolo.compiler.compiler_global import simple_global_representation
[y, x, parms] = model.read_calibration()
sigma = model.read_covariances()
sgm = simple_global_representation(model, substitute_auxiliary=True)
states = sgm['states']
controls = sgm['controls']
shocks = sgm['shocks']
parameters = sgm['parameters']
f_eqs = sgm['f_eqs']
g_eqs = sgm['g_eqs']
g_args = [s(-1) for s in states] + [c(-1) for c in controls] + shocks
f_args = states + controls + [v(1)
for v in states] + [v(1) for v in controls]
p_args = parameters
g_fun = compile_function(g_eqs, g_args, p_args, 2, return_function=True)
f_fun = compile_function(f_eqs, f_args, p_args, 3, return_function=True)
epsilons_0 = np.zeros((sigma.shape[0]))
from numpy import dot
from dolo.numeric.tensor import sdot, mdot
def residuals(X, sigma, parms, g_fun, f_fun):
import numpy
dummy_x = X[0:1, 0]
X_bar = X[1:, 0]
S_bar = X[0, 1:]
X_s = X[1:, 1:]
[n_x, n_s] = X_s.shape
n_e = sigma.shape[0]
xx = np.concatenate([S_bar, X_bar, epsilons_0])
[g_0, g_1, g_2] = g_fun(xx, parms)
[f_0, f_1, f_2,
f_3] = f_fun(np.concatenate([S_bar, X_bar, S_bar, X_bar]), parms)
res_g = g_0 - S_bar
# g is a first order function
g_s = g_1[:, :n_s]
g_x = g_1[:, n_s:n_s + n_x]
g_e = g_1[:, n_s + n_x:]
g_se = g_2[:, :n_s, n_s + n_x:]
g_xe = g_2[:, n_s:n_s + n_x, n_s + n_x:]
# S(s,e) = g(s,x,e)
S_s = g_s + dot(g_x, X_s)
S_e = g_e
S_se = g_se + mdot(g_xe, [X_s, numpy.eye(n_e)])
# V(s,e) = [ g(s,x,e) ; x( g(s,x,e) ) ]
V_s = np.row_stack([S_s, dot(X_s, S_s)]) # ***
V_e = np.row_stack([S_e, dot(X_s, S_e)])
V_se = np.row_stack([S_se, dot(X_s, S_se)])
# v(s) = [s, x(s)]
v_s = np.row_stack([numpy.eye(n_s), X_s])
# W(s,e) = [xx(s,e); yy(s,e)]
W_s = np.row_stack([v_s, V_s])
#return
nn = n_s + n_x
f_v = f_1[:, :nn]
f_V = f_1[:, nn:]
f_1V = f_2[:, :, nn:]
f_VV = f_2[:, nn:, nn:]
f_1VV = f_3[:, :, nn:, nn:]
# E = lambda v: np.tensordot(v, sigma, axes=((2,3),(0,1)) ) # expectation operator
F = f_0 + 0.5 * np.tensordot(
mdot(f_VV, [V_e, V_e]), sigma, axes=((1, 2), (0, 1)))
F_s = sdot(f_1, W_s)
f_see = mdot(f_1VV, [W_s, V_e, V_e]) + 2 * mdot(f_VV, [V_se, V_e])
F_s += 0.5 * np.tensordot(f_see, sigma, axes=(
(2, 3), (0, 1))) # second order correction
resp = np.row_stack(
[np.concatenate([dummy_x, res_g]),
np.column_stack([F, F_s])])
return resp
# S_bar = s_fun_init( numpy.atleast_2d(X_bar).T ,parms).flatten()
# S_bar = S_bar.flatten()
S_bar = [
y[i] for i, v in enumerate(model.variables)
if v in model['variables_groups']['states']
]
S_bar = np.array(S_bar)
X0 = np.row_stack(
[np.concatenate([np.zeros(1), S_bar]),
np.column_stack([X_bar, X_s])])
from dolo.numeric.solver import solver
fobj = lambda X: residuals(X, sigma, parms, g_fun, f_fun)
if verbose:
val = fobj(X0)
print('val')
print(val)
# exit()
t = time.time()
sol = solver(fobj,
X0,
method='lmmcp',
verbose=verbose,
options={
'preprocess': False,
'eps1': 1e-15,
'eps2': 1e-15
})
if verbose:
print('initial guess')
print(X0)
print('solution')
print sol
print('initial residuals')
print(fobj(X0))
print('residuals')
print fobj(sol)
s = time.time()
if verbose:
print('Elapsed : {0}'.format(s - t))
#sol = solver(fobj,X0, method='fsolve', verbose=True, options={'preprocessor':False})
norm = lambda x: numpy.linalg.norm(x, numpy.inf)
if verbose:
print("Initial error: {0}".format(norm(fobj(X0))))
print("Final error: {0}".format(norm(fobj(sol))))
print("Solution")
print(sol)
X_bar = sol[1:, 0]
S_bar = sol[0, 1:]
X_s = sol[1:, 1:]
# compute transitions
n_s = len(states)
n_x = len(controls)
[g, dg, junk] = g_fun(np.concatenate([S_bar, X_bar, epsilons_0]), parms)
g_s = dg[:, :n_s]
g_x = dg[:, n_s:n_s + n_x]
P = g_s + dot(g_x, X_s)
if verbose:
eigenvalues = numpy.linalg.eigvals(P)
print eigenvalues
eigenvalues = [abs(e) for e in eigenvalues]
eigenvalues.sort()
print(eigenvalues)
return [S_bar, X_bar, X_s, P]
points_virtual = []
if pos_camera == "left":
for key in list(project_keyboard.black_keys())[:3]:
pixel_corner_img = find_black_key_corner(key)
bbox_virtual3d = project_keyboard.bounding_box(key)
pixel_corner_virtual = np.append(bbox_virtual3d[-2,:], 1)
points_virtual.append(pixel_corner_virtual)
points_img.append(pixel_corner_img)
else:
for key in list(project_keyboard.black_keys())[-3:]:
pixel_corner_img = find_black_key_corner(key)
bbox_virtual3d = project_keyboard.bounding_box(key)
pixel_corner_virtual = np.append(bbox_virtual3d[-1,:], 1)
points_virtual.append(pixel_corner_virtual)
points_img.append(pixel_corner_img)
points_virtual = np.row_stack(points_virtual)
points_img = np.row_stack(points_img)
# Find 3d projection with black keys
T_virtual3d_to_img = project_keyboard.perspective_transformation_3d(T_virtual_to_img, points_virtual, points_img)
T_virtual3d_to_virtual = T_img_to_virtual.dot(T_virtual3d_to_img)
points_img_flat = points_virtual.dot(T_virtual3d_to_img.T)
points_img_flat /= points_img_flat[:,-1,np.newaxis]
points_virtual_flat = points_img_flat.dot(T_img_to_virtual.T)
points_virtual_flat /= points_virtual_flat[:,-1,np.newaxis]
points_virtual_flat = points_virtual_flat.astype(np.int32)
cv2.circle(img_virtual, tuple(points_virtual_flat[0,:2][::-1]), 10, (0,0,255), 3)
cv2.circle(img_virtual, tuple(points_virtual_flat[1,:2][::-1]), 10, (0,0,255), 3)
cv2.circle(img_virtual, tuple(points_virtual_flat[2,:2][::-1]), 10, (0,0,255), 3)
file_names = [
f for f in listdir('./Labels/001') if isfile(join(r'./Labels/001', f))
]
if file_names[0] == '.DS_Store':
del file_names[0]
#print file_names
start_matrix = [['xmin', 'ymin', 'xmax', 'ymax', 'image_name', 'class']]
for i in range(len(file_names)):
if file_names[i] != 'test.txt':
textfile_name = file_names[i]
label_matrix = np.loadtxt(os.path.join('./Labels/001', textfile_name),
dtype='str')
dim = label_matrix.shape
if dim[0] == 0:
continue
if len(dim) == 1:
label_matrix = label_matrix.reshape((1, dim[0]))
print textfile_name
dim_new = label_matrix.shape
column_vec = np.zeros(shape=(dim_new[0], )).astype(np.str)
#image_name = textfile_name[0:-3] + 'jpg'
column_vec[:, ] = textfile_name
new_matrix = np.zeros(shape=(dim_new[0], 6)).astype(np.str)
new_matrix[:, 0:4] = label_matrix[:, 0:4]
new_matrix[:, 4] = column_vec
new_matrix[:, 5] = label_matrix[:, 4]
start_matrix = np.row_stack((start_matrix, new_matrix))
print start_matrix
fmt = '%s %s %s %s %s %s'
np.savetxt('label.txt', start_matrix, fmt=fmt)
def extend_row(self, dim2arr):
row = np.zeros(dim2arr.shape[1])
dim2arr = np.row_stack((dim2arr, row))
return dim2arr
def findMainDirectionEMA(lines):
#FINDMAINDIRECTION compute vp from set of lines
# Detailed explanation goes here
print('Computing vanishing point:\n')
# arcList = [];
# for i = 1:length(edge)
# panoLst = edge(i).panoLst;
# if size(panoLst,1) == 0
# continue;
# end
# arcList = [arcList; panoLst];
# end
## initial guess
segNormal = lines[:, 0:3]
segLength = lines[:, 6]
segScores = np.ones([lines.shape[0], 1])
#lines(:,8);
shortSegValid = segLength < 5 * np.pi / 180
segNormal = segNormal[~shortSegValid, :]
segLength = segLength[~shortSegValid]
segScores = segScores[~shortSegValid]
numLinesg = segNormal.shape[0]
[candiSet, tri] = icosahedron2sphere.icosahedron2sphere(3)
ang = np.arccos(np.sum(
candiSet[tri[0, 0], :] * candiSet[tri[0, 1], :])) / np.pi * 180
binRadius = ang / 2
[initXYZ, score, angle] = sphereHoughVote(segNormal, segLength, segScores,
2 * binRadius, 2, candiSet, True)
if len(initXYZ) == 0:
print('Initial Failed\n')
mainDirect = []
return
print('Initial Computation: #d candidates, #d line segments\n',
candiSet.shape[0], numLinesg)
print(
'direction 1: #f #f #f\ndirection 2: #f #f #f\ndirection 3: #f #f #f\n',
initXYZ[0, 0], initXYZ[0, 1], initXYZ[0, 2], initXYZ[1, 0],
initXYZ[1, 1], initXYZ[1, 2], initXYZ[2, 0], initXYZ[2, 1], initXYZ[2,
2])
## iterative refine
iter_max = 3
[candiSet, tri] = icosahedron2sphere.icosahedron2sphere(5)
numCandi = candiSet.shape[0]
angD = np.arccos(np.sum(
candiSet[tri[0, 0], :] * candiSet[tri[0, 1], :])) / np.pi * 180
binRadiusD = angD / 2
curXYZ = initXYZ
tol = np.linspace(4 * binRadius, 4 * binRadiusD, iter_max)
# shrink down #ls and #candi
for iter in np.arange(iter_max):
dot1 = np.abs(np.sum(segNormal * repmat(curXYZ[0, :], numLinesg, 1),
1))
dot2 = np.abs(np.sum(segNormal * repmat(curXYZ[1, :], numLinesg, 1),
1))
dot3 = np.abs(np.sum(segNormal * repmat(curXYZ[2, :], numLinesg, 1),
1))
valid1 = dot1 < np.cos((90 - tol[iter]) * np.pi / 180)
valid2 = dot2 < np.cos((90 - tol[iter]) * np.pi / 180)
valid3 = dot3 < np.cos((90 - tol[iter]) * np.pi / 180)
valid = valid1 | valid2 | valid3
if (sum(valid) == 0):
print('ZERO line segment for voting\n')
break
subSegNormal = segNormal[valid, :]
subSegLength = segLength[valid]
subSegScores = segScores[valid]
dot1 = np.abs(np.sum(candiSet * repmat(curXYZ[0, :], numCandi, 1), 1))
dot2 = np.abs(np.sum(candiSet * repmat(curXYZ[1, :], numCandi, 1), 1))
dot3 = np.abs(np.sum(candiSet * repmat(curXYZ[2, :], numCandi, 1), 1))
valid1 = dot1 > np.cos(tol[iter] * np.pi / 180)
valid2 = dot2 > np.cos(tol[iter] * np.pi / 180)
valid3 = dot3 > np.cos(tol[iter] * np.pi / 180)
valid = valid1 | valid2 | valid3
if (sum(valid) == 0):
print('ZERO candidate for voting\n')
break
subCandiSet = candiSet[valid, :]
[tcurXYZ, _,
_] = sphereHoughVote(subSegNormal, subSegLength, subSegScores,
2 * binRadiusD, 2, subCandiSet, True)
if (len(tcurXYZ) == 0):
print('NO answer found!\n')
break
curXYZ = tcurXYZ
print('#d-th iteration: #d candidates, #d line segments\n', iter,
subCandiSet.shape[0], len(subSegScores))
print(
'direction 1: #f #f #f\ndirection 2: #f #f #f\ndirection 3: #f #f #f\n',
curXYZ[0, 0], curXYZ[0, 1], curXYZ[0, 2], curXYZ[1, 0], curXYZ[1, 1],
curXYZ[1, 2], curXYZ[2, 0], curXYZ[2, 1], curXYZ[2, 2])
mainDirect = curXYZ
mainDirect[0, :] = mainDirect[0, :] * np.sign(mainDirect[0, 2])
mainDirect[1, :] = mainDirect[1, :] * np.sign(mainDirect[1, 2])
mainDirect[2, :] = mainDirect[2, :] * np.sign(mainDirect[2, 2])
uv = CoordsTransform.xyz2uvN(mainDirect, 0)
I1 = np.argmax(uv[1, :])
J = np.setdiff1d([
0,
1,
2,
], I1)
I2 = np.argmin(np.abs(np.sin(uv[0, J])))
I2 = J[I2]
I3 = np.setdiff1d([0, 1, 2], [I1, I2])
mainDirect = np.row_stack(
(mainDirect[I1, :], mainDirect[I2, :], mainDirect[I3, :]))
mainDirect[0, :] = mainDirect[0, :] * np.sign(mainDirect[0, 2])
mainDirect[1, :] = mainDirect[1, :] * np.sign(mainDirect[1, 1])
mainDirect[2, :] = mainDirect[2, :] * np.sign(mainDirect[2, 0])
mainDirect = np.row_stack((mainDirect, -mainDirect))
# score = 0;
return [mainDirect, score, angle]
def get_neighbour_info(source_geo_def, target_geo_def, radius_of_influence,
neighbours=8, epsilon=0, reduce_data=True,
nprocs=1, segments=None):
"""Returns neighbour info
Parameters
----------
source_geo_def : object
Geometry definition of source
target_geo_def : object
Geometry definition of target
radius_of_influence : float
Cut off distance in meters
neighbours : int, optional
The number of neigbours to consider for each grid point
epsilon : float, optional
Allowed uncertainty in meters. Increasing uncertainty
reduces execution time
reduce_data : bool, optional
Perform initial coarse reduction of source dataset in order
to reduce execution time
nprocs : int, optional
Number of processor cores to be used
segments : int or None
Number of segments to use when resampling.
If set to None an estimate will be calculated
Returns
-------
(valid_input_index, valid_output_index,
index_array, distance_array) : tuple of numpy arrays
Neighbour resampling info
"""
if source_geo_def.size < neighbours:
warnings.warn('Searching for %s neighbours in %s data points' %
(neighbours, source_geo_def.size))
if segments is None:
cut_off = 3000000
if target_geo_def.size > cut_off:
segments = int(target_geo_def.size / cut_off)
else:
segments = 1
# Find reduced input coordinate set
valid_input_index, source_lons, source_lats = _get_valid_input_index(source_geo_def, target_geo_def,
reduce_data,
radius_of_influence,
nprocs=nprocs)
# Create kd-tree
try:
resample_kdtree = _create_resample_kdtree(source_lons, source_lats,
valid_input_index,
nprocs=nprocs)
except EmptyResult:
# Handle if all input data is reduced away
valid_output_index, index_array, distance_array = \
_create_empty_info(source_geo_def, target_geo_def, neighbours)
return (valid_input_index, valid_output_index, index_array,
distance_array)
if segments > 1:
# Iterate through segments
for i, target_slice in enumerate(geometry._get_slice(segments,
target_geo_def.shape)):
# Query on slice of target coordinates
next_voi, next_ia, next_da = \
_query_resample_kdtree(resample_kdtree, source_geo_def,
target_geo_def,
radius_of_influence, target_slice,
neighbours=neighbours,
epsilon=epsilon,
reduce_data=reduce_data,
nprocs=nprocs)
# Build result iteratively
if i == 0:
# First iteration
valid_output_index = next_voi
index_array = next_ia
distance_array = next_da
else:
valid_output_index = np.append(valid_output_index, next_voi)
if neighbours > 1:
index_array = np.row_stack((index_array, next_ia))
distance_array = np.row_stack((distance_array, next_da))
else:
index_array = np.append(index_array, next_ia)
distance_array = np.append(distance_array, next_da)
else:
# Query kd-tree with full target coordinate set
full_slice = slice(None)
valid_output_index, index_array, distance_array = \
_query_resample_kdtree(resample_kdtree, source_geo_def,
target_geo_def,
radius_of_influence, full_slice,
neighbours=neighbours,
epsilon=epsilon,
reduce_data=reduce_data,
nprocs=nprocs)
# Check if number of neighbours is potentially too low
if neighbours > 1:
if not np.all(np.isinf(distance_array[:, -1])):
warnings.warn(('Possible more than %s neighbours '
'within %s m for some data points') %
(neighbours, radius_of_influence))
return valid_input_index, valid_output_index, index_array, distance_array
def combineEdgesN(edges):
#COMBINEEDGES Combine some small line segments, should be very conservative
# lines: combined line segments
# ori_lines: original line segments
# line format: [nx ny nz projectPlaneID umin umax LSfov score]
arcList = []
for i in np.arange(edges.shape[0]):
panoLst = edges[i].panoLst
if panoLst[0].shape[0] == 0:
continue
if (len(arcList)) == 0:
arcList = panoLst
else:
arcList = np.row_stack((arcList, panoLst))
## ori lines
numLine = arcList.shape[0]
ori_lines = np.zeros((numLine, 8))
areaXY = np.abs(
np.sum(arcList[:, 0:3] * repmat([0, 0, 1], arcList.shape[0], 1), 1))
areaYZ = np.abs(
np.sum(arcList[:, 0:3] * repmat([1, 0, 0], arcList.shape[0], 1), 1))
areaZX = np.abs(
np.sum(arcList[:, 0:3] * repmat([0, 1, 0], arcList.shape[0], 1), 1))
vec = [areaXY, areaYZ, areaZX]
#[_, planeIDs] = np.max(vec, 1); # 1:XY 2:YZ 3:ZX
planeIDs = np.argmax(vec, 0)
for i in np.arange(numLine):
ori_lines[i, 0:3] = arcList[i, 0:3]
ori_lines[i, 3] = planeIDs[i]
coord1 = arcList[i, 3:6]
coord2 = arcList[i, 6:9]
uv = CoordsTransform.xyz2uvN(np.row_stack((coord1, coord2)),
planeIDs[i])
umax = np.max(uv[0, :]) + np.pi
umin = np.min(uv[0, :]) + np.pi
if umax - umin > np.pi:
ori_lines[i, 4:6] = np.column_stack((umax, umin)) / 2 / np.pi
# ori_lines(i,7) = umin + 1 - umax;
else:
ori_lines[i, 4:6] = np.column_stack((umin, umax)) / 2 / np.pi
# ori_lines(i,7) = umax - umin;
ori_lines[i, 6] = np.arccos(
np.sum(coord1 * coord2) /
(np.linalg.norm(coord1, 2) * np.linalg.norm(coord2, 2)))
ori_lines[i, 7] = arcList[i, 9]
# valid = ori_lines(:,3)<0;
# ori_lines(valid,1:3) = -ori_lines(valid,1:3);
## additive combination
lines = ori_lines
# panoEdge = paintParameterLine( lines, 1024, 512);
# figure; imshow(panoEdge);
for iter in np.arange(3):
numLine = lines.shape[0]
valid_line = np.ones([numLine], dtype=bool)
for i in np.arange(numLine):
# fprintf('#d/#d\n', i, numLine);
if valid_line[i] == False:
continue
dotProd = np.sum(lines[:, 0:3] * repmat(lines[i, 0:3], numLine, 1),
1)
valid_curr = (np.abs(dotProd) > np.cos(
1 * np.pi / 180)) & valid_line
valid_curr[i] = False
valid_ang = np.where(valid_curr)
for j in valid_ang[0]:
range1 = lines[i, 4:6]
range2 = lines[j, 4:6]
valid_rag = intersection(range1, range2)
if valid_rag == False:
continue
# combine
I = np.argmax(np.abs(lines[i, 0:3]))
if lines[i, I] * lines[j, I] > 0:
nc = lines[i, 0:3] * lines[i, 6] + lines[j, 0:3] * lines[j,
6]
else:
nc = lines[i, 0:3] * lines[i, 6] - lines[j, 0:3] * lines[j,
6]
nc = nc / np.linalg.norm(nc, 2)
if insideRange(range1[0], range2):
nrmin = range2[0]
else:
nrmin = range1[0]
if insideRange(range1[1], range2):
nrmax = range2[1]
else:
nrmax = range1[1]
u = np.array([nrmin, nrmax]) * 2 * np.pi - np.pi
v = CoordsTransform.computeUVN(nc, u, lines[i, 3])
xyz = CoordsTransform.uv2xyzN(np.column_stack((u, v)),
lines[i, 3])
length = np.arccos(np.sum(xyz[0, :] * xyz[1, :]))
scr = (lines[i, 6] * lines[i, 7] +
lines[j, 6] * lines[j, 7]) / (lines[i, 6] + lines[j, 6])
nc = np.append(nc, lines[i, 3])
nc = np.append(nc, nrmin)
nc = np.append(nc, nrmax)
nc = np.append(nc, length)
nc = np.append(nc, scr)
newLine = nc
lines[i, :] = newLine
valid_line[j] = False
lines = lines[valid_line, :]
print('iter: #d, before: #d, after: #d\n', iter, len(valid_line),
sum(valid_line))
return [lines, ori_lines]
'''
def attack(self,
target,
n_perturbations=None,
direct_attack=True,
structure_attack=True,
feature_attack=False,
n_influencers=5,
ll_constraint=True,
ll_cutoff=0.004,
disable=False):
super().attack(target, n_perturbations, direct_attack,
structure_attack, feature_attack)
if feature_attack and not is_binary(self.x):
raise RuntimeError(
"Attacks on the node features are currently only supported for binary attributes."
)
if ll_constraint and self.allow_singleton:
raise RuntimeError(
'`ll_constraint` is failed when `allow_singleton=True`, please set `attacker.allow_singleton=False`.'
)
logits_start = self.compute_logits()
best_wrong_class = self.strongest_wrong_class(logits_start)
if structure_attack and ll_constraint:
# Setup starting values of the likelihood ratio test.
degree_sequence_start = self.degree
current_degree_sequence = self.degree.astype('float64')
d_min = 2
S_d_start = np.sum(
np.log(degree_sequence_start[degree_sequence_start >= d_min]))
current_S_d = np.sum(
np.log(
current_degree_sequence[current_degree_sequence >= d_min]))
n_start = np.sum(degree_sequence_start >= d_min)
current_n = np.sum(current_degree_sequence >= d_min)
alpha_start = compute_alpha(n_start, S_d_start, d_min)
log_likelihood_orig = compute_log_likelihood(
n_start, alpha_start, S_d_start, d_min)
if len(self.influence_nodes) == 0:
if not direct_attack:
# Choose influencer nodes
infls, add_infls = self.get_attacker_nodes(
n_influencers, add_additional_nodes=True)
self.influence_nodes = np.concatenate(
(infls, add_infls)).astype("int")
# Potential edges are all edges from any attacker to any other node, except the respective
# attacker itself or the node being attacked.
self.potential_edges = np.row_stack([
np.column_stack(
(np.tile(infl, self.n_nodes - 2),
np.setdiff1d(np.arange(self.n_nodes),
np.array([self.target, infl]))))
for infl in self.influence_nodes
])
else:
# direct attack
influencers = [self.target]
self.potential_edges = np.column_stack(
(np.tile(self.target, self.n_nodes - 1),
np.setdiff1d(np.arange(self.n_nodes), self.target)))
self.influence_nodes = np.array(influencers)
self.potential_edges = self.potential_edges.astype("int32")
for _ in tqdm(range(self.n_perturbations),
desc='Peturbing Graph',
disable=disable):
if structure_attack:
# Do not consider edges that, if removed, result in singleton edges in the graph.
if not self.allow_singleton:
filtered_edges = filter_singletons(self.potential_edges,
self.modified_adj)
else:
filtered_edges = self.potential_edges
if ll_constraint:
# Update the values for the power law likelihood ratio test.
deltas = 2 * (1 - self.modified_adj[tuple(
filtered_edges.T)].A.ravel()) - 1
d_edges_old = current_degree_sequence[filtered_edges]
d_edges_new = current_degree_sequence[
filtered_edges] + deltas[:, None]
new_S_d, new_n = update_Sx(current_S_d, current_n,
d_edges_old, d_edges_new, d_min)
new_alphas = compute_alpha(new_n, new_S_d, d_min)
new_ll = compute_log_likelihood(new_n, new_alphas, new_S_d,
d_min)
alphas_combined = compute_alpha(new_n + n_start,
new_S_d + S_d_start, d_min)
new_ll_combined = compute_log_likelihood(
new_n + n_start, alphas_combined, new_S_d + S_d_start,
d_min)
new_ratios = -2 * new_ll_combined + 2 * (
new_ll + log_likelihood_orig)
# Do not consider edges that, if added/removed, would lead to a violation of the
# likelihood ration Chi_square cutoff value.
powerlaw_filter = filter_chisquare(new_ratios, ll_cutoff)
filtered_edges = filtered_edges[powerlaw_filter]
# Compute new entries in A_hat_square_uv
a_hat_uv_new = self.compute_new_a_hat_uv(filtered_edges)
# Compute the struct scores for each potential edge
struct_scores = self.struct_score(a_hat_uv_new,
self.compute_XW())
best_edge_ix = struct_scores.argmin()
best_edge_score = struct_scores.min()
best_edge = filtered_edges[best_edge_ix]
if feature_attack:
# Compute the feature scores for each potential feature perturbation
feature_ixs, feature_scores = self.feature_scores()
best_feature_ix = feature_ixs[0]
best_feature_score = feature_scores[0]
if structure_attack and feature_attack:
# decide whether to choose an edge or feature to change
if best_edge_score < best_feature_score:
change_structure = True
else:
change_structure = False
elif structure_attack:
change_structure = True
elif feature_attack:
change_structure = False
if change_structure:
# perform edge perturbation
u, v = best_edge
modified_adj = self.modified_adj.tolil(copy=False)
modified_adj[(u, v)] = modified_adj[(
v, u)] = 1 - modified_adj[(u, v)]
self.modified_adj = modified_adj.tocsr(copy=False)
self.adj_norm = normalize_adj(modified_adj)
self.structure_flips.append((u, v))
if ll_constraint:
# Update likelihood ratio test values
current_S_d = new_S_d[powerlaw_filter][best_edge_ix]
current_n = new_n[powerlaw_filter][best_edge_ix]
current_degree_sequence[best_edge] += deltas[
powerlaw_filter][best_edge_ix]
else:
modified_x = self.modified_x.tolil(copy=False)
modified_x[tuple(
best_feature_ix)] = 1 - modified_x[tuple(best_feature_ix)]
self.modified_x = modified_x.tocsr(copy=False)
self.feature_flips.append(tuple(best_feature_ix))
all_landmarks[i, :] = (all_landmarks[i, :] - row_min) / (row_max - row_min)
# Convert the numerical traits to be binary classes
for i in range(491):
row_mean = np.mean(all_traits_value[i, :])
all_traits_value[i, :] = all_traits_value[i, :] - row_mean
images = [cv2.imread(file) for file in\
glob.glob('*/data/img/*.jpg')]
# Calculate the Hog features for all 491 images and Combine the hogs features with landmarks
Hogs,hogImage= feature.hog(images[0], orientations=8, pixels_per_cell=(32, 32),cells_per_block=(2, 2),\
transform_sqrt=True, block_norm="L1",visualise=True)
for im in images:
h,_=feature.hog(im, orientations=8, pixels_per_cell=(32, 32),cells_per_block=(2, 2),\
transform_sqrt=True, block_norm="L1",visualise=True)
Hogs = np.row_stack((Hogs, h))
Hogs = Hogs[1:492, :]
hog_dat = np.column_stack((all_landmarks, Hogs)) # 491x6432
# Min-Max Normalization
def min_max(dat):
for i in range(dat.shape[0]):
row_min = min(dat[i, :])
row_max = max(dat[i, :])
dat[i, :] = (dat[i, :] - row_min) / (row_max - row_min)
return (dat)
# Function to extract Hog Features
def extract_hog_features(path):
ax1.scatter(umur, ketebalan[i], s=400) # scatter plot
# posisi label sumbu primer
ax1.set_xlabel("umur (dalam juta tahun)", fontsize=20)
ax1.xaxis.set_label_position('top')
ax1.set_ylabel("kedalaman", fontsize=20)
ax1.set_ylim(bottom=0, top=maximum)
ax1.invert_yaxis()
ax1.tick_params(axis="x", labelsize=20)
ax1.tick_params(axis="y", labelsize=20)
ax2.tick_params(axis="y", labelsize=20)
np.savetxt("csv/file_name.csv",
np.row_stack((umur, ketebalan)),
delimiter=",",
fmt='%s')
np.savetxt("csv/temperature.csv",
np.row_stack((umur, temperature)),
delimiter=",",
fmt='%s')
# np.savetxt("csv/tt_index.csv", TTI, delimiter=",", fmt='%s')
plt.title("Lopatin Burial History", fontsize=20)
ax1.legend(bbox_to_anchor=(1, 0), ncol=4, prop={'size':
20}) # menampilkan legenda
fig.tight_layout()
plt.xlim(min(umur), max(umur))
ax1.invert_xaxis()
# # img.axs[1][1].errorbar(g2, new_pdf_result2[i*sub_row], new_pdf_result2[i*sub_row + 1])
#
# img.axs[0][0].errorbar(g1, pdf_result[0], pdf_result[1], fmt=" ", capsize=img.cap_size, label="g1 ori_PDF", marker="v")
# img.axs[0][1].errorbar(g2, pdf_result[3], pdf_result[4], fmt=" ", capsize=img.cap_size, label="g2 ori_PDF", marker="v")
# img.axs[0][0].legend()
# img.axs[0][1].legend()
# img.show_img()
img = Image_Plot(cap_size=4, xpad=0.2, ypad=0.2)
img.subplots(1, 2)
shear_row_idx = [i*sub_row for i in range(iters)]
shear_sig_row_idx = [i*sub_row+1 for i in range(iters)]
print(shear_row_idx)
new_pdf_g1 = numpy.row_stack((pdf_result[0], new_pdf_result1[shear_row_idx]))
new_pdf_g1_sig = numpy.row_stack((pdf_result[1], new_pdf_result1[shear_sig_row_idx]))
new_pdf_g2 = numpy.row_stack((pdf_result[3], new_pdf_result2[shear_row_idx]))
new_pdf_g2_sig = numpy.row_stack((pdf_result[4], new_pdf_result2[shear_sig_row_idx]))
# print(new_pdf_g1)
# print(new_pdf_result1[shear_row_idx])
for i in range(shear_num):
pass
# img.axs[0][0].errorbar(range(iters+1), new_pdf_g1[:,i]-g1[i], new_pdf_g1_sig[:,i],
# capsize=img.cap_size, label="$\delta$ g1-%d"%i, marker="s")
# img.axs[0][1].errorbar(range(iters+1), new_pdf_g2[:,i]-g2[i], new_pdf_g1_sig[:,i],
# capsize=img.cap_size, label="$\delta$g2-%d"%i, marker="s")
for i in range(iters+1):
img.axs[0][0].errorbar(g1, new_pdf_g1[i]-g1, new_pdf_g1_sig[i],
capsize=img.cap_size, label="$\delta$ g1 iter-%d"%i, fmt=" ",marker="s",mfc="none")
def stackplot(axes,
x,
*args,
labels=(),
colors=None,
baseline='zero',
**kwargs):
"""
Draw a stacked area plot.
Parameters
----------
x : 1d array of dimension N
y : 2d array (dimension MxN), or sequence of 1d arrays (each dimension 1xN)
The data is assumed to be unstacked. Each of the following
calls is legal::
stackplot(x, y) # where y is MxN
stackplot(x, y1, y2, y3, y4) # where y1, y2, y3, y4, are all 1xNm
baseline : {'zero', 'sym', 'wiggle', 'weighted_wiggle'}
Method used to calculate the baseline:
- ``'zero'``: Constant zero baseline, i.e. a simple stacked plot.
- ``'sym'``: Symmetric around zero and is sometimes called
'ThemeRiver'.
- ``'wiggle'``: Minimizes the sum of the squared slopes.
- ``'weighted_wiggle'``: Does the same but weights to account for
size of each layer. It is also called 'Streamgraph'-layout. More
details can be found at http://leebyron.com/streamgraph/.
labels : Length N sequence of strings
Labels to assign to each data series.
colors : Length N sequence of colors
A list or tuple of colors. These will be cycled through and used to
colour the stacked areas.
**kwargs
All other keyword arguments are passed to `Axes.fill_between()`.
Returns
-------
list : list of `.PolyCollection`
A list of `.PolyCollection` instances, one for each element in the
stacked area plot.
"""
y = np.row_stack(args)
labels = iter(labels)
if colors is not None:
axes.set_prop_cycle(color=colors)
# Assume data passed has not been 'stacked', so stack it here.
# We'll need a float buffer for the upcoming calculations.
stack = np.cumsum(y, axis=0, dtype=np.promote_types(y.dtype, np.float32))
if baseline == 'zero':
first_line = 0.
elif baseline == 'sym':
first_line = -np.sum(y, 0) * 0.5
stack += first_line[None, :]
elif baseline == 'wiggle':
m = y.shape[0]
first_line = (y * (m - 0.5 - np.arange(m)[:, None])).sum(0)
first_line /= -m
stack += first_line
elif baseline == 'weighted_wiggle':
total = np.sum(y, 0)
# multiply by 1/total (or zero) to avoid infinities in the division:
inv_total = np.zeros_like(total)
mask = total > 0
inv_total[mask] = 1.0 / total[mask]
increase = np.hstack((y[:, 0:1], np.diff(y)))
below_size = total - stack
below_size += 0.5 * y
move_up = below_size * inv_total
move_up[:, 0] = 0.5
center = (move_up - 0.5) * increase
center = np.cumsum(center.sum(0))
first_line = center - 0.5 * total
stack += first_line
else:
errstr = "Baseline method %s not recognised. " % baseline
errstr += "Expected 'zero', 'sym', 'wiggle' or 'weighted_wiggle'"
raise ValueError(errstr)
# Color between x = 0 and the first array.
color = axes._get_lines.get_next_color()
coll = axes.fill_between(x,
first_line,
stack[0, :],
facecolor=color,
label=next(labels, None),
**kwargs)
coll.sticky_edges.y[:] = [0]
r = [coll]
# Color between array i-1 and array i
for i in range(len(y) - 1):
color = axes._get_lines.get_next_color()
r.append(
axes.fill_between(x,
stack[i, :],
stack[i + 1, :],
facecolor=color,
label=next(labels, None),
**kwargs))
return r
self.sum_data = sum_data
self.sum_all = sum_all
return delta
def predict(self, X):
delta = self.calc_delta(X)
return np.argmax(self.delta, axis=1)
#Test on synthetic data
alpha1 = 10, 1
points1 = np.random.dirichlet(alpha1, size=(100))
alpha2 = 1, 10
points2 = np.random.dirichlet(alpha2, size=(100))
alpha3 = 30, 30
points3 = np.random.dirichlet(alpha3, size=(100))
x = np.row_stack((points1, points2, points3))
#Fit Model
dmm = DirichletMixture(n_clusters=3, max_iter=100, threshold=0.001)
dmm.fit(x)
cluster = dmm.predict(x)
plt.figure()
plt.scatter(x[:, 0], x[:, 1], c=cluster, cmap='plasma')
plt.show()
def train(model_name, fold, run=None, resume_epoch=-1):
global optimizer, scheduler
model_str = build_model_str(model_name, fold, run)
model_info = MODELS[model_name]
checkpoints_dir = f'{BaseConfig.checkpoints_dir}/{model_str}'
tensorboard_dir = f'{BaseConfig.tensorboard_dir}/{model_str}'
oof_dir = f'{BaseConfig.oof_dir}/{model_str}'
os.makedirs(checkpoints_dir, exist_ok=True)
os.makedirs(tensorboard_dir, exist_ok=True)
os.makedirs(oof_dir, exist_ok=True)
print('\n', model_name, '\n')
logger = SummaryWriter(log_dir=tensorboard_dir)
model = model_info.factory(**model_info.args)
model = model.cuda()
# try:
# torchsummary.summary(model, (8, 400, 400))
# print('\n', model_name, '\n')
# except:
# raise
# pass
model = torch.nn.DataParallel(model).cuda()
model = model.cuda()
dataset_train = dataset_3d_v2.IntracranialDataset(
csv_file='5fold-rev3.csv',
folds=[f for f in range(BaseConfig.nb_folds) if f != fold],
random_slice=True,
preprocess_func=albumentations.Compose([
albumentations.ShiftScaleRotate(shift_limit=16. / 256,
scale_limit=0.05,
rotate_limit=30,
interpolation=cv2.INTER_LINEAR,
border_mode=cv2.BORDER_REPLICATE,
p=0.75),
albumentations.Flip(),
albumentations.RandomRotate90(),
albumentations.pytorch.ToTensorV2()
]),
**model_info.dataset_args)
dataset_valid = dataset_3d_v2.IntracranialDataset(
csv_file='5fold.csv',
folds=[fold],
random_slice=False,
return_all_slices=True,
preprocess_func=albumentations.pytorch.ToTensorV2(),
**model_info.dataset_args)
model.train()
if model_info.optimiser == 'radam':
optimizer = radam.RAdam(model.parameters(), lr=model_info.initial_lr)
elif model_info.optimiser == 'sgd':
optimizer = torch.optim.SGD(model.parameters(),
lr=model_info.initial_lr,
momentum=0.95,
nesterov=True)
elif model_info.optimiser == 'adabound':
optimizer = adabound.AdaBound(model.parameters(),
lr=model_info.initial_lr,
final_lr=0.1)
milestones = [32, 48, 64]
if model_info.optimiser_milestones:
milestones = model_info.optimiser_milestones
if model_info.scheduler == 'steps':
scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
milestones=milestones,
gamma=0.2)
elif model_info.scheduler == 'cos_restarts':
scheduler = CosineAnnealingLRWithRestarts(optimizer=optimizer,
T_max=8,
T_mult=1.2)
print(
f'Num training images: {len(dataset_train)} validation images: {len(dataset_valid)}'
)
if resume_epoch > -1:
checkpoint = torch.load(f'{checkpoints_dir}/{resume_epoch:03}.pt')
model.module.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
data_loaders = {
'train':
DataLoader(dataset_train,
num_workers=16,
shuffle=True,
drop_last=True,
batch_size=model_info.batch_size),
'val':
DataLoader(dataset_valid, shuffle=False, num_workers=4, batch_size=1)
}
class_weights = torch.tensor([1.0, 1.0, 1.0, 1.0, 1.0, 2.0]).cuda()
def criterium(y_pred, y_true):
y_pred = y_pred.reshape(-1, 6)
y_true = y_true.reshape(-1, 6)
cw = class_weights.repeat(y_pred.shape[0], 1)
return F.binary_cross_entropy_with_logits(y_pred, y_true, cw)
# fit new layers first:
if resume_epoch == -1 and model_info.is_pretrained:
model.train()
model.module.freeze_encoder()
data_loader = data_loaders['train']
pre_fit_steps = len(dataset_train) // model_info.batch_size // 8
data_iter = tqdm(enumerate(data_loader), total=pre_fit_steps)
epoch_loss = []
initial_optimizer = torch.optim.Adam(model.parameters(), lr=2e-5)
# initial_optimizer = torch.optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)
for iter_num, data in data_iter:
if iter_num > pre_fit_steps:
break
with torch.set_grad_enabled(True):
img = data['image'].float().cuda()
labels = data['labels'].float().cuda()
pred = model(img)
loss = criterium(pred, labels)
# loss.backward()
(loss / model_info.accumulation_steps).backward()
if (iter_num + 1) % model_info.accumulation_steps == 0:
torch.nn.utils.clip_grad_norm_(model.parameters(), 100.0)
initial_optimizer.step()
initial_optimizer.zero_grad()
epoch_loss.append(float(loss))
data_iter.set_description(
f'Loss: Running {np.mean(epoch_loss[-100:]):1.4f} Avg {np.mean(epoch_loss):1.4f}'
)
del initial_optimizer
model.module.unfreeze_encoder()
phase_period = {'train': 1, 'val': 2}
for epoch_num in range(resume_epoch + 1, 80):
for phase in ['train', 'val']:
if epoch_num % phase_period[phase] == 0:
model.train(phase == 'train')
epoch_loss = []
epoch_labels = []
epoch_predictions = []
epoch_sample_paths = []
if 'on_epoch' in model.module.__dir__():
model.module.on_epoch(epoch_num)
data_loader = data_loaders[phase]
data_iter = tqdm(enumerate(data_loader),
total=len(data_loader))
for iter_num, data in data_iter:
img = data['image'].float().cuda()
labels = data['labels'].float().cuda()
with torch.set_grad_enabled(phase == 'train'):
pred = model(img)
loss = criterium(pred, labels)
if phase == 'train':
(loss / model_info.accumulation_steps).backward()
if (iter_num +
1) % model_info.accumulation_steps == 0:
torch.nn.utils.clip_grad_norm_(
model.parameters(), 32.0)
optimizer.step()
optimizer.zero_grad()
epoch_loss.append(float(loss))
epoch_labels.append(
np.row_stack(labels.detach().cpu().numpy()))
epoch_predictions.append(
np.row_stack(
torch.sigmoid(pred).detach().cpu().numpy()))
# print(labels.shape, epoch_labels[-1].shape, pred.shape, epoch_predictions[-1].shape)
epoch_sample_paths += data['path']
data_iter.set_description(
f'{epoch_num} Loss: Running {np.mean(epoch_loss[-100:]):1.4f} Avg {np.mean(epoch_loss):1.4f}'
)
epoch_labels = np.row_stack(epoch_labels)
epoch_predictions = np.row_stack(epoch_predictions)
if phase == 'val':
# recalculate loss as depth dimension is variable
epoch_loss_mean = float(
F.binary_cross_entropy(
torch.from_numpy(epoch_predictions).cuda(),
torch.from_numpy(epoch_labels).cuda(),
class_weights.repeat(epoch_labels.shape[0], 1)))
print(epoch_loss_mean)
logger.add_scalar(f'loss_{phase}', epoch_loss_mean,
epoch_num)
else:
logger.add_scalar(f'loss_{phase}', np.mean(epoch_loss),
epoch_num)
logger.add_scalar('lr', optimizer.param_groups[0]['lr'],
epoch_num) # scheduler.get_lr()[0]
try:
log_metrics(logger=logger,
phase=phase,
epoch_num=epoch_num,
y=epoch_labels,
y_hat=epoch_predictions)
except Exception:
pass
if phase == 'val':
torch.save(
{
'epoch': epoch_num,
'sample_paths': epoch_sample_paths,
'epoch_labels': epoch_labels,
'epoch_predictions': epoch_predictions,
}, f'{oof_dir}/{epoch_num:03}.pt')
logger.flush()
if phase == 'val':
scheduler.step(epoch=epoch_num)
else:
# print(f'{checkpoints_dir}/{epoch_num:03}.pt')
torch.save(
{
'epoch': epoch_num,
'model_state_dict': model.module.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, f'{checkpoints_dir}/{epoch_num:03}.pt')
def calc_detector_projection(self):
h_mat = self.predict_miller_indices(
) # get predicted set of miller indices
h_mat = np.array(h_mat)
cell_mat = np.array([
self.xparamdict['a_axis'], self.xparamdict['b_axis'],
self.xparamdict['c_axis']
]).transpose()
rlp = np.linalg.inv(cell_mat) # (a*, b*, c*) matrix
# incident beam vector
s0 = np.array(self.xparamdict['incident_beam'])
beam_vector = s0 / np.linalg.norm(s0) / self.xparamdict['wavelength']
rot_vector = np.array(
self.xparamdict['gonio_coord']) # obtained from rotation axis
rot = np.empty((3, 3), dtype=np.float)
rot[:, 1] = rot_vector / np.linalg.norm(rot_vector)
rot[:, 0] = np.cross(rot[:, 1], beam_vector)
rot[:, 0] /= np.linalg.norm(rot[:, 0])
rot[:, 2] = np.cross(rot[:, 0], rot[:, 1])
# detector coordinate system
d_mat = np.array([
self.xparamdict['det_axis1'], self.xparamdict['det_axis2'],
self.xparamdict['det_axis3']
])
d_mat = d_mat.transpose()
# 1st predicted spot as vector on Ewald sphere
p0star = np.dot(h_mat, rlp)
p0star_gonio = np.dot(p0star,
rot) # same predicted spot on gonio coordinate
beam_gonio = np.dot(rot.transpose(),
beam_vector) # incident beam on gonio coordinate
print(beam_gonio.shape)
p0star_sq_dist = np.sum(
p0star**2,
axis=1) # sq distance of predicted spot from the rotation axis
pstar_gonio = np.empty(p0star_gonio.shape) # a 3x3 matrix
pstar_gonio[:,
2] = (-0.5 * p0star_sq_dist -
beam_gonio[1] * p0star_gonio[:, 1]) / beam_gonio[2]
pstar_gonio[:, 1] = p0star_gonio[:, 1]
pstar_gonio[:,
0] = p0star_sq_dist - pstar_gonio[:,
1]**2 - pstar_gonio[:,
2]**2 # sqrt taken at later stage
sel_index = pstar_gonio[:,
0] > 0 # exclude blind region with this criteria, filters the matrices
h_mat = h_mat[sel_index]
p0star_gonio = p0star_gonio[sel_index]
pstar_gonio = pstar_gonio[sel_index]
pstar_gonio[:, 0] = np.sqrt(pstar_gonio[:, 0])
projected_hkl = np.empty((0, 3), dtype=np.int) # h,k,l as row vector
position_on_detector = np.empty(
(0, 4),
dtype=np.float) # Find detector projection, x, y, phi, zeta
for sign in (+1, -1):
pstar_gonio[:, 0] *= sign
phi = np.arctan2((p0star_gonio[:, 2] * pstar_gonio[:, 0] -
p0star_gonio[:, 1] * pstar_gonio[:, 2]),
(p0star_gonio[:, 1] * pstar_gonio[:, 1] +
p0star_gonio[:, 2] * pstar_gonio[:, 2]))
refl_std = np.cross(pstar_gonio, beam_gonio)
refl_std /= np.linalg.norm(refl_std, axis=1).reshape(
refl_std.shape[0], 1) # convert into column vector
zeta = refl_std[:, 1]
Svector = beam_vector + np.dot(pstar_gonio, rot.transpose())
Svec_on_det = np.dot(Svector, d_mat)
sel_index = self.xparamdict['detZ'] * Svec_on_det[:, 2] > 0
Svec_on_det, h_mat, phi, zeta = Svec_on_det[sel_index], h_mat[
sel_index], phi[sel_index], zeta[sel_index]
det_pos_x = self.xparamdict['beamX'] + self.xparamdict['detZ']*Svec_on_det[:, 0] / Svec_on_det[:, 2] \
/ self.xparamdict['pixelsize']
det_pos_y = self.xparamdict['beamY'] + self.xparamdict['detZ'] * Svec_on_det[:, 1] / Svec_on_det[:, 2] \
/ self.xparamdict['pixelsize']
projected_hkl = np.row_stack([projected_hkl, h_mat])
position_on_detector = np.row_stack([
position_on_detector,
np.column_stack([det_pos_x, det_pos_y, phi, zeta])
])
return projected_hkl, position_on_detector
def ref_pts(Lico_right, Lico_left, Lico_bot, Lico_up, Lico_top, Lico_side):
eq_A_right = [-Lico_right[0], 1]
eq_A_left = [-Lico_left[0], 1]
eq_A_bot = [-Lico_bot[0], 1]
eq_A_up = [-Lico_up[0], 1]
eq_A_top = [-Lico_top[0], 1]
eq_A_side = [-Lico_side[0], 1]
eq_b_right = Lico_right[1]
eq_b_left = Lico_left[1]
eq_b_bot = Lico_bot[1]
eq_b_up = Lico_up[1]
eq_b_top = Lico_top[1]
eq_b_side = Lico_side[1]
mat_A_up_left = np.row_stack((eq_A_up, eq_A_left))
mat_b_up_left = np.row_stack((eq_b_up, eq_b_left))
mat_A_up_right = np.row_stack((eq_A_up, eq_A_right))
mat_b_up_right = np.row_stack((eq_b_up, eq_b_right))
mat_A_up_side = np.row_stack((eq_A_up, eq_A_side))
mat_b_up_side = np.row_stack((eq_b_up, eq_b_side))
mat_A_bot_left = np.row_stack((eq_A_bot, eq_A_left))
mat_b_bot_left = np.row_stack((eq_b_bot, eq_b_left))
mat_A_bot_right = np.row_stack((eq_A_bot, eq_A_right))
mat_b_bot_right = np.row_stack((eq_b_bot, eq_b_right))
mat_A_bot_side = np.row_stack((eq_A_bot, eq_A_side))
mat_b_bot_side = np.row_stack((eq_b_bot, eq_b_side))
mat_A_top_side = np.row_stack((eq_A_top, eq_A_side))
mat_b_top_side = np.row_stack((eq_b_top, eq_b_side))
mat_A_top_left = np.row_stack((eq_A_top, eq_A_left))
mat_b_top_left = np.row_stack((eq_b_top, eq_b_left))
mat_A_top_right = np.row_stack((eq_A_top, eq_A_right))
mat_b_top_right = np.row_stack((eq_b_top, eq_b_right))
up_left = np.linalg.solve(mat_A_up_left, mat_b_up_left)
up_right = np.linalg.solve(mat_A_up_right, mat_b_up_right)
up_side = np.linalg.solve(mat_A_up_side, mat_b_up_side)
bot_left = np.linalg.solve(mat_A_bot_left, mat_b_bot_left)
bot_right = np.linalg.solve(mat_A_bot_right, mat_b_bot_right)
bot_side = np.linalg.solve(mat_A_bot_side, mat_b_bot_side)
top_side = np.linalg.solve(mat_A_top_side, mat_b_top_side)
top_left = np.linalg.solve(mat_A_top_left, mat_b_top_left)
top_right = np.linalg.solve(mat_A_top_right, mat_b_top_right)
pt_up_left = (int(up_left[0]), int(up_left[1]))
pt_up_right = (int(up_right[0]), int(up_right[1]))
pt_up_side = (int(up_side[0]), int(up_side[1]))
pt_bot_left = (int(bot_left[0]), int(bot_left[1]))
pt_bot_right = (int(bot_right[0]), int(bot_right[1]))
pt_bot_side = (int(bot_side[0]), int(bot_side[1]))
pt_top_side = (int(top_side[0]), int(top_side[1]))
pt_top_left = (int(top_left[0]), int(top_left[1]))
pt_top_right = (int(top_right[0]), int(top_right[1]))
return pt_up_left, pt_up_right, pt_up_side, pt_bot_left, pt_bot_right, \
pt_bot_side, pt_top_side, pt_top_left, pt_top_right
def state_data_processing(v3_state, num_processing_steps):
while len(v3_state) < num_processing_steps + 1:
v3_state = np.row_stack((v3_state, v3_state[len(v3_state) - 1]))
return v3_state
def find_corners(img, pos_camera=None, mark_img=True, show_img=False, img_white_keys=None):
# Convert to HSV color space
img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
# Equalize V channel
img_hsv[:,:,2] = clahe.apply(img_hsv[:,:,2])
if show_img:
imshow(np.hstack((img_hsv[:,:,0], img_hsv[:,:,1], img_hsv[:,:,2])), 3)
# Threshold white keys
img_w = cv2.inRange(img_hsv, np.array([20, 0, 200]), np.array([160, 60, 255]))
img_h = cv2.inRange(img_hsv, np.array([20, 0, 0]), np.array([160, 255, 255]))
img_s = cv2.inRange(img_hsv, np.array([0, 0, 0]), np.array([255, 60, 255]))
img_v = cv2.inRange(img_hsv, np.array([0, 0, 200]), np.array([255, 255, 255]))
if show_img:
imshow(np.hstack((img_h, img_s, img_v)), 3)
# Fill white keys with watershed
img_w[:,:10] = 255
img_w[:,-10:] = 255
img_w[:200,:] = 255
img_w[-200:,:] = 255
cv2.floodFill(img_w, np.zeros((img_w.shape[0]+2, img_w.shape[1]+2), dtype=np.uint8), (0,0), 0)
if show_img:
imshow(img_w, 3)
img_fg = cv2.morphologyEx(img_w, cv2.MORPH_ERODE, np.ones((10, 10)))
img_bg = cv2.morphologyEx(img_w, cv2.MORPH_DILATE, np.ones((100, 100)))
img_bg[:,:10] = 0
img_bg[:,-10:] = 0
img_bg[:200,:] = 0
img_bg[-200:,:] = 0
img_bg = 255 - img_bg
markers = np.zeros(img_w.shape, dtype=np.int32)
markers[img_fg > 0] = 1
markers[img_bg > 0] = 2
markers = cv2.watershed(img, markers)
img_w = (255 * (markers == 1)).astype(np.uint8)
if show_img:
imshow(img_w, 3)
cv2.floodFill(img_w, np.zeros((img_w.shape[0]+2, img_w.shape[1]+2), dtype=np.uint8), (0,0), 0)
# Find orientation of largest connected component
_, contours, _ = cv2.findContours(img_w, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
areas = np.array([cv2.contourArea(c) for c in contours])
vx, vy, x, y = cv2.fitLine(contours[areas.argmax()], cv2.DIST_L2, 0, 0.01, 0.01)
V = np.vstack((vx, vy))
# Determine camera position from keyboard orientation
if pos_camera is None:
if V[0] * V[1] > 0:
pos_camera = "right"
else:
pos_camera = "left"
# Dilate image with keyboard-aligned kernel and extract largest connected component
theta = np.arctan2(V[0], V[1])
kernel = 255 * np.round(scipy.ndimage.rotate(np.ones((150, 2)), theta * 180/np.pi)).astype(np.uint8)
img_cc = cv2.morphologyEx(img_w, cv2.MORPH_DILATE, kernel)
_, contours, _ = cv2.findContours(img_cc, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
areas = np.array([cv2.contourArea(c) for c in contours])
contour = np.squeeze(contours[areas.argmax()], axis=1)
img_cc.fill(0)
cv2.drawContours(img_cc, contours, areas.argmax(), 255, -1)
img_w = 255 * np.logical_and(img_w>0, img_cc>0).astype(np.uint8)
if show_img:
imshow(img_cc, 3)
# Find combined contour of keyboard segmentations
_, contours, _ = cv2.findContours(img_w, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
img_w.fill(0)
if mark_img:
for i in range(len(contours)):
cv2.drawContours(img, contours, i, (0,0,255), 3)
for i in range(len(contours)):
cv2.drawContours(img_w, contours, i, 255, -1)
areas = np.array([cv2.contourArea(c) for c in contours])
contours = [np.squeeze(contour, axis=1) for contour in contours]
contour = np.vstack(contours)
contour = np.column_stack((contour, np.ones(contour.shape[0])))
if show_img:
imshow(img_w, 3)
# Find corners
corner_left = contour[contour[:,0].argmin()].astype(np.int32)
corner_right = contour[contour[:,0].argmax()].astype(np.int32)
corner_bottom = contour[contour.shape[0]-1-contour[:,1][::-1].argmax()].astype(np.int32)
corner_top = contour[contour[:,1].argmin()].astype(np.int32)
# Push up bottom corner to the key's surface
num_white = 0
for i in range(40):
vec_w = img_w[corner_bottom[1]-i,corner_bottom[0]-20:corner_bottom[0]+20]>0
num_white_next = vec_w.sum()
if num_white_next - num_white > 3:
corner_bottom[1] -= i - 1
break
num_white = num_white_next
# Refine corner with subpixel search
corners = np.row_stack((corner_left[:2], corner_right[:2], corner_bottom[:2], corner_top[:2]))
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 100, 0.001)
corners = cv2.cornerSubPix(img_v,np.float32(corners.astype(np.float64)),(10,10),(-1,-1),criteria)
corner_left = np.round(np.append(corners[0], 1)).astype(np.int32)
corner_right = np.round(np.append(corners[1], 1)).astype(np.int32)
corner_bottom = np.round(np.append(corners[2], 1)).astype(np.int32)
corner_top = np.round(np.append(corners[3], 1)).astype(np.int32)
# Determine front and back sides of the keyboard
if pos_camera == "right":
corner_back = corner_right
corner_front = corner_left
contour_back = img_w.shape[1] - 1 - np.argmax(img_w[corner_top[1]:corner_back[1]+1,::-1]>0, axis=1)
else:
corner_back = corner_left
corner_front = corner_right
contour_back = np.argmax(img_w[corner_top[1]:corner_back[1]+1]>0, axis=1)
# Find back contour
idx = np.logical_and(contour_back<img_w.shape[1]-1, contour_back>0)
contour_back = np.column_stack((contour_back, \
np.arange(corner_top[1], corner_back[1]+1),
np.ones((contour_back.shape[0],), dtype=np.int32)))
contour_back = contour_back[idx]
contour_back_origin = contour_back - corner_back
cv2.polylines(img, np.int32([contour_back[:,:2]]), False, (0,255,255), 5)
corner_top = contour_back[contour_back[:,1].argmin()]
# Rotate line from vertical position until it hits the back contour
num_hit = 0
if pos_camera == "right":
sign_theta = 1
else:
sign_theta = -1
for theta in np.linspace(0,np.pi/2,90):
line_back = [sign_theta * np.cos(theta), -np.sin(theta), 0]
num_hit_new = np.sum(np.dot(contour_back_origin, line_back)>0)
# Stop when the gradient of hit pixels spikes
if num_hit_new - num_hit > contour_back.shape[0] / 30 and theta > 0:
break
num_hit = num_hit_new
line_back[-1] = -np.dot(corner_back, line_back)
# Update contour to include only points close to the line
contour_back = contour_back[np.abs(np.dot(contour_back, line_back))<10,:]
if mark_img:
cv2.polylines(img, np.int32([contour_back[:,:2]]), False, (0,255,0), 5)
dir_line_back = np.array([line_back[1], -line_back[0]])
points_line_back = np.array([2000, -2000])[:,np.newaxis] * dir_line_back[np.newaxis,:] + corner_back[np.newaxis,:2]
cv2.line(img, tuple(points_line_back[0].astype(np.int32)), tuple(points_line_back[1].astype(np.int32)), (255,0,255), 5)
# Fit least squares line to back contour
# U, S, VT = np.linalg.svd(contour_back[:,:2] - corner_back[:2])
U, S, VT = np.linalg.svd(contour_back[:,:2] - contour_back[:,:2].mean(axis=0))
line_back = np.append(VT[-1], 0)
line_back[-1] = -np.dot(corner_back, line_back)
if mark_img:
dir_line_back = np.array([line_back[1], -line_back[0]])
points_line_back = np.array([2000, -2000])[:,np.newaxis] * dir_line_back[np.newaxis,:] + corner_back[np.newaxis,:2]
cv2.line(img, tuple(points_line_back[0].astype(np.int32)), tuple(points_line_back[1].astype(np.int32)), (255,0,0), 5)
# Find intersection between back and top lines
corner_top_mid = corner_top
line_top = np.cross(corner_top, corner_front).astype(np.float32)
line_top /= np.linalg.norm(line_top)
corner_top = np.cross(line_back, line_top)
corner_top /= corner_top[-1]
corner_top = np.round(corner_top).astype(np.int32)
# Plot corners
if mark_img:
cv2.circle(img, tuple(corner_top[:2]), 10, (0,255,0), 3)
cv2.circle(img, tuple(corner_right[:2]), 10, (255,255,0), 3)
cv2.circle(img, tuple(corner_bottom[:2]), 10, (255,0,0), 3)
cv2.circle(img, tuple(corner_left[:2]), 10, (255,0,255), 3)
cv2.circle(img, tuple(corner_top_mid[:2]), 10, (255,0,255), 3)
# Collect corners
if pos_camera == "left":
corners = np.row_stack((corner_left, corner_top, corner_right, corner_bottom))
else:
corners = np.row_stack((corner_top, corner_right, corner_bottom, corner_left))
if img_white_keys is not None:
img_white_keys[:,:] = img_w
return corners, pos_camera
x = np.arange(0, len(dis_plt))
plt.figure(figsize=(12, 6))
plt.plot(x, dis_plt, 'g--')
plt.xlabel('步数')
plt.ylabel("路程")
plt.title("路程迭代曲线")
plt.savefig("路程迭代曲线.pdf")
plt.show()
# 画出最优路线
location_plt = location.copy()
for i in range(0, 30):
location_plt[i][0] = location[path[i]][0]
location_plt[i][1] = location[path[i]][1]
row = np.array([location[path[0]][0], location[path[0]][1]])
location_plt = np.row_stack((location_plt, row))
x = location_plt[:, 0]
y = location_plt[:, 1]
plt.figure(figsize=(8, 8))
plt.plot(x, y, 'r-o')
plt.xlabel('经度')
plt.ylabel("纬度")
plt.title("河北路线图")
m = location[:, 0] # 新建画图坐标(m,n)
n = location[:, 1]
name = []
name_file = open("河北城市名称.txt", 'r', encoding='UTF-8')
while True:
city_name = name_file.readline()
name.append(city_name)
if not city_name:
Prot = 0
Jp1 = PTrans(Px, Pz1, Prot)[0:2, :]
Jp2 = PTrans(Px, Pz2, Prot)[0:2, :]
Jp = np.vstack((Jp1, Jp2, Jp2, Jp1))
# Substep b: Calculate pusher wrench
# See figure in Table 1 for reference
FP1 = np.array([Fn, -Ft])
FP2 = np.array([Fn, Ft])
Fp = np.hstack((FP1, FP1, FP2, FP2))
W1 = Jp[0:2, :].T.dot(Fp[0:2])
W2 = Jp[2:4, :].T.dot(Fp[2:4])
W3 = Jp[4:6, :].T.dot(Fp[4:6])
W4 = Jp[6:, :].T.dot(Fp[6:])
W = np.row_stack((W1, W2, W3, W4))
# STEP 3: CALCULATE OBJECT VELOCITY
# Calculate Js
Sx = .1
Sz = 0
Srot = 0
Js = PTrans(
Sx, Sz, Srot
) # Jacobian that maps v_obj (in object frame) to v_s (in support frame)
# Calculate gravity wrench
w_gravity = np.array([0, m * g, 0])
# Define B, wrench on a unit limit surface
B = np.diag((1, 1, k**(-2)))
how='left')
print "make all train dataset ........."
print "make train_noWeekend dataset .........."
#train = pd.concat([final_train1, final_train2, final_train3, final_train4], axis=0, ignore_index=True)
train1_noWeekend_matrix = final_train1_noWeekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train2_noWeekend_matrix = final_train2_noWeekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train3_noWeekend_matrix = final_train3_noWeekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train4_noWeekend_matrix = final_train4_noWeekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
#final_train_matrix = train.drop(['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
final_train_noWeekend_matrix = np.row_stack(
(train1_noWeekend_matrix, train2_noWeekend_matrix, train3_noWeekend_matrix,
train4_noWeekend_matrix))
train_noWeekend_X = final_train_noWeekend_matrix[:, :-1]
train_noWeekend_Y = final_train_noWeekend_matrix[:, -1]
print "make train_Weekend dataset .........."
#train = pd.concat([final_train1, final_train2, final_train3, final_train4], axis=0, ignore_index=True)
train1_Weekend_matrix = final_train1_Weekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train2_Weekend_matrix = final_train2_Weekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train3_Weekend_matrix = final_train3_Weekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
train4_Weekend_matrix = final_train4_Weekend.drop(
['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
#final_train_matrix = train.drop(['user_id', 'day_of_week', 'record_date'], axis=1).as_matrix()
final_train_Weekend_matrix = np.row_stack(
future_reward = sess.run(output, feed_dict={x: temp_Map})
reward = reward + np.amax(future_reward) * future_Discount
Q_value = np.append(Q_value, reward)
temp_Map = train_data_element[0:n][0:m]
temp_Map = np.array(temp_Map)
temp_Map[curr_Pos[0], curr_Pos[1]] = 2
#Q_temp = sess.run(output, feed_dict={x:temp_Map})
#Q_temp = Q_temp[np.array([ACTIONS.index(curr_action)])]
#Q_value_curr_step = np.append([Q_value_curr_step], [Q_temp])
temp_Map = np.reshape(temp_Map, [1, n, m, 1])
train_map = np.append(train_map, temp_Map, axis=0)
temp = np.zeros([1, 4])
temp[0, ACTIONS.index(curr_action)] = 1
action_data = np.row_stack((action_data, temp))
_, tra_loss = sess.run([train_optimizer, train_loss],
feed_dict={
x: train_map,
action: action_data,
y: Q_value
})
#print(tra_loss)
if not is_Terminal:
curr_Map[next_Pos[0], next_Pos[1]] = 0
curr_Pos = np.copy(next_Pos)
temp_ob = sess.run(output, feed_dict={x: ob})
print(temp_ob)
import scipy
import numpy as np
import itertools
import matplotlib.pyplot as plt
# TODO: Run this cell to generate the data
num_samples = 400
cov = np.array([[1., .7], [.7, 1.]]) * 10
mean_1 = [.1, .1]
mean_2 = [6., .1]
x_class1 = np.random.multivariate_normal(mean_1, cov, num_samples // 2)
x_class2 = np.random.multivariate_normal(mean_2, cov, num_samples // 2)
xy_class1 = np.column_stack((x_class1, np.zeros(num_samples // 2)))
xy_class2 = np.column_stack((x_class2, np.ones(num_samples // 2)))
data_full = np.row_stack([xy_class1, xy_class2])
np.random.shuffle(data_full)
data = data_full[:, :2]
labels = data_full[:, 2]
# TODO: Make a scatterplot for the data points showing the true cluster assignments of each point
plt.scatter(x_class1[:, 0], x_class1[:, 1], marker="x") # first class, x shape
plt.scatter(x_class2[:, 0], x_class2[:, 1],
marker="o") # second class, circle shape
plt.show()
def cost(data, R, Mu):
N, D = data.shape
K = Mu.shape[1]
J = 0
def surf(*args, **kwargs):
""" surf(..., axesAdjust=True, axes=None)
Shaded surface plot.
Usage
-----
* surf(Z) - create a surface using the given image with z coordinates.
* surf(Z, C) - also supply a texture image to map.
* surf(X, Y, Z) - give x, y and z coordinates.
* surf(X, Y, Z, C) - also supply a texture image to map.
Parameters
----------
Z : A MxN 2D array
X : A length N 1D array, or a MxN 2D array
Y : A length M 1D array, or a MxN 2D array
C : A MxN 2D array, or a AxBx3 3D array
If 2D, C specifies a colormap index for each vertex of Z. If
3D, C gives a RGB image to be mapped over Z. In this case, the
sizes of C and Z need not match.
Keyword arguments
-----------------
axesAdjust : bool
If axesAdjust==True, this function will call axes.SetLimits(), and set
the camera type to 3D. If daspectAuto has not been set yet,
it is set to False.
axes : Axes instance
Display the image in this axes, or the current axes if not given.
Also see grid()
"""
def checkZ(z):
if z.ndim != 2:
raise ValueError('Z must be a 2D array.')
# Parse input
if len(args) == 1:
z = np.asanyarray(args[0])
checkZ(z)
y = np.arange(z.shape[0])
x = np.arange(z.shape[1])
c = None
elif len(args) == 2:
z, c = map(np.asanyarray, args)
checkZ(z)
y = np.arange(z.shape[0])
x = np.arange(z.shape[1])
elif len(args) == 3:
x, y, z = map(np.asanyarray, args)
checkZ(z)
c = None
elif len(args) == 4:
x, y, z, c = map(np.asanyarray, args)
checkZ(z)
else:
raise ValueError(
'Invalid number of arguments. Must pass 1-4 arguments.')
# Parse kwargs
axes = None
if 'axes' in kwargs:
axes = kwargs['axes']
axesAdjust = True
if 'axesAdjust' in kwargs:
axesAdjust = kwargs['axesAdjust']
# Set y vertices
if y.shape == (z.shape[0], ):
y = y.reshape(z.shape[0], 1).repeat(z.shape[1], axis=1)
elif y.shape != z.shape:
raise ValueError(
'Y must have same shape as Z, or be 1D with length of rows of Z.')
# Set x vertices
if x.shape == (z.shape[1], ):
x = x.reshape(1, z.shape[1]).repeat(z.shape[0], axis=0)
elif x.shape != z.shape:
raise ValueError(
'X must have same shape as Z, or be 1D with length of columns of Z.'
)
# Set vertices
vertices = np.column_stack((x.ravel(), y.ravel(), z.ravel()))
# Create texcoords
if c is None or c.shape == z.shape:
# No texture -> colormap on the z value
# Grayscale texture -> color mapping
texcoords = (c if c is not None else z).ravel()
elif c.ndim == 3:
# color texture -> use texture mapping
U, V = np.meshgrid(np.linspace(0, 1, z.shape[1]),
np.linspace(0, 1, z.shape[0]))
texcoords = np.column_stack((U.ravel(), V.ravel()))
else:
raise ValueError('C must have same shape as Z, or be 3D array.')
# Create faces
w = z.shape[1]
i = np.arange(z.shape[0] - 1)
faces = np.row_stack(
np.column_stack((j + w * i, j + 1 + w * i, j + 1 + w * (i + 1),
j + w * (i + 1))) for j in range(w - 1))
## Visualize
# Get axes
if axes is None:
axes = vv.gca()
# Create mesh
m = vv.Mesh(axes, vertices, faces, values=texcoords, verticesPerFace=4)
# Should we apply a texture?
if c is not None and c.ndim == 3:
m.SetTexture(c)
else:
m.clim = m.clim # trigger correct limits
# Adjust axes
if axesAdjust:
if axes.daspectAuto is None:
axes.daspectAuto = False
axes.cameraType = '3d'
axes.SetLimits()
# Return
axes.Draw()
return m
import numpy as np from matplotlib import pyplot as plt fnx = lambda: np.random.randint(5, 50, 10) y = np.row_stack((fnx(), fnx(), fnx())) x = np.arange(10) y1, y2, y3 = fnx(), fnx(), fnx() fig, ax = plt.subplots() ax.stackplot(x, y) plt.show() fig, ax = plt.subplots() ax.stackplot(x, y1, y2, y3) plt.show()
def findSplitAndThresh(resp, Xsrc, Xtar, F2, splitevaltype, lamdda, minChild,
kappa):
F1 = resp.shape[0]
rerr = float("inf")
fid = 1
thr = float("inf")
Ft = Xtar.shape[0]
Fs = Xsrc.shape[0]
#special treatment for random tree growing
if splitevaltype == 'random':
F1 = 1
F2 = 1
for s in range(F1):
#get thresholds to evaluate
if F2 == 0:
tthrs = np.median(resp[s, :], axis=0)
else:
respmin = min(resp[s, :])
respmax = max(resp[s, :])
tthrs = np.zeros((F2 + 1, 1), np.float32)
tthrs[0:-1] = np.random.rand(
5, 1) * 0.95 * (respmax - respmin) + respmin
tthrs[-1] = np.median(resp[s, :])
for t in range(len(tthrs)):
tthr = tthrs[t][0]
left = resp[s, :] < tthr
right = ~left
nl = len(np.nonzero(left)[0])
nr = len(np.nonzero(right)[0])
'''
nl=0
nr=0
for i in range(len(resp)):
if left[i]==1:
nl=nl+1
else:
nr=nr+1
'''
# left=left.astype('int')
# right=right.astype('int')
if nl < minChild or nr < minChild:
continue
# mat0 = dataSet[nonzero(dataSet[:,feature] > value)[0],:] #nonzero对应于去掉特征数据缺失值
# mat1 = dataSet[nonzero(dataSet[:,feature] <= value)[0],:]
XsrcL = Xsrc[:, left]
XsrcR = Xsrc[:, right]
XtarL = Xtar[:, left]
XtarR = Xtar[:, right]
if splitevaltype == 'random':
trerr = 0
elif splitevaltype == 'banlanced':
trerr = np.square(nl - nr)
elif splitevaltype == 'variance':
trerrL = sum(np.var(XtarL, axis=1, ddof=1)) / Ft
trerrR = sum(np.var(XtarR, axis=1, ddof=1)) / Ft
if kappa > 0:
trerrLsrc = sum(np.var(XsrcL, axis=1, ddof=1)) / Fs
trerrRsrc = sum(np.var(XsrcR, axis=1, ddof=1)) / Fs
trerrL = (trerrL + kappa * trerrLsrc) / 2
trerrR = (trerrR + kappa * trerrRsrc) / 2
trerr = (nl * trerrL + nr * trerrR) / (nl + nr)
elif splitevaltype == 'reconstruction':
XsrcL = np.row_stack((XsrcL, np.ones(XsrcL.shape[1])))
TL = XtarL.dot(
np.linalg.lstsq(((XsrcL.dot(XsrcL.T)) +
lamdda * np.eye(XsrcL.shape[0])),
XsrcL)[0].T)
XsrcR = np.row_stack((XsrcR, np.ones(XsrcR.shape[1])))
TR = XtarR.dot(
np.linalg.lstsq(((XsrcR.dot(XsrcR.T)) +
lamdda * np.eye(XsrcR.shape[0])),
XsrcR)[0].T)
trerrL = np.sqrt(sum(sum((XtarL - TL * XsrcL)**2)) / nl)
trerrR = np.sqrt(sum(sum((XtarR - TR * XsrcR)**2)) / nr)
if kappa > 0:
trerrLsrc = sum(np.var(XsrcL, axis=1, ddof=1)) / Fs
trerrRsrc = sum(np.var(XsrcR, axis=1, ddof=1)) / Fs
trerrL = (trerrL + kappa * trerrLsrc) / 2
trerrR = (trerrR + kappa * trerrRsrc) / 2
trerr = (nl * trerrL + nr * trerrR) / (nl + nr)
else:
raise ValueError('Unknown split evaluation type')
if trerr < rerr:
rerr = trerr
thr = tthr
fid = s
return fid, thr, rerr
def get_surface_distance(surf,
dlabel=None,
medial=None,
medial_labels=None,
drop_labels=None,
use_wb=False,
n_proc=1,
verbose=False):
"""
Calculates surface distance for vertices in `surf`
Parameters
----------
surf : str or os.PathLike
Path to surface file on which to calculate distance
dlabel : str or os.PathLike, optional
Path to file with parcel labels for provided `surf`. If provided will
calculate parcel-parcel distances instead of vertex distances. Default:
None
medial : str or os.PathLike, optional
Path to file containing labels for vertices corresponding to medial
wall. If provided (and `use_wb=False`), will disallow calculation of
surface distance along the medial wall. Default: None
medial_labels : list of str, optional
List of parcel names that comprise the medial wall and through which
travel should be disallowed (if `dlabel` provided and `use_wb=False`).
Will supersede `medial` if both are provided. Default: None
drop_labels : list of str, optional
List of parcel names that should be dropped from the final distance
matrix (if `dlabel` is provided). If not specified, will ignore all
parcels commonly used to reference the medial wall (e.g., 'unknown',
'corpuscallosum', '???', 'Background+FreeSurfer_Defined_Medial_Wall').
Default: None
use_wb : bool, optional
Whether to use calls to `wb_command -surface-geodesic-distance` for
computation of the distance matrix; this will involve significant disk
I/O. If False, all computations will be done in memory using the
:func:`scipy.sparse.csgraph.dijkstra` function. Default: False
n_proc : int, optional
Number of processors to use for parallelizing distance calculation. If
negative, will use max available processors plus 1 minus the specified
number. Default: 1 (no parallelization)
verbose : bool, optional
Whether to print progress bar while distances are calculated. Default:
True
Returns
-------
distance : (N, N) numpy.ndarray
Surface distance between vertices/parcels on `surf`
Notes
-----
The distance matrix computed with `use_wb=False` will have slightly lower
values than when `use_wb=True` due to known estimation errors. These will
be fixed at a later date.
"""
if drop_labels is None:
drop_labels = [
'unknown', 'corpuscallosum', '???',
'Background+FreeSurfer_Defined_Medial_Wall'
]
if medial_labels is None:
medial_labels = []
# convert to paths, if necessary
surf, dlabel, medial = pathify(surf), pathify(dlabel), pathify(medial)
# wb_command requires gifti files so convert if we receive e.g., a FS file
# also return a "remove" flag that will be used to delete the temporary
# gifti file at the end of this process
surf, remove_surf = _surf_to_gii(surf)
n_vert = len(nib.load(surf).agg_data()[0])
# check if dlabel / medial wall files were provided
labels, mask = None, np.zeros(n_vert, dtype=bool)
dlabel, remove_dlabel = _labels_to_gii(dlabel, surf)
medial, remove_medial = _labels_to_gii(medial, surf)
# get data from dlabel / medial wall files if they provided
if dlabel is not None:
labels = nib.load(dlabel).agg_data()
if medial is not None:
mask = nib.load(medial).agg_data().astype(bool)
# determine which parcels should be ignored (if they exist)
delete, uniq_labels = [], np.unique(labels)
if (len(drop_labels) > 0 or len(medial_labels) > 0) and labels is not None:
# get vertex labels
n_labels = len(uniq_labels)
# get parcel labels and reverse dict to (name : label)
table = nib.load(dlabel).labeltable.get_labels_as_dict()
table = {v: k for k, v in table.items()}
# generate dict mapping label to array indices (since labels don't
# necessarily start at 0 / aren't contiguous)
idx = dict(zip(uniq_labels, np.arange(n_labels)))
# get indices of parcel distance matrix to be deleted
for lab in set(table) & set(drop_labels):
lab = table.get(lab)
delete.append(idx.get(lab))
for lab in set(table) & set(medial_labels):
lab = table.get(lab)
mask[labels == lab] = True
# calculate distance from each vertex to all other parcels
parallel = Parallel(n_jobs=n_proc, max_nbytes=None)
if use_wb:
parfunc = delayed(_get_workbench_distance)
graph = surf
else:
parfunc = delayed(_get_graph_distance)
graph = make_surf_graph(*nib.load(surf).agg_data(), mask=mask)
bar = trange(n_vert, verbose=verbose, desc='Calculating distances')
dist = np.row_stack(parallel(parfunc(n, graph, labels) for n in bar))
# average distance for all vertices within a parcel + set diagonal to 0
if labels is not None:
dist = np.row_stack(
[dist[labels == lab].mean(axis=0) for lab in uniq_labels])
dist[np.diag_indices_from(dist)] = 0
# remove distances for parcels that we aren't interested in
if len(delete) > 0:
for axis in range(2):
dist = np.delete(dist, delete, axis=axis)
# if we created gifti files then remove them
if remove_surf:
surf.unlink()
if remove_dlabel:
dlabel.unlink()
if remove_medial:
medial.unlink()
return dist
