def sqcover(A,n):
edge = sp.sqrt(A) # the length of an edge
d = edge/n # the distance between two adjacent points
r = d/2 # the "radius of "
end = edge - r # end point
base = sp.linspace(r, end, n)
first_line = sp.transpose(sp.vstack((base, r*sp.ones(n))))
increment = sp.transpose(sp.vstack((sp.zeros(n), d*sp.ones(n))))
pts = first_line
y_diff = increment
for i in range(n-1):
pts = sp.vstack((pts, first_line + y_diff))
y_diff = y_diff + increment
# Color matter
colors = []
for p in pts:
cval = n*p[0] + p[1] # the x-coord has a higher weight
cval = colormap.Spectral(cval/((n+1)*end)) # normalize by the max value that cval can take.
colors.append(cval)
colors = sp.array(colors)
cover = (pts, r, colors)
return cover
def PESvsaq(optstate,persist,**para):
para = copy.deepcopy(para)
if persist==None:
persist = {'n':0,'d':len(para['ub'])}
n = persist['n']
d = persist['d']
if n<para['nrandinit']:
persist['n']+=1
return randomaq(optstate,persist,**para)
logger.info('PESvsaq')
#logger.debug(sp.vstack([e[0] for e in optstate.ev]))
#raise
x=sp.vstack(optstate.x)
y=sp.vstack(optstate.y)
s= sp.vstack([e['s'] for e in optstate.ev])
dx=[e['d'] for e in optstate.ev]
pesobj = PES.PES(x,y,s,dx,para['lb'],para['ub'],para['kindex'],para['mprior'],para['sprior'],DH_SAMPLES=para['DH_SAMPLES'],DM_SAMPLES=para['DM_SAMPLES'], DM_SUPPORT=para['DM_SUPPORT'],DM_SLICELCBPARA=para['DM_SLICELCBPARA'],mode=para['SUPPORT_MODE'],noS=para['noS'])
[xmin,ymin,ierror] = pesobj.search_acq(para['cfn'],para['logsl'],para['logsu'],volper=para['volper'])
logger.debug([xmin,ymin,ierror])
para['ev']['s']=10**xmin[-1]
xout = [i for i in xmin[:-1]]
return xout,para['ev'],persist,{'HYPdraws':[k.hyp for k in pesobj.G.kf],'mindraws':pesobj.Z,'DIRECTmessage':ierror,'PESmin':ymin}
return
def EIMAPaq(optstate,persist,ev=None, ub = None, lb=None, nrandinit=None, mprior=None,sprior=None,kindex = None,directmaxiter=None):
para = copy.deepcopy(para)
if persist==None:
persist = {'n':0,'d':len(ub)}
n = persist['n']
d = persist['d']
if n<nrandinit:
persist['n']+=1
return randomaq(optstate,persist,ev=ev,lb=lb,ub=ub)
logger.info('EIMAPaq')
#logger.debug(sp.vstack([e[0] for e in optstate.ev]))
#raise
x=sp.vstack(optstate.x)
y=sp.vstack(optstate.y)
s= sp.vstack([e['s'] for e in optstate.ev])
dx=[e['d'] for e in optstate.ev]
MAP = GPdc.searchMAPhyp(x,y,s,dx,mprior,sprior, kindex)
logger.info('MAPHYP {}'.format(MAP))
G = GPdc.GPcore(x,y,s,dx,GPdc.kernel(kindex,d,MAP))
def directwrap(xq,y):
xq.resize([1,d])
a = G.infer_lEI(xq,[ev['d']])
return (-a[0,0],0)
[xmin,ymin,ierror] = DIRECT.solve(directwrap,lb,ub,user_data=[], algmethod=0, maxf = directmaxiter, logfilename='/dev/null')
#logger.debug([xmin,ymin,ierror])
persist['n']+=1
return [i for i in xmin],ev,persist,{'MAPHYP':MAP,'logEImin':ymin,'DIRECTmessage':ierror}
def update():
global i
if i == tvec.shape[0]-1:
i = 0
else:
i = i + 1
if show_left:
poi_left_scatter.setData(pos=sp.expand_dims(poi_left_pos[i],0))
hand_left_scatter.setData(pos=sp.expand_dims(hand_left_pos[i],0))
string_left_line.setData(pos=sp.vstack((hand_left_pos[i],poi_left_pos[i])))
# arm_left.setData(pos=sp.vstack((hand_left_pos[i],[0,-1*shoulder_width/2,0])))
arm_left.setData(pos=sp.vstack((hand_left_pos[i],[0,0,offset])))
else:
poi_left_scatter.hide()
poi_left_line.hide()
hand_left_scatter.hide()
hand_left_line.hide()
string_left_line.hide()
arm_left.hide()
if show_right:
poi_right_scatter.setData(pos=sp.expand_dims(poi_right_pos[i],0))
hand_right_scatter.setData(pos=sp.expand_dims(hand_right_pos[i],0))
string_right_line.setData(pos=sp.vstack((hand_right_pos[i],poi_right_pos[i])))
# arm_right.setData(pos=sp.vstack((hand_right_pos[i],[0,shoulder_width/2,0])))
arm_right.setData(pos=sp.vstack((hand_right_pos[i],[0,0,offset])))
else:
poi_right_scatter.hide()
poi_right_line.hide()
hand_right_scatter.hide()
hand_right_line.hide()
string_right_line.hide()
arm_right.hide()
def Ei(self, Pp, i):
""" Calculate E_i^P
Parameters
-------------
Pp : ndarray, shape (n, k)
Conditional choice probabilities for provinces
i : int, 1 to k
Province
Returns
-----------
Ei : ndarray, shape (n, )
Values of :math:`E_i^P(l, a)` in part (b)
Notes
----------
.. math::
E_i^P(l, s) = \sum_{a=0}^1 P_i[a | l, s] E_i^P(a, l, s)
"""
E = sp.vstack((self.Ei_ai(Pp, i, a) for a in (0, 1))).T
W = sp.vstack((Pp[:, _pp(i, a)] for a in (0, 1))).T
return (E * W).sum(1)
def calc_probability_matrix(trains_a, trains_b, metric, tau, z):
""" Calculates the probability matrix that one spike train from stimulus X
will be classified as spike train from stimulus Y.
:param list trains_a: Spike trains of stimulus A.
:param list trains_b: Spike trains of stimulus B.
:param str metric: Metric to base the classification on. Has to be a key in
:const:`metrics.metrics`.
:param tau: Time scale parameter for the metric.
:type tau: Quantity scalar.
:param float z: Exponent parameter for the classifier.
"""
with warnings.catch_warnings():
warnings.filterwarnings("ignore", "divide by zero")
dist_mat = calc_single_metric(trains_a + trains_b, metric, tau) ** z
dist_mat[sp.diag_indices_from(dist_mat)] = 0.0
assert len(trains_a) == len(trains_b)
l = len(trains_a)
classification_of_a = sp.argmin(sp.vstack((
sp.sum(dist_mat[:l, :l], axis=0) / (l - 1),
sp.sum(dist_mat[l:, :l], axis=0) / l)) ** (1.0 / z), axis=0)
classification_of_b = sp.argmin(sp.vstack((
sp.sum(dist_mat[:l, l:], axis=0) / l,
sp.sum(dist_mat[l:, l:], axis=0) / (l - 1))) ** (1.0 / z), axis=0)
confusion = sp.empty((2, 2))
confusion[0, 0] = sp.sum(classification_of_a == 0)
confusion[1, 0] = sp.sum(classification_of_a == 1)
confusion[0, 1] = sp.sum(classification_of_b == 0)
confusion[1, 1] = sp.sum(classification_of_b == 1)
return confusion / 2.0 / l
def _pair_overlap(waves1, waves2, mean1, mean2, cov1, cov2):
""" Calculate FP/FN estimates for two gaussian clusters
"""
from sklearn import mixture
means = sp.vstack([[mean1], [mean2]])
covars = sp.vstack([[cov1], [cov2]])
weights = sp.array([waves1.shape[1], waves2.shape[1]], dtype=float)
weights /= weights.sum()
# Create mixture of two Gaussians from the existing estimates
mix = mixture.GMM(n_components=2, covariance_type="full", init_params="")
mix.covars_ = covars
mix.weights_ = weights
mix.means_ = means
posterior1 = mix.predict_proba(waves1.T)[:, 1]
posterior2 = mix.predict_proba(waves2.T)[:, 0]
return (
posterior1.mean(),
posterior2.sum() / len(posterior1),
posterior2.mean(),
posterior1.sum() / len(posterior2),
)
def gpmapasrecc(optstate, **para):
if para["onlyafter"] > len(optstate.y) or not len(optstate.y) % para["everyn"] == 0:
return [sp.NaN for i in para["lb"]], {"didnotrun": True}
logger.info("gpmapas reccomender")
d = len(para["lb"])
x = sp.hstack([sp.vstack(optstate.x), sp.vstack([e["xa"] for e in optstate.ev])])
y = sp.vstack(optstate.y)
s = sp.vstack([e["s"] for e in optstate.ev])
dx = [e["d"] for e in optstate.ev]
MAP = GPdc.searchMAPhyp(x, y, s, dx, para["mprior"], para["sprior"], para["kindex"])
logger.info("MAPHYP {}".format(MAP))
G = GPdc.GPcore(x, y, s, dx, GPdc.kernel(para["kindex"], d + 1, MAP))
def directwrap(xq, y):
xq.resize([1, d])
xe = sp.hstack([xq, sp.array([[0.0]])])
# print xe
a = G.infer_m(xe, [[sp.NaN]])
return (a[0, 0], 0)
[xmin, ymin, ierror] = DIRECT.solve(
directwrap, para["lb"], para["ub"], user_data=[], algmethod=1, volper=para["volper"], logfilename="/dev/null"
)
logger.info("reccsearchresult: {}".format([xmin, ymin, ierror]))
return [i for i in xmin], {"MAPHYP": MAP, "ymin": ymin}
def gettimes(ionocontlist):
"""
This static method will take a list of files, or a single string, and
deterimine the time ordering and give the sort order for the files to be in.
Inputs
ionocontlist- A list of IonoContainer h5 files. Can also be a single
string of a file name.
Outputs
sortlist - A numpy array of integers that will chronilogically order
the files
outtime - A Nt x 2 numpy array of all of the times.
timebeg - A list of beginning times
"""
if isinstance(ionocontlist,string_types):
ionocontlist=[ionocontlist]
timelist=[]
fileslist = []
for ifilenum,ifile in enumerate(ionocontlist):
with tables.open_file(str(ifile)) as f:
times = f.root.Time_Vector.read()
timelist.append(times)
fileslist.append(ifilenum*sp.ones(len(times)))
times_file =sp.array([i[:,0].min() for i in timelist])
sortlist = sp.argsort(times_file)
timelist_s = [timelist[i] for i in sortlist]
timebeg = times_file[sortlist]
fileslist = sp.vstack([fileslist[i][0] for i in sortlist]).flatten().astype('int64')
outime = sp.vstack(timelist_s)
return (sortlist,outime,fileslist,timebeg,timelist_s)
def MNEfit(stim,resp,order):
# in order for dlogloss to work, we need to know -<g(yt(n),xt)>data
# == calculate the constrained averages over the data set
Nsamples = sp.size(stim,0)
Ndim = sp.size(stim,1)
psp = sp.mean(sp.mean(resp)) #spike probability (first constraint)
avg = (1.0*stim.T*resp)/(Nsamples*1.0)
avgs = sp.vstack((psp,avg))
if(order > 1):
avgsqrd = (stim.T*1.0)*(sp.array(sp.tile(resp,(1,Ndim)))*sp.array(stim))/(Nsamples*1.0)
avgsqrd = sp.reshape(avgsqrd,(Ndim**2,1))
avgs = sp.vstack((avgs,avgsqrd))
#initialize params:
pstart = sp.log(1/avgs[0,0] - 1)
pstart = sp.hstack((pstart,(.001*(2*sp.random.rand(Ndim)-1))))
if(order > 1):
temp = .0005*(2*sp.random.rand(Ndim,Ndim)-1)
pstart = sp.hstack((pstart,sp.reshape(temp+temp.T,(1,Ndim**2))[0]))
#redefine functions with fixed vals:
def logLoss(p):
return LLF.log_loss(p, stim, resp, order)
def dlogLoss(p):
return LLF.d_log_loss(p, stim, avgs, order)
#run the function:
#pfinal = opt.fmin_tnc(logLoss,pstart,fprime=dlogLoss)
# conjugate-gradient:
pfinal = opt.fmin_cg(logLoss,pstart,fprime=dlogLoss)
#pfinal = opt.fmin(logLoss,pstart,fprime=dlogLoss)
return pfinal
def infer_diag(self,X_i,D_i):
ns=X_i.shape[0]
D = [0 if sp.isnan(x[0]) else int(sum([8**i for i in x])) for x in D_i]
R=sp.vstack([sp.empty([2,ns])]*self.size)
libGP.infer_diag(self.s,cint(self.size), ns,X_i.ctypes.data_as(ctpd),(cint*len(D))(*D),R.ctypes.data_as(ctpd))
m = sp.vstack([R[i*2,:] for i in xrange(self.size)])
V = sp.vstack([R[i*2+1,:] for i in xrange(self.size)])
return [m,V]
def my_bh_fdr(p_val_vec):
index = scipy.argsort(p_val_vec)
exp_err = scipy.vstack((float(len(p_val_vec))/scipy.arange(1,len(p_val_vec) + 1)*p_val_vec[index],
scipy.tile(1, [1, len(p_val_vec)]))).min(axis = 0)
exp_err = scipy.vstack((exp_err,exp_err[scipy.r_[0,scipy.arange(len(exp_err)-1)]])).max(axis=0)
#scipy.r_[index[0], index[range(len(index)-1)]
resort_index = scipy.argsort(index)
return exp_err[resort_index]
def test_skip():
"""Test if only keeping every n'th sample works."""
X = scipy.vstack((scipy.arange(25), scipy.arange(25)))
X_ = skip(X, 2, 5)
print X_
des = scipy.vstack((scipy.array([0, 1, 2, 3, 4, 10, 11, 12, 13, 14, 20, 21, 22, 23 ,24]),
scipy.array([0, 1, 2, 3, 4, 10, 11, 12, 13, 14, 20, 21, 22, 23 ,24])))
assert (X_ == des).all(), 'wrong result'
def extract_spikes(train, signals, length, align_time):
""" Extract spikes with waveforms from analog signals using a spike train.
Spikes that are too close to the beginning or end of the shortest signal
to be fully extracted are ignored.
:type train: :class:`neo.core.SpikeTrain`
:param train: The spike times.
:param sequence signals: A sequence of :class:`neo.core.AnalogSignal`
objects from which the spikes are extracted. The waveforms of
the returned spikes are extracted from these signals in the
same order they are given.
:type length: Quantity scalar
:param length: The length of the waveform to extract as time scalar.
:type align_time: Quantity scalar
:param align_time: The alignment time of the spike times as time scalar.
This is the time delta from the start of the extracted waveform
to the exact time of the spike.
:returns: A list of :class:`neo.core.Spike` objects, one for each
time point in ``train``. All returned spikes include their
``waveform`` property.
:rtype: list
"""
if len(set(s.sampling_rate for s in signals)) > 1:
raise ValueError(
'All signals for spike extraction need the same sampling rate')
wave_unit = signals[0].units
srate = signals[0].sampling_rate
end = min(s.shape[0] for s in signals)
aligned_train = train - align_time
cut_samples = int((length * srate).simplified)
st = sp.asarray((aligned_train * srate).simplified)
# Find extraction epochs
st_ok = (st >= 0) * (st < end - cut_samples)
epochs = sp.vstack((st[st_ok], st[st_ok] + cut_samples)).T
nspikes = epochs.shape[0]
if not nspikes:
return []
# Create data
data = sp.vstack([sp.asarray(s.rescale(wave_unit)) for s in signals])
nc = len(signals)
spikes = []
for s in xrange(nspikes):
waveform = sp.zeros((cut_samples, nc))
for c in xrange(nc):
waveform[:, c] = \
data[c, epochs[s, 0]:epochs[s, 1]]
spikes.append(neo.Spike(train[st_ok][s], waveform=waveform * wave_unit))
return spikes
def stripe2():
Y1 = sp.vstack((sp.ones((50,1)), sp.zeros((50,1))))
Y2 = sp.vstack((sp.zeros((50,1)), sp.ones((50,1))))
Y = sp.hstack([Y1, Y2])
X1 = sp.random.multivariate_normal([-2,2], [[1,.8],[.8,1]],size=50)
X2 = sp.random.multivariate_normal([2,-1], [[1,.8],[.8,1]], size=50)
X = sp.hstack((sp.ones((100,1)),sp.vstack([X1,X2])))
return Y, X
def load_single_player_data(use_existing=False, num_train=0):
aa=np.load('/Users/amit/Desktop/Dropbox/Markov/IMSPL.npy')
bb=np.load('/Users/amit/Desktop/Dropbox/Markov/IMSBGD.npy')
aa=standardize_data(aa)
bb=standardize_data(bb)
#ii=np.int32(np.floor(np.random.rand(100)*bb.shape[0]))
# py.figure(1)
# for j,i in enumerate(ii):
# py.subplot(10,10,j+1)
# py.imshow(bb[i,:,:,:])
# py.axis('off')
# py.axis('equal')
# py.show()
if (num_train==0):
num=aa.shape[0]
else:
num=np.minimum(aa.shape[0],num_train)
if (not use_existing):
ii=range(num)
np.random.shuffle(ii)
np.save('ii.npy',ii)
aa=aa[ii,]
else:
if (os.path.isfile('ii.npy')):
ii=np.load('ii.npy')
aa=aa[ii,]
train_num=np.int32(num/2)
val_num=np.int32(num/4)
test_num=np.int32(num/4)
head=aa[:,0:25,:,:]
body=aa[:,20:45,:,:]
legs=aa[:,35:60,:,:]
bgd=bb[:,20:45,:,:]
val_start=train_num
val_end=val_num+val_start
test_start=val_end
test_end=test_num+test_start
X_train=scipy.vstack((head[0:train_num,],body[0:train_num,],legs[0:train_num],bgd[0:train_num,]))
X_val=scipy.vstack((head[val_start:val_end,],body[val_start:val_end,],
legs[val_start:val_end,],bgd[val_start:val_end,]))
X_test=scipy.vstack((head[test_start:test_end,],
body[test_start:test_end,],
legs[test_start:test_end,],
bgd[test_start:test_end,]))
X_train=X_train.transpose((0,3,1,2)) #/256.
X_val=X_val.transpose((0,3,1,2)) #/256.
X_test=X_test.transpose((0,3,1,2)) #/256.
y_train=np.repeat(range(4),train_num)
y_val=np.repeat(range(4),val_num)
y_test=np.repeat(range(4),test_num)
return (np.float32(X_train),np.uint8(y_train),np.float32(X_val),np.uint8(y_val),np.float32(X_test),np.uint8(y_test))
def broyden(func, x1, x2, tol=1e-5, maxiter=50):
"""Calculate the zero of a multi-dimensional function using Broyden's method"""
def isscalar(x):
if isinstance(x, sp.ndarray):
if x.size == 1:
return x.flatten()[0]
else:
return x
else:
return x
def update_Jacobian(preJac, ch_x, ch_F):
"""Update Jacobian from preX to newX
preX and newX are assumed to be array objects of the same shape"""
frac = (ch_F-(preJac.dot(ch_x)))/(la.norm(ch_x)**2)
Jac = preJac+sp.dot(isscalar(frac),ch_x.T)
return Jac
#truncate list to two tiems and sort
x1 = sp.vstack(x1.flatten())
x2 = sp.vstack(x2.flatten())
fx1 = func(x1)
fx2 = func(x2)
#check our original points for zeros
if abs(fx1) < tol:
return x1
elif abs(fx2) < tol:
return x2
#Calculate initial Jacobian matrix
jac = Jacobian(func)(x1)
mi = maxiter
while abs(fx2) > tol and mi > 0:
fx1 = func(x1)
fx2 = func(x2)
ch_x=x2-x1
ch_F=fx2-fx1
jac = update_Jacobian(jac, ch_x, ch_F)
y = la.lstsq(jac, sp.array([-fx2]))[0]
xnew = y+x2
x1 = x2
x2 = xnew
mi -= 1
if mi==0:
raise StopIteration("Did not converge in {} iterations".format(maxiter))
else:
return x2, maxiter-mi
def TCA(X_S, X_T, m=40, mu=0.1, kernel_para=1, p=2, random_sample_T=0.01):
X_S = sp.mat(X_S)
X_T = sp.mat(X_T)
n_S = X_S.shape[0]
n_T = X_T.shape[0]
if random_sample_T != 1:
print str(int(n_T * random_sample_T)) + " samples taken from the task domain"
index_sample = sp.random.choice([i for i in range(n_T)], size=int(n_T * random_sample_T))
X_T = X_T[index_sample, :]
n_T = X_T.shape[0]
n = n_S + n_T
if m > (n):
print ("m is larger then n_S+n_T, so it has been changed")
m = n
L = sp.zeros(shape=(n, n))
L_SS = sp.ones(shape=(n_S, n_S)) / (n_S ** 2)
L_TT = sp.ones(shape=(n_T, n_T)) / (n_T ** 2)
L_ST = -sp.ones(shape=(n_S, n_T)) / (n_S * n_T)
L_TS = -sp.ones(shape=(n_T, n_S)) / (n_S * n_T)
L[0:n_S, 0:n_S] = L_SS
L[n_S : n_S + n_T, n_S : n_S + n_T] = L_TT
L[n_S : n_S + n_T, 0:n_S] = L_TS
L[0:n_S, n_S : n_S + n_T] = L_ST
R = pdist(sp.vstack([X_S, X_T]), metric="euclidean", p=p, w=None, V=None, VI=None)
K = Gaussian(R, kernel_para, p)
Id = sp.zeros(shape=(n, n))
H = sp.zeros(shape=(n, n))
sp.fill_diagonal(Id, 1)
sp.fill_diagonal(H, 1)
H -= 1.0 / n
Id = sp.mat(Id)
H = sp.mat(H)
K = sp.mat(K)
L = sp.mat(L)
matrix = sp.linalg.inv(K * L * K + mu * Id) * sp.mat(K * H * K)
eigen_values = sp.linalg.eig(matrix)
eigen_val = eigen_values[0][0:m]
eigen_vect = eigen_values[1][:, 0:m]
return (eigen_val, eigen_vect, K, sp.vstack([X_S, X_T]))
def radius( self, frame ):
'''
Bubble radius at one frame.
Method:
1. Load the snapshot at frame
2. Load x, y, z coordinates
3. Calculate density grid mesh at grid points
4. Filter the shell grids with density between low * max density and high * max density
5. Calculate the average radius
'''
start = time.clock()
self.set_frame( frame )
# Load x, y, z coordinates
data = pd.DataFrame( list(self.universe.coord), columns=['x','y','z'])
x = data[ 'x' ].values
y = data[ 'y' ].values
z = data[ 'z' ].values
# Density grid
xyz = scipy.vstack( [ x, y, z ] )
kde = scipy.stats.gaussian_kde( xyz )
xmin, ymin, zmin = x.min(), y.min(), z.min()
xmax, ymax, zmax = x.max(), y.max(), z.max()
NI = complex( imag=self.density_grid_length)
xi, yi, zi = scipy.mgrid[ xmin:xmax:NI, ymin:ymax:NI, zmin:zmax:NI ]
coords = scipy.vstack([item.ravel() for item in [xi, yi, zi]])
density = kde(coords).reshape(xi.shape)
# Filter density grid
density_max = density.max()
density_low = self.density_low * density_max
density_high = self.density_high * density_max
xyzs = []
N = self.density_grid_length
for idx, idy, idz in product( xrange(N), xrange(N), xrange(N) ):
if density_low < density[ idx, idy, idz ] <= density_high:
xyzs.append( [ xi[ idx, idy, idz ], yi[ idx, idy, idz ], zi[ idx, idy, idz ] ] )
xyzs = np.array( xyzs )
# Average radius
center = xyzs.mean( axis=0 )
rs = []
for xyz_ele in xyzs:
rs.append( np.linalg.norm( center - xyz_ele ) )
duration = time.clock() - start
print( "Radius for frame {} calculated in {:.2f} seconds".format( frame, duration ) )
return center, scipy.mean( rs )
def store(old, new): old=old.reshape((1,len(old))) lold=old.shape[1] lnew=new.shape[1] if (lold==lnew): X=sc.vstack((old,new)) elif (lold>lnew): new =sc.hstack(([0]*(lold-lnew),new)) X=X=sc.vstack((old,new)) elif (lnew>lold): old =sc.hstack((old,[0]*(lnew-lold))) X=X=sc.vstack((old,new)) return(X)
def load(filename, network=None):
r"""
Loads data onto the given network from an appropriately formatted
'mat' file (i.e. MatLAB output).
Parameters
----------
filename : string (optional)
The name of the file containing the data to import. The formatting
of this file is outlined below.
network : OpenPNM Network Object
The Network object onto which the data should be loaded. If no
Network is supplied than one will be created and returned.
Returns
-------
If no Network object is supplied then one will be created and returned.
"""
net = {}
import scipy.io as _spio
data = _spio.loadmat(filename)
# Deal with pore coords and throat conns specially
if 'throat_conns' in data.keys():
net.update({'throat.conns': _sp.vstack(data['throat_conns'])})
Nt = _sp.shape(net['throat.conns'])[0]
net.update({'throat.all': _sp.ones((Nt,), dtype=bool)})
del data['throat_conns']
else:
logger.warning('\'throat_conns\' not found')
if 'pore_coords' in data.keys():
net.update({'pore.coords': _sp.vstack(data['pore_coords'])})
Np = _sp.shape(net['pore.coords'])[0]
net.update({'pore.all': _sp.ones((Np,), dtype=bool)})
del data['pore_coords']
else:
logger.warning('\'pore_coords\' not found')
# Now parse through all the other items
items = [i for i in data.keys() if '__' not in i]
for item in items:
element = item.split('_')[0]
prop = item.split('_', maxsplit=1)[1]
net[element+'.'+prop] = _sp.squeeze(data[item].T)
if network is None:
network = OpenPNM.Network.GenericNetwork()
network = _update_network(network=network, net=net)
return network
def optimize(f, left, right, tol=1e-3, n=200, **kwargs):
x_obs = sp.array([[(left + right)/2.]])
y_obs = f(x_obs)
x_test = sp.linspace(left, right, n)[:, None]
lcb = sp.array([[-sp.inf]])
while lcb.min() - y_obs.min() < -tol:
lcb = lower_confidence_bound(x_test, x_obs, y_obs, **kwargs)
print lcb.min() - y_obs.min()
x_new = x_test[sp.argmin(lcb)]
x_obs = sp.vstack([x_obs, [x_new]])
y_obs = sp.vstack([y_obs, f(x_new)])
return x_obs, y_obs.flatten()
def intersect(self, conset2):
"""Determine intersection between current constraint set and another"""
def remredcons(A, b, verts):
"""Reduce a constraint set by removing unnecessary constraints."""
eps = 10e-9
#1 Co-planar constraints;
# Remove as not to affect 3rd check
Ab = c_[A, b]
Abnorms = ones((Ab.shape[0], 1))
for i in range(Ab.shape[0]):
Abnorms[i] = linalg.norm(Ab[i, :])
Abn = Ab/Abnorms
Abkeep = ones((0, Ab.shape[1]))
Abtest = ones((0, Ab.shape[1]))
for r1 in range(Abn.shape[0]):
noocc = ones((1, 0))
for r2 in range(Abn.shape[0]):
#print abs(Abn[r1, :] - Abn[r2, :])
if numpy.all(abs(Abn[r1, :] - Abn[r2, :]) < eps):
noocc = c_[noocc, r2]
if noocc.size == 1:
Abtest = vstack([Abtest, Ab[r1, :]])
else:
Abkeep = vstack([Abkeep, Ab[r1, :]])
if Abkeep.shape[0] > 1:
Abkeep = uniqm(Abkeep, eps)
#2 Vert subset satisfying; no action needed (redundancy uncertain)
#3 All vert satisfying constraints;
A, b = splitAb(array(Abtest).ravel(), verts.shape[1])
keepA = ones((0, A.shape[1]))
keepb = ones((0, 1))
bt = tile(b, (1, verts.shape[0]))
k = mat(A)*mat(verts.T) - bt
kk = sum(k > eps, axis=1)
for i in range(len(kk)):
if kk[i] != 0:
keepA = vstack([keepA, A[i, :]])
keepb = vstack([keepb, b[i, :]])
outAb = vstack([c_[keepA, keepb], Abkeep])
return splitAb(outAb.ravel(), verts.shape[1])
#Combine constraints and vertices
combA = vstack((self.A, conset2.A))
combb = vstack((self.b, conset2.b))
combv = vstack((self.vert, conset2.vert))
#Remove redundant constraints
ncombA, ncombb = remredcons(combA, combb, combv)
#Calc and return intersection
intcombvert = con2vert(combA, combb)[0]
return intcombvert
def stripe3():
zero = sp.zeros((33,1))
ones = sp.ones((33,1))
Y1 = sp.vstack([ones, zero, zero])
Y2 = sp.vstack([zero, ones, zero])
Y3 = sp.vstack([zero, zero, ones])
Y = sp.hstack((Y1, Y2, Y3))
X1 = sp.random.multivariate_normal([-2,2], [[1,.8],[.8,1]], size=33)
X2 = sp.random.multivariate_normal([2,-2], [[1,.8],[.8,1]], size=33)
X3 = sp.random.multivariate_normal([0,0], [[1,.8],[.8,1]], size=33)
X = sp.hstack((sp.vstack((ones,ones,ones)),sp.vstack((X1,X2,X3))))
return Y, X
def save_network_tocsv(self,path='',filename='network'):
r'''
'''
if path=='':
path = os.path.abspath('')+'\\LocalFiles\\'
Xp = self.get_pore_indices()
Xt = self.get_throat_indices()
for p in self._pore_data.keys():
if sp.shape(sp.shape(self.get_pore_data(prop=p)))==(1,):
Xp = sp.vstack((Xp,self.get_pore_data(prop=p)))
sp.savetxt(path+'\\'+filename+'_pores_'+p+'.csv',self.get_pore_data(prop=p))
for t in self._throat_data.keys():
if sp.shape(sp.shape(self.get_throat_data(prop=t)))==(1,):
Xt = sp.vstack((Xt,self.get_throat_data(prop=t)))
sp.savetxt(path+'\\'+filename+'_throats_'+t+'.csv',self.get_throat_data(prop=t))
def diffmat(dims,order = 'C'):
""" This function will return a tuple of difference matricies for data from an
Nd array that has been rasterized. The order parameter determines whether
the array was rasterized in a C style (python) of FORTRAN style (MATLAB).
Inputs:
dims- A list of the size of the x,y,z.. dimensions.
order- Specifies the vectorization of the matrix
Outputs:
dx,dy,dy... - The finite difference operators for a vectorized array.
If these are to be stacked together as one big operator then
sp.sparse.vstack should be used.
"""
# flip the dimensions around
dims=[int(i) for i in dims]
xdim = dims[0]
ydim = dims[1]
dims[0]=ydim
dims[1]=xdim
if order.lower() == 'c':
dims = dims[::-1]
outD = []
for idimn, idim in enumerate(dims):
if idim==0:
outD.append(sp.array([]))
continue
e = sp.ones(idim)
dthing = sp.vstack((-e,e))
D = sp.sparse.spdiags(dthing,[0,1],idim-1,idim).toarray()
D = sp.vstack((D,D[-1]))
if idim>0:
E = sp.sparse.eye(sp.prod(dims[:idimn]))
D = sp.sparse.kron(D,E)
if idimn<len(dims)-1:
E = sp.sparse.eye(sp.prod(dims[idimn+1:]))
D = sp.sparse.kron(E,D)
outD.append(sp.sparse.csc_matrix(D))
if order.lower() == 'c':
outD=outD[::-1]
Dy=outD[0]
Dx = outD[1]
outD[0]=Dx
outD[1]=Dy
return tuple(outD)
def _test_scz_():
# Load Schizophrenia data
singleton_snps = genotypes.simulate_k_tons(n=500, m=1000)
doubleton_snps = genotypes.simulate_k_tons(k=2, n=500, m=1000)
common_snps = genotypes.simulate_common_genotypes(500, 1000)
snps = sp.vstack([common_snps, singleton_snps, doubleton_snps])
test_snps = sp.vstack([singleton_snps, doubleton_snps])
print snps
phen_list = phenotypes.simulate_traits(snps, hdf5_file_prefix='/home/bv25/tmp/test', num_traits=30, p=1.0)
singletons_thres = []
doubletons_thres = []
common_thres = []
for i, y in enumerate(phen_list):
K = kinship.calc_ibd_kinship(snps)
K = kinship.scale_k(K)
lmm = lm.LinearMixedModel(y)
lmm.add_random_effect(K)
r1 = lmm.get_REML()
print 'pseudo_heritability:', r1['pseudo_heritability']
ex_res = lm.emmax(snps, y, K)
plt.figure()
plt.hist(y, 50)
plt.savefig('/home/bv25/tmp/test_%d_phen.png' % i)
plt.clf()
agr.plot_simple_qqplots_pvals('/home/bv25/tmp/test_%d' % i,
[ex_res['ps'][:1000], ex_res['ps'][1000:2000], ex_res['ps'][2000:]],
result_labels=['Common SNPs', 'Singletons', 'Doubletons'],
line_colors=['b', 'r', 'y'],
num_dots=200, max_neg_log_val=3)
# Now permutations..
res = lm.emmax_perm_test(singleton_snps, y, K, num_perm=1000)
print 1.0 / (20 * 1000.0), res['threshold_05']
singletons_thres.append(res['threshold_05'][0])
res = lm.emmax_perm_test(doubleton_snps, y, K, num_perm=1000)
print 1.0 / (20 * 1000.0), res['threshold_05']
doubletons_thres.append(res['threshold_05'][0])
res = lm.emmax_perm_test(common_snps, y, K, num_perm=1000)
print 1.0 / (20 * 1000.0), res['threshold_05']
common_thres.append(res['threshold_05'][0])
print sp.mean(singletons_thres), sp.std(singletons_thres)
print sp.mean(doubletons_thres), sp.std(doubletons_thres)
print sp.mean(common_thres), sp.std(common_thres)
def plotstate2D(self,llimit,ulimit,allK=False):
assert self.dim==2
size=60
Esum=sp.zeros([size,size])
Msum=sp.zeros([size,size])
for i,kf in enumerate(self.KFs):
KyR = spl.cho_factor(self.buildKsym(kf,self.X))
xaxis=sp.linspace(llimit[0],ulimit[0],size)
yaxis=sp.linspace(llimit[1],ulimit[1],size)
E=[]
m=[]
C=[]
for y in yaxis:
for x in xaxis:
(Et,mt,Ct)=self.evalEI(self.X,self.Y,KyR,kf,self.best[1],sp.matrix([x,y]))
E.append(Et[0,0])
m.append(mt)
C.append(Ct)
Egrid=sp.vstack(np.split(np.array(E),size))
mgrid=sp.vstack(np.split(np.array(m),size))
Cgrid=sp.vstack(np.split(np.array(C),size))
Esum=Esum+Egrid*sp.exp(self.llks[i])
Msum=Msum+mgrid*sp.exp(self.llks[i])
if allK:
plt.figure(figsize=(20,5))
plt.subplot(131)
try:
plt.contour(xaxis,yaxis,mgrid,50)
except ValueError:
pass
plt.subplot(132)
plt.contour(xaxis,yaxis,Cgrid,50)
plt.subplot(133)
plt.contour(xaxis,yaxis,Egrid,50)
plt.figure(figsize=(20,5))
plt.subplot(131)
try:
plt.contour(xaxis,yaxis,Msum,50)
except ValueError:
pass
plt.subplot(133)
plt.contour(xaxis,yaxis,Esum,50)
plt.show()
return
def invert_epochs(epochs, end=None):
"""inverts epochs inverted
The first epoch will be mapped to [0, start] and the last will be mapped
to [end of last epoch, :end:]. Epochs that accidentally become negative
or zero-length will be omitted.
:type epochs: ndarray
:param epochs: epoch set to invert
:type end: int
:param end: If not None, it i taken for the end of the last epoch,
else max(index-dtype) is taken instead.
Default=None
:returns: ndarray - inverted epoch set
"""
# checks
if end is None:
end = sp.iinfo(INDEX_DTYPE).max
else:
end = INDEX_DTYPE.type(end)
# flip them
rval = sp.vstack((sp.concatenate(([0], epochs[:, 1])), sp.concatenate((epochs[:, 0], [end])))).T
return (rval[rval[:, 1] - rval[:, 0] > 0]).astype(INDEX_DTYPE)
def _read_iop_from_file(self, file_name):
"""
Generic IOP reader that interpolates the iop to the common wavelengths defined in the constructor
returns: interpolated iop
"""
lg.info('Reading :: ' + file_name + ' :: and interpolating to ' + str(self.wavelengths))
if os.path.isfile(file_name):
iop_reader = csv.reader(open(file_name), delimiter=',', quotechar='"')
wave = scipy.float32(iop_reader.next())
iop = scipy.zeros_like(wave)
for row in iop_reader:
iop = scipy.vstack((iop, row))
iop = scipy.float32(iop[1:, :]) # drop the first row of zeros
else:
lg.exception('Problem reading file :: ' + file_name)
raise IOError
try:
int_iop = scipy.zeros((iop.shape[0], self.wavelengths.shape[1]))
for i_iter in range(0, iop.shape[0]):
# r = scipy.interp(self.wavelengths[0, :], wave, iop[i_iter, :])
int_iop[i_iter, :] = scipy.interp(self.wavelengths, wave, iop[i_iter, :])
return int_iop
except IOError:
lg.exception('Error interpolating IOP to common wavelength')
return -1
batch_kalman = []
deltaT = sp.mean(t[1:] - t[0:-1])
state0 = sp.array([0, 0]).T
P0 = sp.identity(2) * 0.1
F0 = sp.array([[1, deltaT],\
[0, 1]])
H0 = sp.identity(2)
Q0 = sp.diagflat([0.005, 0.0001])
R0 = sp.diagflat([0.25, 0.001])
for i in range(BATCH_SIZE):
data = Data1D(sp.squeeze(batch_x[i, :, 0]), sp.squeeze(batch_x[i, :, 1]),
[])
filter1b = LinearKalmanFilter1D(F0, H0, P0, Q0, R0, state0)
kalman_data = filter1b.process_data(data)
batch_kalman.append(sp.vstack([kalman_data.x[1:], kalman_data.vx[1:]]).T)
xk_batch = sp.stack(batch_kalman)
print(xk_batch.shape)
#%% Plot the fit
plt.figure(figsize=(14, 16))
for batch_idx in range(BATCH_SIZE):
out_x = sp.squeeze(out[batch_idx, :, 0])
out_vx = sp.squeeze(out[batch_idx, :, 1])
noisy_x = batch_x[batch_idx, :, 0]
noisy_vx = batch_x[batch_idx, :, 1]
true_x = batch_y[batch_idx, :, 0]
true_vx = batch_y[batch_idx, :, 1]
plt.subplot(20 + (BATCH_SIZE) * 100 + batch_idx * 2 + 1)
if batch_idx == 0: plt.title('Location x')
def _generate_throats(self):
r"""
Generate the throats connections
"""
logger.info('Define connections between pores')
pts = self['pore.coords']
Np = len(pts)
# Generate 6 dummy domains to pad onto each face of real domain This
# prevents surface pores from making long range connections to each other
x, y, z = self['pore.coords'].T
if x.max() > self._Lx:
Lx = x.max() * 1.05
else:
Lx = self._Lx
if y.max() > self._Ly:
Ly = y.max() * 1.05
else:
Ly = self._Ly
if z.max() > self._Lz:
Lz = z.max() * 1.05
else:
Lz = self._Lz
# Reflect in X = Lx and 0
Pxp = pts.copy()
Pxp[:, 0] = 2 * Lx - Pxp[:, 0]
Pxm = pts.copy()
Pxm[:, 0] = Pxm[:, 0] * (-1)
# Reflect in Y = Ly and 0
Pyp = pts.copy()
Pyp[:, 1] = 2 * Ly - Pxp[:, 1]
Pym = pts.copy()
Pym[:, 1] = Pxm[:, 1] * (-1)
# Reflect in Z = Lz and 0
Pzp = pts.copy()
Pzp[:, 2] = 2 * Lz - Pxp[:, 2]
Pzm = pts.copy()
Pzm[:, 2] = Pxm[:, 2] * (-1)
# Add dummy domains to real domain
# Order important for boundary logic
pts = np.vstack((pts, Pxp, Pxm, Pyp, Pym, Pzp, Pzm))
# Perform tessellation
logger.debug('Beginning tessellation')
Tri = sptl.Delaunay(pts)
logger.debug('Converting tessellation to adjacency matrix')
adjmat = sprs.lil_matrix((Np, Np), dtype=int)
for i in sp.arange(0, sp.shape(Tri.simplices)[0]):
# Keep only simplices that are fully in real domain
# this used to be vectorize, but it stopped working...change in scipy?
for j in Tri.simplices[i]:
if j < Np:
adjmat[j, Tri.simplices[i][Tri.simplices[i] < Np]] = 1
# Remove duplicate (lower triangle) and self connections (diagonal)
# and convert to coo
adjmat = sprs.triu(adjmat, k=1, format="coo")
logger.debug('Conversion to adjacency matrix complete')
self['throat.conns'] = sp.vstack((adjmat.row, adjmat.col)).T
self['pore.all'] = np.ones(len(self['pore.coords']), dtype=bool)
self['throat.all'] = np.ones(len(self['throat.conns']), dtype=bool)
# Do Voronoi diagram - creating voronoi polyhedra around each pore and save
# vertex information
self._vor = Voronoi(pts)
all_vert_index = sp.ndarray(Np, dtype=object)
for i, polygon in enumerate(self._vor.point_region[0:Np]):
if -1 not in self._vor.regions[polygon]:
all_vert_index[i] = \
dict(zip(self._vor.regions[polygon],
self._vor.vertices[self._vor.regions[polygon]]))
# Add throat vertices by looking up vor.ridge_dict
throat_verts = sp.ndarray(len(self['throat.conns']), dtype=object)
for i, (p1, p2) in enumerate(self['throat.conns']):
try:
throat_verts[i] = \
dict(zip(self._vor.ridge_dict[(p1, p2)],
self._vor.vertices[self._vor.ridge_dict[(p1, p2)]]))
except KeyError:
try:
throat_verts[i] = \
dict(zip(self._vor.ridge_dict[(p2, p1)],
self._vor.vertices[self._vor.ridge_dict[(p2, p1)]]))
except KeyError:
logger.error(
'Throat Pair Not Found in Voronoi Ridge Dictionary')
self['pore.vert_index'] = all_vert_index
self['throat.vert_index'] = throat_verts
logger.debug(sys._getframe().f_code.co_name + ': End of method')
import networkx
from itertools import chain
import scipy.stats as stats
import pandas as pd
import numpy as np
import mendelian_mutation
alt_scores = comorbidity_genetics.comorbidity_scores('mendelian_disease_alterations_nogermline.pkl','comorb_focal', set(cens_germline))
cg_score = (1 - stats.binom.cdf((coex_mat > rho_cut).sum(axis = 0) - 1, len(msel), (coexpression_c.loc[notm, csel] > rho_cut).sum(axis = 0)/float(len(notm))))
sc = pd.DataFrame(np.array(cg_score),index = coex_mat.columns,columns=['p'])
v = sc.sort(['p'])
v = pd.DataFrame(scipy.vstack(((gene_scores < .05*float(nrand)).sum(axis=0),gscore_vs_background,gene_scores.min(axis=0), background_probability)),columns = rand_score[alt]['cancers'])
1 - stats.binom.cdf((gene_scores < .05*float(nrand)).sum(axis=0),gene_scores.shape[0],
background_probability),
gs = pd.DataFrame(gene_scores, columns = rand_score[alt]['cancers'], index = mend_gn)
background_probability)),
gscore_vs_background = [0]*len(rand_score[alt]['cancers'])
for (c_i, c) in enumerate(rand_score[alt]['cancers']):
background_genes = background_set - alterations[alt][c][1]
p = (background[alt].loc[background_genes, c] < .05).sum()/float(len(background_genes))
background_probability[c_i] =p
net_scores = mendelian_mutation.open_network_scores()
alt_enrichments = mendelian_mutation.open_alteration_enrichments()
xx = zip(*tuple([stats.ranksums(coexpression_c.loc[notm, csel_g], coex_mat.loc[:,csel_g]) for csel_g in csel]))
def weighted_pmi_lrec_items(matrix_train,
embeded_matrix=np.empty((0)),
iteration=4,
lam=80,
rank=200,
alpha=100,
gpu=True,
seed=1,
root=1,
**unused):
"""
Function used to achieve generalized projected lrec w/o item-attribute embedding
:param matrix_train: user-item matrix with shape m*n
:param embeded_matrix: item-attribute matrix with length n (each row represents one item)
:param iteration: number of SVD iterations
:param lam: parameter of penalty
:param rank: the latent dimension/number of items
:param alpha: weights of the U-I ratings
:param gpu: whether use gpu power
:return: prediction in sparse matrix
"""
progress = WorkSplitter()
matrix_input = matrix_train
if embeded_matrix.shape[0] > 0:
matrix_input = vstack((matrix_input, embeded_matrix.T))
progress.subsection("Create PMI matrix")
pmi_matrix = get_pmi_matrix(matrix_input, root)
progress.subsection("Randomized SVD")
start_time = time.time()
P, sigma, Qt = randomized_svd(pmi_matrix, n_components=rank, n_iter=iteration, random_state=seed)
print("Elapsed: {0}".format(inhour(time.time() - start_time)))
start_time = time.time()
if gpu:
import cupy as cp
progress.subsection("Create Cacheable Matrices")
# RQ = matrix_input.dot(sparse.csc_matrix(Qt).T).toarray()
# sqrt sigma injection
RQ = matrix_input.dot(sparse.csc_matrix(Qt.T * np.sqrt(sigma))).toarray()
# Exact
matrix_B = cp.array(RQ)
matrix_BT = matrix_B.T
matrix_A = matrix_BT.dot(matrix_B) + cp.array((lam * sparse.identity(rank, dtype=np.float32)).toarray())
# Approx
# matrix_A = cp.array(sparse.diags(sigma * sigma + lam).todense())
# matrix_B = cp.array(P*sigma)
# matrix_BT = cp.array(matrix_B.T)
print("Elapsed: {0}".format(inhour(time.time() - start_time)))
progress.subsection("Item-wised Optimization")
start_time = time.time()
# For loop
m, n = matrix_train.shape
Y = []
alpha = cp.array(alpha, dtype=cp.float32)
for i in tqdm(xrange(n)):
vector_r = matrix_train[:, i]
vector_y = per_item_gpu(vector_r, matrix_A, matrix_B, matrix_BT, alpha)
y_i_gpu = cp.asnumpy(vector_y)
y_i_cpu = np.copy(y_i_gpu)
Y.append(y_i_cpu)
Y = scipy.vstack(Y)
print("Elapsed: {0}".format(inhour(time.time() - start_time)))
else:
progress.subsection("Create Cacheable Matrices")
RQ = matrix_input.dot(sparse.csc_matrix(Qt).T).toarray()
# Exact
matrix_B = RQ
matrix_BT = RQ.T
matrix_A = matrix_BT.dot(matrix_B) + (lam * sparse.identity(rank, dtype=np.float32)).toarray()
# Approx
# matrix_B = P * sigma
# matrix_BT = matrix_B.T
# matrix_A = sparse.diags(sigma * sigma - lam).todense()
print("Elapsed: {0}".format(inhour(time.time() - start_time)))
progress.subsection("Item-wised Optimization")
start_time = time.time()
# For loop
m, n = matrix_train.shape
Y = []
for i in tqdm(xrange(n)):
vector_r = matrix_train[:, i]
vector_y = per_item_cpu(vector_r, matrix_A, matrix_B, matrix_BT, alpha)
y_i_cpu = vector_y
Y.append(y_i_cpu)
Y = scipy.vstack(Y)
print("Elapsed: {0}".format(inhour(time.time() - start_time)))
return RQ, Y.T, None
def score_2_dof(self, X, snp_dim="col", debug=False):
"""
Parameters
----------
X : (`N`, `1`) ndarray
genotype vector (TODO: X should be small)
Returns
-------
pvalue : float
P value
"""
import scipy as sp
import scipy.linalg as la
import scipy.stats as st
# 1. calculate Qs and pvs
Q_rho = sp.zeros(len(self.rho_list))
Py = P(self.gp, self.y)
for i in range(len(self.rho_list)):
rho = self.rho_list[i]
LT = sp.vstack((rho ** 0.5 * self.vec_ones, (1 - rho) ** 0.5 * self.Env.T))
LTxoPy = sp.dot(LT, X * Py)
Q_rho[i] = 0.5 * sp.dot(LTxoPy.T, LTxoPy)
# Calculating pvs is split into 2 steps
# If we only consider one value of rho i.e. equivalent to SKAT and used for interaction test
if len(self.rho_list) == 1:
rho = self.rho_list[0]
L = sp.hstack((rho ** 0.5 * self.vec_ones.T, (1 - rho) ** 0.5 * self.Env))
xoL = X * L
PxoL = P(self.gp, xoL)
LToxPxoL = 0.5 * sp.dot(xoL.T, PxoL)
try:
pval = davies_pvalue(Q_rho[0], LToxPxoL)
except AssertionError:
eighQ, UQ = la.eigh(LToxPxoL)
pval = mod_liu_corrected(Q_rho[0], eighQ)
# Script ends here for interaction test
return pval
# or if we consider multiple values of rho i.e. equivalent to SKAT-O and used for association test
else:
pliumod = sp.zeros((len(self.rho_list), 4))
for i in range(len(self.rho_list)):
rho = self.rho_list[i]
L = sp.hstack(
(rho ** 0.5 * self.vec_ones.T, (1 - rho) ** 0.5 * self.Env)
)
xoL = X * L
PxoL = P(self.gp, xoL)
LToxPxoL = 0.5 * sp.dot(xoL.T, PxoL)
eighQ, UQ = la.eigh(LToxPxoL)
pliumod[i,] = mod_liu_corrected(Q_rho[i], eighQ)
T = pliumod[:, 0].min()
# if optimal_rho == 0.999:
# optimal_rho = 1
# 2. Calculate qmin
qmin = sp.zeros(len(self.rho_list))
percentile = 1 - T
for i in range(len(self.rho_list)):
q = st.chi2.ppf(percentile, pliumod[i, 3])
# Recalculate p-value for each Q rho of seeing values at least as extreme as q again using the modified matching moments method
qmin[i] = (q - pliumod[i, 3]) / (2 * pliumod[i, 3]) ** 0.5 * pliumod[
i, 2
] + pliumod[i, 1]
# 3. Calculate quantites that occur in null distribution
Px1 = P(self.gp, X)
m = 0.5 * sp.dot(X.T, Px1)
xoE = X * self.Env
PxoE = P(self.gp, xoE)
ETxPxE = 0.5 * sp.dot(xoE.T, PxoE)
ETxPx1 = sp.dot(xoE.T, Px1)
ETxPx11xPxE = 0.25 / m * sp.dot(ETxPx1, ETxPx1.T)
ZTIminusMZ = ETxPxE - ETxPx11xPxE
eigh, vecs = la.eigh(ZTIminusMZ)
eta = sp.dot(ETxPx11xPxE, ZTIminusMZ)
vareta = 4 * sp.trace(eta)
OneZTZE = 0.5 * sp.dot(X.T, PxoE)
tau_top = sp.dot(OneZTZE, OneZTZE.T)
tau_rho = sp.zeros(len(self.rho_list))
for i in range(len(self.rho_list)):
tau_rho[i] = self.rho_list[i] * m + (1 - self.rho_list[i]) / m * tau_top
MuQ = sp.sum(eigh)
VarQ = sp.sum(eigh ** 2) * 2 + vareta
KerQ = sp.sum(eigh ** 4) / (sp.sum(eigh ** 2) ** 2) * 12
Df = 12 / KerQ
# 4. Integration
# from time import time
# start = time()
pvalue = optimal_davies_pvalue(
qmin, MuQ, VarQ, KerQ, eigh, vareta, Df, tau_rho, self.rho_list, T
)
# print("Elapsed: {} seconds".format(time() - start))
# Final correction to make sure that the p-value returned is sensible
multi = 3
if len(self.rho_list) < 3:
multi = 2
idx = sp.where(pliumod[:, 0] > 0)[0]
pval = pliumod[:, 0].min() * multi
if pvalue <= 0 or len(idx) < len(self.rho_list):
pvalue = pval
if pvalue == 0:
if len(idx) > 0:
pvalue = pliumod[:, 0][idx].min()
if debug:
info = {
"Qs": Q_rho,
"pvs_liu": pliumod,
"qmin": qmin,
"MuQ": MuQ,
"VarQ": VarQ,
"KerQ": KerQ,
"lambd": eigh,
"VarXi": vareta,
"Df": Df,
"tau": tau_rho,
}
return pvalue, info
else:
return pvalue
def labelconsistentksvd(self,
Y,
Dinit,
labels,
Q_train,
Tinit,
Winit=None):
"""
Label consistent KSVD1 algorithm and Discriminative LC-KSVD2 implementation
Args:
Y : training features
Dinit : initialized dictionary
labels : labels matrix for training feature (numberred from 1 to nb of classes)
Q_train : optimal code matrix for training feature
Tinit : initialized transform matrix
Winit : initialized classifier parameters (None for LC-KSVD1)
Returns:
D : learned dictionary
X : sparsed codes
T : learned transform matrix
W : learned classifier parameters
"""
# H_train = sp.zeros((int(labels.max()), Y.shape[1]), dtype=float)
# print(H_train.shape)
# for c in range(int(labels.max())):
# H_train[c, labels == (c+1)] = 1.
H_train = labels
# ksvd process
runKsvd = ApproximateKSVD(Dinit.shape[1],
max_iter=self.iterations,
tol=self.tol,
transform_n_nonzero_coefs=self.sparsitythres)
if Winit is None:
runKsvd.fit(sp.vstack((Y, self.sqrt_alpha * Q_train)),
Dinit=normcols(
sp.vstack((Dinit, self.sqrt_alpha * Tinit))))
else:
runKsvd.fit(sp.vstack(
(Y, self.sqrt_alpha * Q_train, self.sqrt_beta * H_train)),
Dinit=normcols(
sp.vstack((Dinit, self.sqrt_alpha * Tinit,
self.sqrt_beta * Winit))))
# get back the desired D, T and W (if sqrt_beta is not None)
i_end_D = Dinit.shape[0]
i_start_T = i_end_D
i_end_T = i_end_D + Tinit.shape[0]
D = runKsvd.components_[:i_end_D, :]
T = runKsvd.components_[i_start_T:i_end_T, :]
if Winit is not None:
i_start_W = i_end_T
i_end_W = i_end_T + Winit.shape[0]
W = runKsvd.components_[i_start_W:i_end_W, :]
# normalization
l2norms = splin.norm(D, axis=0)[sp.newaxis, :] + self.tol
D /= l2norms
T /= l2norms
T /= self.sqrt_alpha
X = runKsvd.gamma_
if Winit is None:
# Learning linear classifier parameters
xxt = X.dot(X.T)
W = splin.pinv(xxt + sp.eye(*(xxt).shape)).dot(X).dot(H_train.T)
W = W.T
else:
W /= l2norms
W /= self.sqrt_beta
return D, X, T, W
def test1():
zs = np.linspace(0.01, 1, 21)
r = np.array([[0, 0, zz] for zz in zs])
thetas = np.linspace(0, np.pi * 0.25, 10)
print "thetas: ", thetas
k_dir = np.vstack([np.sin(thetas), np.zeros_like(thetas), np.cos(thetas)])
k_dir = k_dir.transpose()
wavelength = 10
k0 = np.pi * 2 / wavelength
a1 = np.array([1, 0, 0])
a2 = np.array([0, 1, 0])
class gratinglobes(object):
def check(self):
dx = np.sqrt(np.sum(a1 * a1, axis=-1))
dy = np.sqrt(np.sum(a2 * a2, axis=-1))
threshold_d = wavelength / (1 + np.sin(thetas))
temp = np.array([dx < threshold_d, dy < threshold_d])
return np.sum(temp, axis=0) == temp.shape[0]
checker = gratinglobes().check()
print "grating lobe condition: ", checker
result_direct = PGF_Direct(k0, a1, a2, 100, 100).pgf(k_dir, r)
result_poisson = PGF_Poisson(k0, a1, a2, 1, 1).pgf(k_dir, r)
result_ewald = PGF_EWALD(k0, a1, a2, 20, 20).pgf(k_dir, r)
import matplotlib.pylab as plt
plt.figure()
class Method(object):
def __init__(self, result, marker, line):
self.result = result
self.marker = marker
self.line = line
def angle(self, it):
plt.plot(zs,\
np.angle(self.result)[it]/np.pi*180,\
self.line,\
label='angle %d %s'%(it,self.marker))
def absolute(self, it):
plt.plot(zs,\
np.absolute(self.result)[it],\
self.line,\
label='abs %d %s'%(it,self.marker))
map(Method(result_poisson, 'pois', "s").angle, xrange(k_dir.shape[0]))
map(Method(result_direct, 'dir', "-").angle, xrange(k_dir.shape[0]))
map(Method(result_ewald, 'ewald', "+").angle, xrange(k_dir.shape[0]))
plt.ylabel("angle (degree)")
plt.xlabel("zs")
#plt.legend()
plt.show()
plt.figure()
map(Method(result_poisson, 'pois', "s").absolute, xrange(k_dir.shape[0]))
map(Method(result_direct, 'dir', "-").absolute, xrange(k_dir.shape[0]))
map(Method(result_ewald, 'ewald', "+").absolute, xrange(k_dir.shape[0]))
plt.xlabel("zs")
plt.ylabel("log10(abs) ")
#plt.legend()
plt.show()
plt.figure()
#map(Method((result_direct-result_poisson)/result_direct,'dir_pois',"-s").absolute,xrange(k_dir.shape[0]))
map(
Method((result_direct - result_ewald) / result_direct, 'dir_ewald',
"-d").absolute, xrange(k_dir.shape[0]))
#map(Method((result_poisson-result_ewald)/result_ewald,'pois_ewald',"-o").absolute,xrange(k_dir.shape[0]))
plt.xlabel("zs")
plt.ylabel("log10(abs) ")
#plt.legend()
plt.show()
i = i + 1
u1 = (Wbig < 0) * -Wbig + Wbig
u1 = sp.sqrt(u1)
u1 = (Wbig < 0) * -10**10 + u1
x, y = discretenorm(k, 4 * sp.sqrt(.25), .25)
for j in range(k):
u[:, :, j] = x[j] * u1
b = .9
states = sp.zeros((1, 100, 7))
policy = sp.zeros((1, 100, 7))
i = 0
d = 1
while d > 10**-9:
E = states[i, :, :] * y
V = sp.vstack(E.sum(1)) * b
Value1 = V.copy()
for x in range(100 - 1):
Value1 = sp.concatenate((Value1, V), 1)
Value = sp.zeros((100, 100, 7))
for x in range(7):
Value[:, :, x] = Value1.T
total = u + Value
temp = total.max(1)
temp.resize(1, 100, 7)
temp1 = total.argmax(1)
temp1.resize(1, 100, 7)
states = sp.concatenate((states, temp), 0)
policy = sp.concatenate((policy, temp1), 0)
i = i + 1
d = la.norm(states[i - 1, :, :] - states[i, :, :])
def parcellate_region(roilist,
sub,
nClusters,
scan,
scan_type,
savepng=0,
session=1,
algo=0,
type_cor=0):
p_dir = '/big_disk/ajoshi/HCP100-fMRI-NLM/HCP100-fMRI-NLM'
r_factor = 3
ref_dir = os.path.join(p_dir, 'reference')
ref = '100307'
fn1 = ref + '.reduce' + str(r_factor) + '.LR_mask.mat'
fname1 = os.path.join(ref_dir, fn1)
msk = scipy.io.loadmat(fname1)
dfs_left_sm = readdfs(
os.path.join('/home/ajoshi/for_gaurav',
'100307.BCI2reduce3.very_smooth.' + scan_type + '.dfs'))
dfs_left = readdfs(
os.path.join('/home/ajoshi/for_gaurav',
'100307.BCI2reduce3.very_smooth.' + scan_type + '.dfs'))
data = scipy.io.loadmat(
os.path.join(
p_dir, sub, sub + '.rfMRI_REST' + str(session) + scan +
'.reduce3.ftdata.NLM_11N_hvar_25.mat'))
LR_flag = msk['LR_flag']
# 0= right hemisphere && 1== left hemisphere
if scan_type == 'right':
LR_flag = np.squeeze(LR_flag) == 0
else:
LR_flag = np.squeeze(LR_flag) == 1
data = data['ftdata_NLM']
temp = data[LR_flag, :]
m = np.mean(temp, 1)
temp = temp - m[:, None]
s = np.std(temp, 1) + 1e-16
temp = temp / s[:, None]
msk_small_region = np.in1d(dfs_left.labels, roilist)
# (dfs_left.labels == 46) | (dfs_left.labels == 28) \
# | (dfs_left.labels == 29) # % motor
d = temp[msk_small_region, :]
rho = np.corrcoef(d)
rho[~np.isfinite(rho)] = 0
# rho=np.abs(rho)
d_corr = temp[~msk_small_region, :]
rho_1 = np.corrcoef(d, d_corr)
rho_1 = rho_1[range(d.shape[0]), d.shape[0]:]
rho_1[~np.isfinite(rho_1)] = 0
if type_cor == 1:
f_rho = np.arctanh(rho_1)
f_rho[~np.isfinite(f_rho)] = 0
B = np.corrcoef(f_rho)
B[~np.isfinite(B)] = 0
# B = np.abs(B)
# SC = DBSCAN()
if algo == 0:
SC = SpectralClustering(n_clusters=nClusters, affinity='precomputed')
# SC=SpectralClustering(n_clusters=nClusters,gamma=0.025)
if type_cor == 0 and rho.size > 0:
# affinity_matrix = affinity_mat(rho)
affinity_matrix = np.arcsin(rho)
labels = SC.fit_predict(np.abs(affinity_matrix))
if type_cor == 1 and rho.size > 0:
labels = SC.fit_predict(affinity_mat(B))
# affinity_matrix=SC.fit(np.abs(d))
elif algo == 1:
g = nx.Graph()
g.add_edges_from(dfs_left.faces[:, (0, 1)])
g.add_edges_from(dfs_left.faces[:, (1, 2)])
g.add_edges_from(dfs_left.faces[:, (2, 0)])
Adj = nx.adjacency_matrix(g)
AdjS = Adj[(msk_small_region), :]
AdjS = AdjS[:, (msk_small_region)]
AdjS = AdjS.todense()
np.fill_diagonal(AdjS, 1)
SC = AgglomerativeClustering(n_clusters=nClusters, connectivity=AdjS)
labels = SC.fit_predict(rho)
elif algo == 2:
GM = GMM(n_components=nClusters, covariance_type='full', n_iter=100)
GM.fit(rho)
labels = GM.predict(rho)
elif algo == 3:
neighbour_correlation(rho, dfs_left_sm.faces, dfs_left_sm.vertices,
msk_small_region)
if savepng > 0:
r = dfs_left_sm
r.labels = np.zeros([r.vertices.shape[0]])
r.labels[msk_small_region] = labels + 1
cent = separate(labels, r, r.vertices, nClusters)
manual_order = np.array([0 for x in range(nClusters)])
save = np.array([0 for x in range(nClusters)])
for i in range(0, nClusters):
if nClusters > 1:
choose_vector = np.argmax(cent.transpose(), axis=1)
save[i] = cent[choose_vector[1]][1]
correspondence_point = find_location_smallmask(
r.vertices, cent[choose_vector[1]], msk_small_region)
cent[choose_vector[1]][1] = -np.Inf
manual_order[i] = choose_vector[1]
if i == 0:
# change
correlation_within_precuneus_vector = sp.array(
rho[correspondence_point])
correlation_with_rest_vector = sp.array(
rho_1[correspondence_point])
else:
correlation_within_precuneus_vector = sp.vstack([
correlation_within_precuneus_vector,
[rho[correspondence_point]]
])
correlation_with_rest_vector = sp.vstack([
correlation_with_rest_vector,
[rho_1[correspondence_point]]
])
else:
choose_vector = 0
correspondence_point = find_location_smallmask(
r.vertices, cent, msk_small_region)
manual_order[i] = choose_vector
if i == 0:
# change
correlation_within_precuneus_vector = sp.array(
rho[correspondence_point])
correlation_with_rest_vector = sp.array(
rho_1[correspondence_point])
manual_order = change_order(manual_order, nClusters)
r.labels = change_labels(r.labels, manual_order, nClusters)
new_cent = separate(r.labels, r, temp, nClusters)
if nClusters > 1:
for i in range(0, nClusters):
cent[manual_order[i]][1] = save[i]
return (r, correlation_within_precuneus_vector,
correlation_with_rest_vector, msk_small_region, new_cent)
@author: 56977
"""
import scipy as sp
import matplotlib.pyplot as plt
data = sp.loadtxt("datos2_corrida0.txt")
N = data[:, 0]
dts = data[:, 1]
mem = data[:, 2]
for i in range(1, 10):
data = sp.loadtxt(f"datos2_corrida{i}.txt")
dts = sp.vstack((dts, data[:, 1]))
absisas = [10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000]
TPO = [0.1e-3, 1e-3, 1e-2, 0.1, 1., 10., 60., 60 * 10]
TPO_label = [
"0.1 ms", "1 ms", "10 ms", "0.1 s", "1 s", "10 s", "1 min", "10 min"
]
RAM = [10**3, 10**4, 10**5, 10**6, 10**7, 10**8, 10**9, 10**10, 10**11]
RAM_label = [
"1 KB", "10 KB", "100 KB", "1 MB", "10 MB", "100 MB", "1 GB", "10 GB", ""
]
plt.figure()
temp=np.ones((cMshape[1],cMshape[2],cMshape[3]+1)) for i in range(cMshape[1]): print 'Shape check:',np.shape(temp[i,:,:2]),np.shape(contractionMatrix[0][i]) temp[i,:,:2]=(contractionMatrix[0][i])-256 print 'zPos:',contourInformation[i][0] temp[i,:,2]=temp[i,:,2]*np.float(contourInformation[i][0]) #temp[i,:,2]=temp[i,:,2]*-np.float(contourInformation[i][0])#-375 #Read in any fixed points fixedContour1=np.loadtxt(workingDir+'FixedContours/InletFixedContour1') fixedContour2=np.loadtxt(workingDir+'FixedContours/InletFixedContour2') fixedContour3=np.loadtxt(workingDir+'FixedContours/FixedContour1') fixedContour4=np.loadtxt(workingDir+'FixedContours/OutletFixedContour1') fixedContour5=np.loadtxt(workingDir+'FixedContours/OutletFixedContour2') fixedContours=scipy.vstack([fixedContour1,fixedContour2,fixedContour3,fixedContour4,fixedContour5]) print 'countour shape:',np.shape(temp.reshape((cMshape[1]*cMshape[2],cMshape[3]+1))) #print 'fixedcontour shape',fixedContour.shape source_points_fitting = temp.reshape((cMshape[1]*cMshape[2],cMshape[3]+1)) #With fixed contours #source_points_fitting = scipy.vstack([temp.reshape((cMshape[1]*cMshape[2],cMshape[3]+1)),fixedContours]) ######################################################################################################### #Loop through time points for timePoint in range(11,12):#range(20,cMshape[0]): print 'fitting time: ',timePoint temp=np.ones((cMshape[1],cMshape[2],cMshape[3]+1))
def vl_phow(im,
verbose=True,
fast=True,
sizes=[4, 6, 8, 10],
step=2,
color='rgb',
floatdescriptors=False,
magnif=6,
windowsize=1.5,
contrastthreshold=0.005):
opts = Options(verbose, fast, sizes, step, color, floatdescriptors, magnif,
windowsize, contrastthreshold)
dsiftOpts = DSiftOptions(opts)
# make sure image is float, otherwise segfault
im = array(im, 'float32')
# Extract the features
imageSize = shape(im)
if im.ndim == 3:
if imageSize[2] < 3: #if imageSize[2] != 3:
# "IndexError: tuple index out of range" if both if's are checked at the same time
raise ValueError("Image data in unknown format/shape")
if opts.color == 'gray':
numChannels = 1
if (im.ndim == 2):
im = vl_rgb2gray(im)
else:
numChannels = 3
if (im.ndim == 2):
im = dstack([im, im, im])
if opts.color == 'rgb':
pass
elif opts.color == 'opponent':
# from https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_phow.m
# Note that the mean differs from the standard definition of opponent
# space and is the regular intesity (for compatibility with
# the contrast thresholding).
# Note also that the mean is added pack to the other two
# components with a small multipliers for monochromatic
# regions.
mu = 0.3 * im[:, :, 0] + 0.59 * im[:, :, 1] + 0.11 * im[:, :, 2]
alpha = 0.01
im = dstack([
mu, (im[:, :, 0] - im[:, :, 1]) / sqrt(2) + alpha * mu,
(im[:, :, 0] + im[:, :, 1] - 2 * im[:, :, 2]) / sqrt(6) +
alpha * mu
])
else:
raise ValueError('Color option ' + str(opts.color) +
' not recognized')
if opts.verbose:
print('{0}: color space: {1}'.format('vl_phow', opts.color))
print('{0}: image size: {1} x {2}'.format('vl_phow', imageSize[0],
imageSize[1]))
print('{0}: sizes: [{1}]'.format('vl_phow', opts.sizes))
frames_all = []
descrs_all = []
for size_of_spatial_bins in opts.sizes:
# from https://github.com/vlfeat/vlfeat/blob/master/toolbox/sift/vl_phow.m
# Recall from VL_DSIFT() that the first descriptor for scale SIZE has
# center located at XC = XMIN + 3/2 SIZE (the Y coordinate is
# similar). It is convenient to align the descriptors at different
# scales so that they have the same geometric centers. For the
# maximum size we pick XMIN = 1 and we get centers starting from
# XC = 1 + 3/2 MAX(OPTS.SIZES). For any other scale we pick XMIN so
# that XMIN + 3/2 SIZE = 1 + 3/2 MAX(OPTS.SIZES).
# In pracrice, the offset must be integer ('bounds'), so the
# alignment works properly only if all OPTS.SZES are even or odd.
off = floor(3.0 / 2 * (max(opts.sizes) - size_of_spatial_bins)) + 1
# smooth the image to the appropriate scale based on the size
# of the SIFT bins
sigma = size_of_spatial_bins / float(opts.magnif)
ims = vl_imsmooth(im, sigma)
# extract dense SIFT features from all channels
frames = []
descrs = []
for k in range(numChannels):
size_of_spatial_bins = int(size_of_spatial_bins)
# vl_dsift does not accept numpy.int64 or similar
f_temp, d_temp = vl_dsift(data=ims[:, :, k],
step=dsiftOpts.step,
size=size_of_spatial_bins,
fast=dsiftOpts.fast,
verbose=dsiftOpts.verbose,
norm=dsiftOpts.norm,
bounds=[off, off, maxint, maxint])
frames.append(f_temp)
descrs.append(d_temp)
frames = array(frames)
descrs = array(descrs)
d_new_shape = [descrs.shape[0] * descrs.shape[1], descrs.shape[2]]
descrs = descrs.reshape(d_new_shape)
# remove low contrast descriptors
# note that for color descriptors the V component is
# thresholded
if (opts.color == 'gray') | (opts.color == 'opponent'):
contrast = frames[0][2, :]
elif opts.color == 'rgb':
contrast = mean(
[frames[0][2, :], frames[1][2, :], frames[2][2, :]], 0)
else:
raise ValueError('Color option ' + str(opts.color) +
' not recognized')
descrs[:, contrast < opts.contrastthreshold] = 0
# save only x,y, and the scale
frames_temp = array(frames[0][0:3, :])
padding = array(size_of_spatial_bins * ones(frames[0][0].shape))
frames_all.append(vstack([frames_temp, padding]))
descrs_all.append(array(descrs))
frames_all = hstack(frames_all)
descrs_all = hstack(descrs_all)
return frames_all, descrs_all
def globallocalregret(optstate, persist, **para):
#doublenormdist
#norprior
if persist == None:
persist = defaultdict(list)
persist['raiseS'] = False
persist['R'] = sp.eye(len(para['lb']))
if optstate.n < para['onlyafter']:
return 0, persist, dict()
if persist['flip']:
return 1, persist, dict()
logging.info('globallocalregretchooser with {} inflated diagonal'.format(
persist['raiseS']))
if para['rotate']:
logging.info('rotate\n{}'.format(persist['R']))
d = len(para['lb'])
lb = para['lb']
ub = para['ub']
#build a GP with slice-samples hypers
x = sp.vstack(optstate.x).dot(persist['R'].T)
y = sp.vstack(optstate.y)
s = sp.vstack([e['s'] + 10**optstate.condition for e in optstate.ev])
dx = [e['d'] for e in optstate.ev]
logger.info('building GP')
G = PES.makeG(x,
y,
s,
dx,
para['kindex'],
para['mprior'],
para['sprior'],
para['nhyp'],
prior=para['priorshape'])
xminr, ymin, ierror = gpbo.core.optutils.twopartopt(
lambda x: G.infer_m_post(persist['R'].dot(x.flatten()).reshape(
[1, d]), [[sp.NaN]])[0, 0], para['lb'], para['ub'], para['dpara'],
para['lpara'])
xmin = persist['R'].dot(xminr.flatten()).reshape([1, d])
#:mxmin,vxmin = [j[0,0] for j in G.infer_diag_post(optstate.x[0],[[sp.NaN]])]
logger.info('post min at {}(true) {}(rotated) is {}'.format(
xminr, xmin, ymin))
dropdims = []
for i in range(d):
if xminr[i] > 0.995 * (ub[i] - lb[i]) + lb[i] or xminr[i] < lb[i] + (
1. - 0.995) * (ub[i] - lb[i]):
dropdims.append(i)
if not sp.allclose(sp.eye(d), persist['R']):
print('edge isn\'t working with nonzero rotation')
logger.info('post min in on edge in axes {}'.format(dropdims))
#get hessian/grad posterior
#local probmin elipse at post min
GH = gpbo.core.optutils.gpGH(G, xmin)
Gr, cG, H, Hvec, varHvec, M, varM = GH
#est the local regret
Mdraws = gpbo.core.GPdc.draw(M[0, :], varM, 200)
lrest = 0.
for i in xrange(200):
sM = Mdraws[i, :]
sG = sM[:d]
sH = gpbo.core.optutils.Hvec2H(sM[d:], d)
sR = 0.5 * sG.dot(sp.linalg.solve(sH, sG))
lrest += max(0., sR)
lrest /= 200.
logger.info('localregretest {}'.format(lrest))
m = sp.diag(H)
v = sp.diag(gpbo.core.optutils.Hvec2H(sp.diagonal(varHvec), d))
logger.debug('H,stH\n{}\n{}'.format(
H, sp.sqrt(gpbo.core.optutils.Hvec2H(sp.diagonal(varHvec), d))))
logger.debug('axisprobs {}'.format(
1. - sp.stats.norm.cdf(sp.zeros(d), loc=m, scale=sp.sqrt(v))))
#step out to check +ve defininteness
logger.info('checking for +ve definite ball')
from gpbo.core import debugoutput
pc = gpbo.core.optutils.probgppve(G,
sp.array(xmin),
tol=para['pvetol'],
dropdims=dropdims)
logger.info('prob pvedef at xmin {}'.format(pc))
_ = prob(G, sp.array(xmin), tol=para['pvetol'], dropdims=dropdims)
if para['rotate']:
#U,S,V = sp.linalg.svd(H)
eva, eve = sp.linalg.eigh(H)
V = eve.T
persist['R'] = V.dot(persist['R'])
mask = sp.ones(d)
for i in dropdims:
mask[i] = 0.
def PDcondition(x):
P = gpbo.core.optutils.probgppve(G,
sp.array(x) * mask + sp.array(xmin),
tol=para['pvetol'],
dropdims=dropdims)
C = P > 1 - para['pvetol']
#print(C,P,sp.array(x)*mask)
return C
#todo assuming unit radius search region for Rinit=1
rmax = gpbo.core.optutils.ballradsearch(d,
1.,
PDcondition,
ndirs=para['nlineS'],
lineSh=para['lineSh'])
if gpbo.core.debugoutput['adaptive']:
import matplotlib
matplotlib.rcParams['text.usetex'] = False
fig, ax = plt.subplots(nrows=3, ncols=4, figsize=(85, 85))
xmin = xmin.flatten()
# plot the current GP
if d == 2:
#gpbo.core.optutils.gpplot(ax[0,0],ax[0,1],G,para['lb'],para['ub'],ns=60)
ax[0, 0].set_title('GP post mean')
ax[0, 1].set_title('GP post var')
ax[0, 0].plot(xmin[0], xmin[1], 'ro')
#plot some draws from H
for i in xrange(4):
Gm, Gv, Hd = gpbo.core.drawconditionH(*GH)
try:
sp.linalg.cholesky(Hd)
gpbo.core.optutils.plotprobstatellipse(Gv,
Hd,
xmin,
ax[1, 1],
logr=True)
except sp.linalg.LinAlgError:
pass
if rmax > 0:
ax[1, 1].plot([sp.log10(rmax)] * 2, [0., 2 * sp.pi], 'purple')
else:
logger.debug('plotting some draws...')
#draw support points
xvmaxr, vmax, ierror = gpbo.core.optutils.twopartopt(
lambda x: -G.infer_diag_post(persist['R'].dot(x.flatten(
)), [[sp.NaN]])[1][0, 0], para['lb'], para['ub'],
para['dpara'], para['lpara'])
xvmax = persist['R'].dot(xvmaxr.flatten())
mvmax, vvmax = [
j[0, 0] for j in G.infer_diag_post(xvmax, [[sp.NaN]])
]
W = sp.vstack([
ESutils.draw_support(G,
lb,
ub,
para['support'] / 2,
ESutils.SUPPORT_LAPAPROT,
para=20,
rotation=persist['R']),
ESutils.draw_support(G,
lb,
ub,
para['support'] / 2,
ESutils.SUPPORT_VARREJ,
para=vvmax,
rotation=persist['R'])
])
nd = 1500
#draw mins and value of g at xmin as pair
R, Y, A = ESutils.draw_min_xypairgrad(G, W, nd, xmin)
#plot support
if d == 2:
gpbo.core.optutils.plotaslogrtheta(W[:, 0], W[:, 1], xmin[0],
xmin[1], ax[1, 1], 'b.')
ax[0, 2].plot(W[:, 0], W[:, 1], 'b.')
#plot mindraws
gpbo.core.optutils.plotaslogrtheta(R[:, 0], R[:, 1], xmin[0],
xmin[1], ax[1, 1], 'r.')
ax[0, 2].plot(R[:, 0], R[:, 1], 'r.')
ax[1, 3].text(0, 0, 'prob +ve at min {}\nR+ve {}'.format(pc, rmax))
if rmax == 0:
if gpbo.core.debugoutput['adaptive']:
try:
fname = 'lotsofplots' + time.strftime(
'%d_%m_%y_%H:%M:%S') + '.png'
print('saving as {}'.format(fname))
fig.savefig(os.path.join(gpbo.core.debugoutput['path'], fname))
except BaseException as e:
logger.error(str(e))
fig.clf()
plt.close(fig)
del (fig)
logger.info('no +ve def region, choosereturns 0')
return 0, persist, {
'reuseH': [k.hyp for k in G.kf],
'ppveatx': pc,
'rpve': rmax,
'R': persist['R']
}
xvmaxr, vmax, ierror = gpbo.core.optutils.twopartopt(
lambda x: -G.infer_diag_post(persist['R'].dot(x.flatten()), [[sp.NaN]])
[1][0, 0], para['lb'], para['ub'], para['dpara'], para['lpara'])
xvmax = persist['R'].dot(xvmaxr.flatten())
mvmax, vvmax = [j[0, 0] for j in G.infer_diag_post(xvmax, [[sp.NaN]])]
logger.info('post var max {} at {} with mean {}'.format(
vvmax, xvmax, mvmax))
#draw support points
W = sp.vstack([
ESutils.draw_support(G,
lb,
ub,
para['support'] / 2,
ESutils.SUPPORT_LAPAPROT,
para=20,
weighted=para['weighted'],
rotation=persist['R']),
ESutils.draw_support(G,
lb,
ub,
para['support'] / 2,
ESutils.SUPPORT_VARREJ,
para=vvmax,
rotation=persist['R'])
])
Q, maxRin = drawpartitionmin2(G, W, xmin, rmax, para['draws'])
logger.info('+ve region radius {} max sample radius {}'.format(
rmax, maxRin))
#pcurves from Q
def data2cdf(X):
n = X.size
C = sp.linspace(1. / n, 1, n)
XC = sorted(X)
return XC, C
Yin, Cin = data2cdf(Q[:, 1])
normin = sp.stats.norm.fit(Yin)
Yat, Cat = data2cdf(Q[:, 3])
normat = sp.stats.norm.fit(Yat)
Yout, Cout = data2cdf(Q[:, 2])
#normal dist with var same as max in gp model and passing through estimated prob of min sample
ydrawmin = Yout[0]
cdfymin = Cout[0]
mu = ydrawmin - sp.sqrt(vvmax * 2) * sp.special.erfinv(2 * cdfymin - 1.)
logger.info('upper norm at y {} c {} has mu {},var {}'.format(
ymin, cdfymin, mu, vvmax))
logger.info('lower norm at x {} has mu {},var {}'.format(
xvmax, mvmax, vvmax))
#interpolator for cdf
def splicecdf(y):
if y < Yout[0]:
return sp.stats.norm.cdf(y, loc=mu, scale=sp.sqrt(vvmax))
elif y >= Yout[-1]:
return 1. - 1e-20
else:
i = 0
while Yout[i] < y:
i += 1
return Cout[i]
return
m, std = normin
logger.debug(
'inner approx m{} std{}\noutsample stats min{} max{} mean{}'.format(
m, std,
sp.array(Yout).min(),
sp.array(Yout).max(), sp.mean(Yout)))
racc = 0.
n = len(Cout)
#regret from samples after the min
for i in xrange(1, n):
racc += gpbo.core.GPdc.EI(-Yout[i], -m, std)[0, 0] / float(n)
tmp = racc
#regret from the tail bound
I, err = spi.quad(
lambda y: gpbo.core.GPdc.EI(-y, -m, std)[0, 0] * sp.stats.norm.pdf(
y, mu, sp.sqrt(vvmax)), -sp.inf, Yout[0])
racc += I
logger.info('outer regret {} (due to samples: {} due to tail: {}'.format(
racc, tmp, racc - tmp))
#regret lower bound
#rlow,err = spi.quad(lambda y:gpbo.core.GPdc.EI(-y,-m,v)[0,0]*sp.stats.norm.pdf(y,mvmax,sp.sqrt(vvmax)),-sp.inf,mvmax)
#regret from samples
rsam = 0.
for i in xrange(Q.shape[0]):
rsam += max(0., Q[i, 1] - Q[i, 2])
rsam /= Q.shape[0]
#local regret from incumbent from samples
rloc = 0.
for i in xrange(Q.shape[0]):
rloc += max(0., Q[i, 3] - Q[i, 1])
rloc /= Q.shape[0]
persist['localsampleregret'].append(rloc)
#set switch to local if condition achieved
if racc < para['regretswitch']:
rval = 1
persist['flip'] = True
optstate.startlocal = xmin
elif maxRin < 0.9 * rmax:
rval = 2
else:
rval = 0
persist['flip'] = False
if gpbo.core.debugoutput['adaptive']:
if d == 2:
gpbo.core.optutils.plotaslogrtheta(W[:, 0], W[:, 1], xmin[0],
xmin[1], ax[1, 1], 'b.')
ax[0, 2].plot(W[:, 0], W[:, 1], 'b.')
#plot mindraws
R, Y, A = ESutils.draw_min_xypairgrad(G, W, 1500, xmin)
gpbo.core.optutils.plotaslogrtheta(R[:, 0], R[:, 1], xmin[0],
xmin[1], ax[1, 1], 'r.')
ax[0, 2].plot(R[:, 0], R[:, 1], 'r.')
ax[2, 2].plot(Q[:, 1], Q[:, 2], 'r.')
ax[2, 2].set_xlabel('inR')
ax[2, 2].set_ylabel('outR')
ax[2, 2].plot([ymin], [ymin], 'go')
ax[2, 1].plot(Q[:, 1], Q[:, 3], 'r.')
ax[2, 1].set_xlabel('inR')
ax[2, 1].set_ylabel('atArg')
ax[2, 1].plot([ymin], [ymin], 'go')
def pltcdf(Y, C, ax, col):
return ax.plot(sp.hstack([[i, i] for i in Y])[1:-1],
sp.hstack([[i - C[0], i] for i in C])[1:-1],
color=col,
label='Sampled CDF')
pltcdf(Yin, Cin, ax[2, 0], 'b')
rin = sp.linspace(Yin[0], Yin[-1], 150)
ax[2, 0].plot(rin, map(lambda x: sp.stats.norm.cdf(x, *normin), rin),
'k')
pltcdf(Yat, Cat, ax[2, 0], 'g')
rat = sp.linspace(Yat[0], Yat[-1], 150)
ax[2, 0].plot(rat, map(lambda x: sp.stats.norm.cdf(x, *normat), rat),
'k')
ax[2, 0].set_yscale('logit')
pltcdf(Yout, Cout, ax[1, 0], 'r')
ax[1, 0].set_yscale('logit')
rl = min(Yout)
ru = max(Yout)
sup = sp.linspace(rl - 0.25 * (ru - rl), 0.5 * (rl + ru), 50)
ax[1, 0].plot(sup,
sp.stats.norm.cdf(sup, loc=mu, scale=sp.sqrt(vvmax)),
'b--',
label='Approximate Tail Upper Bound')
ax[1, 0].plot(sup,
sp.stats.norm.cdf(sup, loc=mvmax, scale=sp.sqrt(vvmax)),
'g--',
label='Lower Bound')
ax[1, 0].axvline(ymin)
if True:
f2, a2 = plt.subplots(figsize=[8, 5])
pltcdf(Yout, Cout, a2, 'r')
a2.set_yscale('logit')
a2.plot(sup,
sp.stats.norm.cdf(sup, loc=mu, scale=sp.sqrt(vvmax)),
color='b',
linestyle='--',
label='Approx Tail Upper Bound')
a2.plot(sup,
sp.stats.norm.cdf(sup, loc=mvmax, scale=sp.sqrt(vvmax)),
color='b',
linestyle='-.',
label='Lower Bound')
a2.axvline(ymin,
label='Posterior Mean Minimum',
color='k',
linestyle=':')
a2.set_ylabel('CDF')
a2.set_xlabel('y')
from matplotlib.ticker import NullFormatter
a2.yaxis.set_minor_formatter(NullFormatter())
a2.spines['left']._adjust_location()
a2.legend()
f2.savefig(os.path.join(debugoutput['path'], 'ends.png'))
plt.close(f2)
import pickle
pickle.dump([sup, mu, vvmax, mvmax, ymin, Yout, Cout],
open('results/bounddata.p', 'w'))
mxo = Yout[-1]
mno = Yout[0]
ro = sp.linspace(min(mno - 0.05 * (mxo - mno), ymin),
mxo + 0.05 * (mxo - mno), 200)
ax[1, 2].text(0, 0.34, 'regretg sample {}'.format(rsam))
ax[1, 2].text(0, 0.24, 'regretg tailest {}'.format(racc))
#ax[1,2].text(0,0.18, 'regretg binormest {}'.format(rbin))
#ax[1,2].text(0,0.08, 'regretg lowerb {} '.format(rlow))
ax[1, 2].text(0, 0.5, 'maxRin {} / {}'.format(maxRin, rmax))
ax[1, 2].text(0, 0.6, 'mode {}'.format(rval))
ax[1, 2].text(0, 0.74, 'localr sample {}'.format(rloc))
ax[1, 2].text(0, 0.8, 'localr Taylor est {} '.format(lrest))
persist['Rexists'].append(optstate.n)
persist['sampleregret'].append(rsam)
persist['expectedregret'].append(racc)
#persist['expectedRbinorm'].append(rbin)
persist['localrsam'].append(rloc)
#persist['regretlower'].append(rlow)
persist['localrest'].append(lrest)
ax[0, 3].plot(persist['Rexists'], persist['localrest'], 'k')
#ax[0,3].plot(persist['Rexists'],persist['expectedRbinorm'],'purple')
ax[0, 3].plot(persist['Rexists'], persist['sampleregret'], 'b')
ax[0, 3].plot(persist['Rexists'], persist['expectedregret'], 'g')
ax[0, 3].plot(persist['Rexists'], persist['localrsam'], 'r')
#ax[0,3].plot(persist['Rexists'],persist['regretlower'],'purple')
ax[0, 3].set_yscale('log')
#ax[2,3].plot(K[:,0],K[:,1],'b.')
try:
fname = 'lotsofplots' + time.strftime('%d_%m_%y_%H:%M:%S') + '.png'
print('saving as {}'.format(fname))
fig.savefig(os.path.join(gpbo.core.debugoutput['path'], fname))
except BaseException as e:
logger.error(str(e))
fig.clf()
plt.close(fig)
del (fig)
#if a cheat objective as available see how we would do on starting a local opt now
if 'cheatf' in para.keys():
try:
C = sp.linalg.cholesky(H)
except:
logger.info('not +ve definite at posterior min')
C = sp.linalg.cholesky(sp.eye(H.shape[0]))
print('C {} \nxmin {}\nC.T.xmin{}'.format(C, xmin, C.T.dot(xmin)))
def fn2(x):
print(
x, para['cheatf'](sp.linalg.solve(C.T, x), **{
's': 0.,
'd': [sp.NaN]
})[0])
return para['cheatf'](sp.linalg.solve(C.T, x), **{
's': 0.,
'd': [sp.NaN]
})[0]
R = minimize(fn2, C.T.dot(xmin), method='Nelder-Mead')
logger.warn('cheat testopt result with precondition {}:\n{}'.format(
H, R))
return rval, persist, {
'start': xminr.flatten(),
'H': H,
'reuseH': [k.hyp for k in G.kf],
'offsetEI': m,
'ppveatx': pc,
'rpve': rmax,
'log10GRest': sp.log10(racc)
}
def merge(s1,s2):
s=[]
s=sp.array(s1)
s=sp.vstack([s,s2])
return s
def power_law(physics,
phase,
A1='',
A2='',
A3='',
x='',
return_rate=True,
**kwargs):
r"""
For the following source term:
.. math::
r = A_{1} x^{A_{2}} + A_{3}
If return_rate is True, it returns the value of source term for the
provided x in each pore.
If return_rate is False, it calculates the slope and intercept for the
following linear form :
.. math::
r = S_{1} x + S_{2}
Parameters
----------
A1 -> A3 : string
The property name of the coefficients in the source term model
x : string or float/int or array/list
The property name or numerical value or array for the main quantity
Notes
-----
"""
if x is '':
X = _sp.ones(physics.Np) * _sp.nan
else:
if type(x) == str:
x = 'pore.' + x.split('.')[-1]
try:
X = physics[x]
except KeyError:
raise Exception(physics.name +
' does not have the pore property :' + x + '!')
else:
X = _sp.array(x)
length_X = _sp.size(X)
if length_X != physics.Np:
if length_X == 1:
X = X * _sp.ones(physics.Np)
elif length_X >= phase.Np:
X = X[physics.map_pores()]
else:
raise Exception('Wrong size for the numerical array of x!')
a = {}
source_params = [A1, A2, A3]
for ind in _sp.arange(_sp.size(source_params)):
A = source_params[ind]
if A is '':
a[str(ind + 1)] = 0
else:
if type(A) == str:
A = 'pore.' + A.split('.')[-1]
try:
a[str(ind + 1)] = physics[A]
except KeyError:
raise Exception(physics.name + '/' + phase.name +
' does not have the pore property :' + A +
'!')
else:
raise Exception('source_term parameters can only be string '
'type!')
if return_rate:
return (a['1'] * X**a['2'] + a['3'])
else:
S1 = a['1'] * a['2'] * X**(a['2'] - 1)
S2 = a['1'] * X**a['2'] * (1 - a['2']) + a['3']
return (_sp.vstack((S1, S2)).T)
def _parse_decode_genotypes_(decode_file, sids, pns, ocg):
ih5f = h5py.File(decode_file, 'r')
#Determine individual filter
pns1 = ih5f['PNs'][...]
assert len(sp.unique(pns1)) == len(pns1), 'WTF?'
pn_sort_indices = sp.argsort(pns1)
pn_overlap_filter = sp.in1d(pns1, pns)
pn_sort_indices = pn_sort_indices[pn_overlap_filter]
assert sp.all(
pns == pns1[pn_sort_indices]), 'Re-ordering of individuals failed?'
ocg.create_dataset('indivs', data=pns)
#Determine marker filter
mns1 = ih5f['Marker-names'][...]
assert len(sp.unique(mns1)) == len(mns1), 'WTF?'
mn_filter = sp.in1d(sids, mns1)
mns = sids[mn_filter]
indices = range(len(mns1))
mn_indices_dict = dict(
(key, value) for (key, value) in it.izip(mns1, indices))
mn_indices = []
for mn in mns:
mn_indices.append(mn_indices_dict[mn])
# mn_indices = sp.array(mn_indices)
mn_indices = sp.array(mn_indices)
assert sp.all(mns == mns1[mn_indices]), 'Re-ordering failed?'
#Pull off position from marker name
positions = sp.array([int(mn.split(':')[1]) for mn in mns])
#Sort by position
order = sp.argsort(positions)
positions = positions[order]
mn_indices = mn_indices[order]
mns = mns[order]
#Get nucleotide
alleles = ih5f["Alleles"][...]
a = sp.arange(len(alleles))
even_map = a % 2 == 0
odd_map = a % 2 == 1
nts = (sp.vstack([alleles[even_map], alleles[odd_map]])).T
nts = nts[mn_indices]
n_snps = len(mns)
n_indivs = len(pns)
print 'Parsing SNPs (%d x %d matrix)' % (n_snps, n_indivs)
snps = ocg.create_dataset('raw_snps_ref',
shape=(n_snps, n_indivs),
dtype='single',
compression='lzf')
freqs = sp.zeros(len(mn_indices))
snp_means = sp.zeros(len(mn_indices))
for i, m_i in enumerate(mn_indices):
if i % 1000 == 0:
print "Reached %d'th SNP" % i
probs = ih5f["Probabilities2"][m_i, pn_sort_indices]
pat_snp = sp.array(map(lambda x: x[0], probs), 'float32')
mat_snp = sp.array(map(lambda x: x[1], probs), 'float32')
snp = pat_snp + mat_snp
ok_filter = (pat_snp >= 0) * (mat_snp >= 0)
if not sp.all(ok_filter):
#impute missing
mean_gt = sp.mean(snp[ok_filter])
snp[~ok_filter] = mean_gt
snps[i] = snp
freq = mean_gt / 2.0
snp_means[i] = mean_gt
freqs[i] = freq
#Calculate stds
snp_stds = sp.sqrt(2 * freqs * (1 - freqs)) #sp.std(raw_snps, 1)
ocg.create_dataset('snp_stds_ref', data=snp_stds)
ocg.create_dataset('snp_means_ref', data=snp_means)
ocg.create_dataset('freqs_ref', data=freqs)
ocg.create_dataset('positions', data=positions)
ocg.create_dataset('nts', data=nts)
ocg.create_dataset('sids', data=mns)
return {'ss_filter': mn_filter, 'ss_order': order}
#import scipy as np
# In[]
if __name__ == '__main__':
'''
test1()
'''
'''
test2()
'''
thetas = np.linspace(0, np.pi * 0.3, 7)
print "thetas: ", thetas
rho = np.array([[0.1, 0.12, 0], [0, 0, 0]])
print "rho:", rho
k_dir = np.vstack([np.sin(thetas), np.zeros_like(thetas), np.cos(thetas)])
k_dir = k_dir.transpose()
wavelength = 10
k0 = np.pi * 2 / wavelength
a1 = np.array([1, 0, 0])
a2 = np.array([0, 1, 0])
class Gratinglobes(object):
def check(self):
dx = np.sqrt(np.sum(a1 * a1, axis=-1))
dy = np.sqrt(np.sum(a2 * a2, axis=-1))
threshold_d = wavelength / (1 + np.sin(thetas))
temp = np.array([dx < threshold_d, dy < threshold_d])
return np.sum(temp, axis=0) == temp.shape[0]
def perform_selection(self,delta_values,strategy,plots_fn=None,results_fn=None):
"""Perform delta selection for kernel ridge regression
delta_values : array-like, shape = [n_steps_delta]
Array of delta values to test
strategy : {'full_cv','insample_cv'}
Strategy to perform feature selection:
- 'full_cv' perform cross-validation over delta
- 'insample_cv' pestimates delta in sample using maximum likelihood.
plots_fn : str, optional, default=None
File name for generated plot. if not specified, the plot is not saved
results_fn : str, optional, default=None
file name for saving cross-validation results. if not specified, nothing is saved
Returns
-------
best_delta : float
best regularization parameter delta for ridge regression
"""
import matplotlib
matplotlib.use('Agg',warn=False) #This lets it work even on machines without graphics displays
import matplotlib.pylab as PLT
# use precomputed data if available
if self.K == None:
self.setup_kernel()
print 'run selection strategy %s'%strategy
model = fastlmm.lmm()
nInds = self.K.shape[0]
if strategy=='insample':
# take delta with largest likelihood
model.setK(self.K)
model.sety(self.y)
model.setX(self.X)
best_delta = None
best_nLL = SP.inf
# evaluate negative log-likelihood for different values of alpha
nLLs = SP.zeros(len(delta_values))
for delta_idx, delta in enumerate(delta_values):
res = model.nLLeval(delta=delta,REML=True)
if res["nLL"] < best_nLL:
best_delta = delta
best_nLL = res["nLL"]
nLLs[delta_idx] = res['nLL']
fig = PLT.figure()
fig.add_subplot(111)
PLT.semilogx(delta_values,nLLs,color='g',linestyle='-')
PLT.axvline(best_delta,color='r',linestyle='--')
PLT.xlabel('logdelta')
PLT.ylabel('nLL')
PLT.title('Best delta: %f'%best_delta)
PLT.grid(True)
if plots_fn!=None:
PLT.savefig(plots_fn)
if results_fn!=None:
SP.savetxt(results_fn, SP.vstack((delta_values,nLLs)).T,delimiter='\t',header='delta\tnLLs')
if strategy=='cv':
# run cross-validation for determining best delta
kfoldIter = SKCV.KFold(n_splits=self.num_folds,shuffle=True,random_state=self.random_state).split(range(nInds))
Ypred = SP.zeros((len(delta_values),nInds))
for Itrain,Itest in kfoldIter:
model.setK(self.K[Itrain][:,Itrain])
model.sety(self.y[Itrain])
model.setX(self.X[Itrain])
model.setTestData(Xstar=self.X[Itest],K0star=self.K[Itest][:,Itrain])
for delta_idx,delta in enumerate(delta_values):
res = model.nLLeval(delta=delta,REML=True)
beta = res['beta']
Ypred[delta_idx,Itest] = model.predictMean(beta=beta,delta=delta)
MSE = SP.zeros(len(delta_values))
for i in range(len(delta_values)):
MSE[i] = SKM.mean_squared_error(self.y,Ypred[i])
idx_bestdelta = SP.argmin(MSE)
best_delta = delta_values[idx_bestdelta]
fig = PLT.figure()
fig.add_subplot(111)
PLT.semilogx(delta_values,MSE,color='g',linestyle='-')
PLT.axvline(best_delta,color='r',linestyle='--')
PLT.xlabel('logdelta')
PLT.ylabel('MSE')
PLT.grid(True)
PLT.title('Best delta: %f'%best_delta)
if plots_fn!=None:
PLT.savefig(plots_fn)
if results_fn!=None:
SP.savetxt(results_fn, SP.vstack((delta_values,MSE)).T,delimiter='\t',header='delta\tnLLs')
return best_delta
def QFun(x,y):
xll=domain.geo_reference.xllcorner
yll=domain.geo_reference.yllcorner
inDat=scipy.vstack([x+xll,y+yll]).transpose()
return rasterValuesAtPoints(xy=inDat,rasterFile=rasterFile,
interpolation=interpolation)
mprior = sp.array([0.,-1.,-5.,-0.5,0.5])
sprior = sp.array([1.,1.,3.,1.,1.])
MAPH = GPdc.searchMAPhyp(X,Y,S,D,mprior,sprior,GPdc.SQUEXPPS,mx=20000)
print "MLEH: "+str(MLEH)
print "MAPH: "+str(MAPH)
G = GPdc.GPcore(X,Y,S,D,GPdc.kernel(GPdc.SQUEXPPS,1,sp.array(MAPH)))
print G.llk()
np=180
sup = sp.linspace(-1,1,np)
Dp = [[sp.NaN]]*np
Xp = sp.vstack([sp.array([i]) for i in sup])
[m,v] = G.infer_diag(Xp,Dp)
sq = sp.sqrt(v)
S = sp.empty([np,1])
for i in xrange(np):
S[i,0] = -MAPH[2]*(Xp[i,0]-MAPH[3])*(Xp[i,0]-MAPH[4])
sc= sp.sqrt(S.flatten())
a0.plot(sup,m.flatten())
sc = sp.sqrt(S.flatten())
a0.fill_between(sup, sp.array(m-1.*sq).flatten(), sp.array(m+1.*sq).flatten(), facecolor='lightblue',edgecolor='lightblue')
a0.plot(sup,(m+2*sc).flatten(),'g')
a0.plot(sup,(m-2*sc).flatten(),'g')
plt.show()
# (46,28,29) motor 243 is precuneus
labs1, correlation_within_precuneus_vector, correlation_with_rest_vector, mask, centroid = parcellate_region(
(33, 34, 35, 74), sub, nClusters, sdir[i],
scan_type[i], 1, session_type[j], 0, 0)
count1 += 1
if count1 == 1:
labs_all_1 = sp.array(labs1.labels)
vert_all_1 = sp.array(labs1.vertices)
faces_all_1 = sp.array(labs1.faces)
correlation_within_precuneus = sp.array(
correlation_within_precuneus_vector)
correlation_with_rest = sp.array(
correlation_with_rest_vector)
all_centroid = sp.array(centroid)
else:
labs_all_1 = sp.vstack([labs_all_1, labs1.labels])
vert_all_1 = sp.array([labs1.vertices])
faces_all_1 = sp.array([labs1.faces])
correlation_within_precuneus = sp.vstack([
correlation_within_precuneus,
correlation_within_precuneus_vector
])
correlation_with_rest = sp.vstack([
correlation_with_rest, correlation_with_rest_vector
])
all_centroid = sp.vstack([all_centroid, centroid])
# sp.savez_compressed('clustering_results_sessions_region_pc', R_all=R_all)
data_file = 'data_file' + str(nClusters)
sp.savez(data_file + str(i * 2 + j) + 'precuneus_sine.npz',
correlation_within_precuneus=correlation_within_precuneus,
dmrt += h[2]['RT'][:]
dmz += h[2]['Z'][:]
nbdm += 1.
h.close()
for p in da:
w = we[p] > 0
da[p][w] /= we[p][w]
rp /= wet
rt /= wet
z /= wet
if not args.no_dmat:
dm /= wet[:, None]
da = sp.vstack(list(da.values()))
we = sp.vstack(list(we.values()))
co = smooth_cov(da, we, rp, rt)
da = (da * we).sum(axis=0)
da /= wet
if 'dmrp' in locals():
dmrp /= nbdm
dmrt /= nbdm
dmz /= nbdm
if ('dmrp' not in locals()) or (dmrp.size == rp.size):
dmrp = rp.copy()
dmrt = rt.copy()
dmz = z.copy()
def plotLine(vector,
val=1.0,
close=False,
tube_radius=None,
index=None,
**kwargs):
"""
PlotLine creates a single plot object from a singular vector or from a n-dimensional
tuple or list.
"""
plot = False
try:
x = vector.x()
temp0 = x[0]
temp1 = x[1]
temp2 = x[2]
s = val * scipy.ones(temp0.shape)
# For surface objects, this keyword allows for the last corner to connect with the first
if close:
temp0 = scipy.concatenate((temp0, scipy.atleast_1d(temp0[0])))
temp1 = scipy.concatenate((temp1, scipy.atleast_1d(temp1[0])))
temp2 = scipy.concatenate((temp2, scipy.atleast_1d(temp2[0])))
s = scipy.concatenate((s, scipy.atleast_1d(s[0])))
if not index is None:
N = len(temp0)
connect = scipy.vstack([
scipy.arange(index, index + N - 1.5),
scipy.arange(index + 1, index + N - .5)
]).T # I want to rewrite this...
index += N
except AttributeError:
temp0 = []
temp1 = []
temp2 = []
s = []
connect = []
# if it is not some sort of vector or vector-derived class, iterate through and make a surface object
if index is None:
index = 0
plot = True
for i in vector:
output = plotLine(i, close=close, index=index, **kwargs)
temp0 += [output[0]]
temp1 += [output[1]]
temp2 += [output[2]]
s += [output[3]]
connect += [output[4]]
index = output[5]
#turn to arrays here so I don't accidentally nest lists or tuples
temp0 = scipy.hstack(temp0)
temp1 = scipy.hstack(temp1)
temp2 = scipy.hstack(temp2)
s = scipy.hstack(s)
connect = scipy.vstack(connect)
if index is None:
try:
mlab.plot3d(temp0,
temp1,
temp2,
s,
vmin=0.,
vmax=1.,
tube_radius=tube_radius,
**kwargs)
except ValueError:
mlab.plot3d(temp0.flatten(),
temp1.flatten(),
temp2.flatten(),
s.flatten(),
vmin=0.,
vmax=1.,
tube_radius=tube_radius,
**kwargs)
else:
if plot:
# follows http://docs.enthought.com/mayavi/mayavi/auto/example_plotting_many_lines.html#example-plotting-many-lines
src = mlab.pipeline.scalar_scatter(temp0, temp1, temp2, s)
src.mlab_source.dataset.lines = connect
lines = mlab.pipeline.stripper(src)
mlab.pipeline.surface(lines, **kwargs)
else:
return (temp0, temp1, temp2, s, connect, index)
else:
print('Loading data from pickle: %s' % picklefile)
(w_g, V_g, ctypes, tn_labels, psi) = pickle.load(open(picklefile, 'r'))
### choose colormap and adapt normalization
cmap = plt.get_cmap('jet')
norm = plt.Normalize(0, sp.unique(ctypes).shape[0])
### plot first k main axes of variation
print('Plotting by ctype ... ')
fig = plt.figure(figsize=(k * 4, k * 4))
for k1 in range(0, k):
cnt = 1
for k2 in range(k1 + 1, k):
ax = fig.add_subplot(k - 1, k - 1, (k1 * (k - 1)) + cnt)
cnt += 1
trans_data = sp.vstack([V_g[k1, :], V_g[k2, :]]).dot(psi)
for idx, ct in enumerate(sp.unique(ctypes)):
c_idx = sp.where(ctypes == ct)[0]
if c_idx.shape[0] > 0:
ax.plot(trans_data[0, c_idx],
trans_data[1, c_idx],
markers.MarkerStyle.filled_markers[idx % 13],
color=cmap(norm(idx)),
label=ct,
ms=4,
alpha=0.75)
ax.set_title('PC %i vs %i' % (k1 + 1, k2 + 1))
ax.set_xticks(ax.get_xticks()[::2])
ax.set_yticks(ax.get_yticks()[::2])
ax.tick_params(axis='both', which='major', labelsize=10)
ax.tick_params(axis='both', which='minor', labelsize=10)
if os.path.isfile(
os.path.join(
p_dir, sub, sub + '.rfMRI_REST1_RL.reduce3.ftdata.NLM_11N\
_hvar_25.mat')):
labs1 = parcellate_motor(sub, nClusters + 1, 1, 1, 2)
if os.path.isfile(
os.path.join(
p_dir, sub, sub + '.rfMRI_REST2_RL.reduce3.ftdata.NLM_11N\
_hvar_25.mat')):
labs2 = parcellate_motor(sub, nClusters + 1, 1, 2, 2)
count1 += 1
if count1 == 1:
labs_all_1 = sp.array(labs1)
labs_all_2 = sp.array(labs2)
else:
labs_all_1 = sp.vstack([labs_all_1, labs1])
labs_all_2 = sp.vstack([labs_all_2, labs2])
R = sp.zeros(count1)
for a in range(count1):
R = adjusted_rand_score(labs_all_1, labs_all_2)
R_all.append(R)
print('Clusters=', nClusters)
sp.savez_compressed('clustering_results_sessions_GMM', R_all=R_all)
#%%
fig = plt.figure()
plt.plot(R_all)
fig.savefig('across_subjects_adj_rand_sessions_GMM.pdf')
def test2():
thetas = np.linspace(0, np.pi * 0.3, 7)
# thetas = np.array([np.pi*0.3])
print "thetas: ", thetas
zmin = 1
zmax = 5
zs = np.linspace(zmin, zmax, 100)
r = np.array([[0.1, 0.12, zz] for zz in zs])
k_dir = np.vstack([np.sin(thetas), np.zeros_like(thetas), np.cos(thetas)])
k_dir = k_dir.transpose()
# print k_dir
wavelength = 10
k0 = np.pi * 2 / wavelength
a1 = np.array([1, 0, 0])
a2 = np.array([0, 1, 0])
class Gratinglobes(object):
def check(self):
dx = np.sqrt(np.sum(a1 * a1, axis=-1))
dy = np.sqrt(np.sum(a2 * a2, axis=-1))
threshold_d = wavelength / (1 + np.sin(thetas))
temp = np.array([dx < threshold_d, dy < threshold_d])
return np.sum(temp, axis=0) == temp.shape[0]
checker = Gratinglobes().check()
print "grating lobe condition: ", checker
pgf_gen_ewald = PGF_EWALD(k0, a1, a2, 2, 2)
theta_phi = np.vstack([thetas, np.zeros_like(thetas)])
theta_phi = theta_phi.transpose()
class DGF_INT(object):
def interp(self):
x_sample = np.linspace(0, 0.2, 3)
y_sample = np.linspace(0, 0.2, 3)
z_sample = np.linspace(zmin, zmax, 20)
theta_sample = np.linspace(0, np.pi * 0.67, 10)
phi_sample = np.array([-0.01, 0.01]) #np.linspace(0,np.pi*2,10)
pdf = DGF_Interp_3D(x=x_sample,y=y_sample,z=z_sample, \
pgf_gen=pgf_gen_ewald,\
k_dir_theta=theta_sample, k_dir_phi=phi_sample)
result_interp = pdf.interp_dir_r(theta_phi, r)
return result_interp
result_interp = DGF_INT().interp()
result_ewald = pgf_gen_ewald.pgf(k_dir, r)
import matplotlib.pylab as plt
plt.figure()
class Method(object):
def __init__(self, result, marker, line):
self.result = result
self.marker = marker
self.line = line
def angle(self, it):
plt.plot(zs,\
np.angle(self.result)[it]/np.pi*180,\
self.line,\
label='angle %d %s'%(it,self.marker))
def absolute(self, it):
plt.plot(zs,\
np.absolute(self.result)[it],\
self.line,\
label='abs %d %s'%(it,self.marker))
map(Method(result_interp, 'interp', "s").angle, xrange(theta_phi.shape[0]))
map(Method(result_ewald, 'ewald', "-").angle, xrange(theta_phi.shape[0]))
plt.ylabel("angle (degree)")
plt.xlabel("zs")
plt.legend()
plt.show()
plt.figure()
map(
Method(result_interp, 'interp', "+").absolute,
xrange(theta_phi.shape[0]))
map(
Method(result_ewald, 'ewald', "-").absolute,
xrange(theta_phi.shape[0]))
plt.xlabel("zs")
plt.ylabel("log10(abs) ")
plt.legend()
plt.show()
plt.figure()
map(
Method((result_interp - result_ewald) / result_ewald, 'diff',
"-+").absolute, xrange(theta_phi.shape[0]))
# map(Method(result_ewald,'ewald',"-").absolute,xrange(theta_phi.shape[0]))
plt.xlabel("zs")
plt.ylabel("log10(abs) ")
plt.legend()
plt.show()
def gphinasrecc(optstate, persist, **para):
if para['onlyafter'] >= optstate.n or not optstate.n % para['everyn'] == 0:
#return [sp.NaN for i in para['lb']],{'didnotrun':True}
return argminrecc(optstate, persist, **para)
logger.info('gpmapas reccomender')
d = len(para['lb'])
x = sp.hstack(
[sp.vstack([e['xa'] for e in optstate.ev]),
sp.vstack(optstate.x)])
y = sp.vstack(optstate.y)
s = sp.vstack([e['s'] + 10**optstate.condition for e in optstate.ev])
dx = [e['d'] for e in optstate.ev]
G = GPdc.GPcore(x, y, s, dx, [
GPdc.kernel(optstate.aux['kindex'], d + 1, h)
for h in optstate.aux['HYPdraws']
])
# def directwrap(xq,y):
# xq.resize([1,d])
# xe = sp.hstack([sp.array([[0.]]),xq])
# #print xe
# a = G.infer_m_post(xe,[[sp.NaN]])
# return (a[0,0],0)
# [xmin,ymin,ierror] = direct(directwrap,para['lb'],para['ub'],user_data=[], algmethod=1, maxf=para['maxf'], logfilename='/dev/null')
def wrap(x):
xq = copy.copy(x)
xq.resize([1, d])
xe = sp.hstack([sp.array([[0.]]), xq])
a = G.infer_m_post(xe, [[sp.NaN]])
return a[0, 0]
xmin, ymin, ierror = gpbo.core.optutils.twopartopt(wrap, para['lb'],
para['ub'],
para['dpara'],
para['lpara'])
logger.info('reccsearchresult: {}'.format([xmin, ymin, ierror]))
from gpbo.core import debugoutput
if debugoutput['datavis']:
if not os.path.exists(debugoutput['path']):
os.mkdir(debugoutput['path'])
l = sp.mean([h[3] for h in optstate.aux['HYPdraws']])
from matplotlib import pyplot as plt
fig, ax = plt.subplots(nrows=3, ncols=1, figsize=(10, 30))
n = 200
x_ = sp.linspace(-1, 1, n)
y_ = sp.linspace(-1, 1, n)
z_ = sp.empty([n, n])
s_ = sp.empty([n, n])
for i in xrange(n):
for j in xrange(n):
m_, v_ = G.infer_diag_post(sp.array([0., y_[j], x_[i]]),
[[sp.NaN]])
z_[i, j] = m_[0, 0]
s_[i, j] = sp.sqrt(v_[0, 0])
CS = ax[1].contour(x_, y_, z_, 20)
ax[1].clabel(CS, inline=1, fontsize=10)
CS = ax[2].contour(x_, y_, s_, 20)
ax[2].clabel(CS, inline=1, fontsize=10)
for i in xrange(x.shape[0] - 1):
ax[0].plot(x[i, 1], x[i, 2], 'b.')
circle = plt.Circle([x[i, 1], x[i, 2]],
radius=0.5 * x[i, 0] * l,
edgecolor="none",
color='lightblue',
alpha=0.8 - 0.6 * x[i, 2])
ax[0].add_patch(circle)
ax[0].plot(x[i + 1, 1], x[i + 1, 2], 'r.')
circle = plt.Circle([x[i + 1, 1], x[i + 1, 2]],
radius=0.5 * x[i, 0] * l,
edgecolor="none",
color='lightblue',
alpha=0.8 - 0.6 * x[i, 2])
ax[0].add_patch(circle)
ax[0].axis([-1., 1., -1., 1.])
ax[1].plot(xmin[0], xmin[1], 'ro')
fig.savefig(
os.path.join(
debugoutput['path'],
'datavis' + time.strftime('%d_%m_%y_%H:%M:%S') + '.png'))
fig.clf()
plt.close(fig)
del (fig)
return [i for i in xmin], persist, {'ymin': ymin}
def linear(physics, phase, A1='', A2='', x='', return_rate=True, **kwargs):
r"""
For the following source term:
.. math::
r = A_{1} x + A_{2}
If return_rate is True, it returns the value of source term for the
provided x in each pore.
If return_rate is False, it calculates the slope and intercept for the
following linear form :
.. math::
r = S_{1} x + S_{2}
Parameters
----------
A1 , A2 : string
The property name of the coefficients in the source term model.
With A2 set to zero this equation takes on the familiar for of r=kx.
x : string or float/int or array/list
The property name or numerical value or array for the main quantity
Notes
-----
Because this source term is linear in concentration (x) is it not necessary
to iterate during the solver step. Thus, when using the
``set_source_term`` method for an algorithm, it is recommended to set the
``maxiter``
argument to 0. This will save 1 unncessary solution of the system, since
the solution would coverge after the first pass anyway.
"""
if x is '':
X = _sp.ones(physics.Np) * _sp.nan
else:
if type(x) == str:
x = 'pore.' + x.split('.')[-1]
try:
X = physics[x]
except KeyError:
raise Exception(physics.name +
' does not have the pore property :' + x + '!')
else:
X = _sp.array(x)
length_X = _sp.size(X)
if length_X != physics.Np:
if length_X == 1:
X = X * _sp.ones(physics.Np)
elif length_X >= phase.Np:
X = X[physics.map_pores()]
else:
raise Exception('Wrong size for the numerical array of x!')
a = {}
source_params = [A1, A2]
for ind in _sp.arange(_sp.size(source_params)):
A = source_params[ind]
if A is '':
a[str(ind + 1)] = 0
else:
if type(A) == str:
A = 'pore.' + A.split('.')[-1]
try:
a[str(ind + 1)] = physics[A]
except KeyError:
raise Exception(physics.name + '/' + phase.name +
' does not have the pore property :' + A +
'!')
else:
raise Exception('source_term parameters can only be string '
'type!')
if return_rate:
return (a['1'] * X + a['2'])
else:
S1 = a['1']
S2 = a['2']
return (_sp.vstack((S1, S2)).T)
def PESbsaq(optstate, persist, **para):
para = copy.deepcopy(para)
if persist == None:
persist = {'n': 0, 'd': len(para['ub'])}
n = persist['n']
d = persist['d']
if n < para['nrandinit']:
persist['n'] += 1
para['ev']['xa'] = sp.random.uniform(para['xal'], para['xau'])
return randomaq(optstate, persist, **para)
logger.info('PESssaq')
x = sp.hstack(
[sp.vstack([e['xa'] for e in optstate.ev]),
sp.vstack(optstate.x)])
y = sp.vstack(optstate.y)
s = sp.vstack([e['s'] for e in optstate.ev])
dx = [e['d'] for e in optstate.ev]
print "\n at pesinplane x {} axis 0".format(x)
pesobj = PES.PES_inplane(x,
y,
s,
dx, [para['xal']] + para['lb'],
[para['xau']] + para['ub'],
para['kindex'],
para['mprior'],
para['sprior'],
0,
0,
DH_SAMPLES=para['DH_SAMPLES'],
DM_SAMPLES=para['DM_SAMPLES'],
DM_SUPPORT=para['DM_SUPPORT'],
DM_SLICELCBPARA=para['DM_SLICELCBPARA'],
mode=para['SUPPORT_MODE'])
if para['traincfn']: #
#print "XXXXXXXXXXXXXXx"
cx = sp.vstack([e['xa'] for e in optstate.ev])
cc = sp.vstack([e for e in optstate.c])
#print cx
#print cc
#print optstate.ev
#print optstate.x
cfn = objectives.traincfn(cx, cc)
"""
if len(cc)%5==0:
from matplotlib import pyplot as plt
f,a = plt.subplots(1)
xt = sp.linspace(0,1,100)
m = sp.empty(100)
for i in xrange(100):
m[i]=cfn(0.,**{'xa':xt[i]})
a.plot(xt,m,'b')
for i in xrange(len(optstate.c)):
a.plot(cx[i,0],cc[i,0],'ro')
plt.show()
"""
else:
cfn = para['cfn']
[xmin, ymin, ierror] = pesobj.search_acq(cfn,
lambda s: para['ev']['s'],
volper=para['volper'])
logger.debug([xmin, ymin, ierror])
para['ev']['xa'] = xmin[0]
xout = [i for i in xmin[1:]]
return xout, para['ev'], persist, {
'HYPdraws': [k.hyp for k in pesobj.G.kf],
'mindraws': pesobj.Z,
'DIRECTmessage': ierror,
'PESmin': ymin
}
def creAllvecSamp():
Tlines = open("res/RELATIONS_train.txt").readlines()
TLlines = open("res/RELATIONS_trainLable.txt").readlines()
batchNum = 10
batchSize = len(Tlines) / 10
for b in range(0, batchNum):
line = Tlines[b * batchSize]
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2)
Mat = wordvec
for i in range(b * batchSize + 1, (b + 1) * batchSize):
line = Tlines[i]
print(i)
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2)
Mat = vstack([Mat, wordvec])
print(Mat.shape)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelSimTrain_" + str(b) +
".mat", {"train": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组
list = []
for i in range(b * batchSize, (b + 1) * batchSize):
line = TLlines[i]
list.append(int(line.strip()))
Mat = np.array(list)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelSimTrainLable_" +
str(b) + ".mat", {"trainLable": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组
Telines = open("res/RELATIONS_test.txt").readlines()
line = Telines[0]
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2) #300维度的相似度
Mat = wordvec
for i in range(1, 600):
line = Telines[i]
print(i)
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2)
Mat = vstack([Mat, wordvec])
print(Mat.shape)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelTestSim_0.mat",
{"tests": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组
# print(trainMat[])
line = Telines[600]
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2) #300维度的相似度
Mat = wordvec
for i in range(601, len(Telines)):
line = Telines[i]
print(i)
word1 = line.strip("\n").split("\t")[0]
word2 = line.strip("\n").split("\t")[1]
wordvec = getWordPairVec(word1, word2)
Mat = vstack([Mat, wordvec])
print(Mat.shape)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelTestSim_1",
{"tests": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组
# print(trainMat[])
lines = open("res/RELATIONS_testLable.txt").readlines()
list = []
for i in range(0, 600):
line = lines[i]
list.append(int(line.strip()))
Mat = np.array(list)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelSimTestLable_0.mat",
{"testLable": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组
list = []
for i in range(600, len(lines)):
line = lines[i]
list.append(int(line.strip()))
Mat = np.array(list)
sio.savemat(u"I:/数据/wordpairRelExpt/allVec/PairRelSimTestLable_1.mat",
{"testLable": Mat.transpose()},
oned_as='column') #行大于列 按列存成一维数组