def reduce_system(self, states, inputs, outputs):
"""Returns reduced state space matrices.
Parameters
----------
states : list
A list of the state names of the reduced system.
inputs : list
A list of the input names of the reduced system.
outputs : list
A list of the output names of the reduced system.
Returns
-------
A : ndarray
State matrix.
B : ndarray
Input matrix.
C : ndarray
Output matrix.
D : ndarray
Feed forward matrix.
"""
stateIndices = sorted([self.stateNames.index(s) for s in states])
inputIndices = sorted([self.inputNames.index(s) for s in inputs])
outputIndices = sorted([self.outputNames.index(s) for s in outputs])
A = self.A[np.ix_(stateIndices, stateIndices)]
B = self.B[np.ix_(stateIndices, inputIndices)]
C = self.C[np.ix_(outputIndices, stateIndices)]
D = self.D[np.ix_(outputIndices, inputIndices)]
return A, B, C, D
def impute(self, states):
S = self.S
if states.ndim == 1:
states = states[np.newaxis, :]
N = states.shape[0]
mask = self.state.slice_array(states, 'mask').astype(bool)
feature_mask = self.state.get_feature_mask(mask)
# first, mean impute
states_imputed = states.copy()
Xm = self.state.slice_array(states_imputed, 'observations')
means = np.tile(self.mean, (N, 1))
Xm[feature_mask] = means[feature_mask]
# now condition on observed values, if any
for i in xrange(N):
obs_ind = ~feature_mask[i, :]
# if no features observed or all features observed, don't act
if obs_ind.sum() == 0 or (~obs_ind).sum() == 0:
continue
# otherwise, do the dirty
A = S[np.ix_(obs_ind, obs_ind)]
C_T = S[np.ix_(~obs_ind, obs_ind)]
ctainv = np.dot(C_T, np.linalg.pinv(A))
mean = np.dot(ctainv, Xm[i, obs_ind])
Xm[i, ~obs_ind] = mean
if N == 1:
return states_imputed.flatten()
return states_imputed
def filter_otu_table_by_min(sample_names, taxon_names, data, lineages, min=1):
# loop through and remove every otu present at less than min
# this should be replaced by native QIIME filter code.
s_vec = []
s_names = []
taxonomies = []
# create list of OTUs to keep
for otu in range(data.shape[0]):
if data[otu, :].sum() >= min:
s_vec.append(otu)
s_names.append(taxon_names[otu])
if lineages:
taxonomies.append(lineages[otu])
h_vec = []
h_names = []
for sample in range(data.shape[1]):
if data[numpy.ix_(s_vec), sample].sum() >= 1:
h_vec.append(sample)
h_names.append(sample_names[sample])
# slice data
data = data[numpy.ix_(s_vec, h_vec)]
return h_names, s_names, data, taxonomies
def Barabasi_Albert(m0, m, N):
# if m > m0:
# raise ValueError('m must be smaller than or equal to m0')
# # initial graph
# Graph = [Node() for _ in range(m0)]
# for (ix, node) in enumerate(Graph):
# node.connect(Graph[ix + 1:])
# degrees = np.array([node.degree for node in Graph])
# cum_degrees = np.float(np.cumsum(degrees)) / np.sum(degrees)
K = np.eye(N, dtype=np.bool)
K[np.ix_(np.arange(m0), np.arange(m0))] = True
for ix in np.arange(m0, N):
selected = np.zeros((ix,), dtype=np.bool)
for conn in np.arange(m):
free = np.logical_not(selected)
p = np.array(np.sum(K[..., free], axis=0), dtype=np.float)
cdf = np.cumsum(p) / np.sum(p)
r = np.random.uniform()
link = np.where(np.logical_and(r < cdf,
np.logical_not(r >= cdf)))
K[ix, free[link]] = True
K[free[link], ix] = True
selected[free[link]] = True
rp = np.random.permutation(N)
return K[np.ix_(rp, rp)]
def subset(self, variables=None, samples=None):
"""Returns a subset of the dataset (and metadata).
Specify the variables and samples for creating a subset of the data.
variables and samples should be a list of ids. If not specified, it is
assumed to be all variables or samples.
Some examples:
- d.subset([3], [4])
- d.subset([3,1,2])
- d.subset(samples=[5,2,7,1])
Note: order matters! d.subset([3,1,2]) != d.subset([1,2,3])
"""
variables = variables if variables is not None else range(self.variables.size)
samples = samples if samples is not None else range(self.samples.size)
skip_stats = not (self.has_interventions or self.has_missing)
d = Dataset(
self.observations[N.ix_(samples,variables)],
self.missing[N.ix_(samples,variables)],
self.interventions[N.ix_(samples,variables)],
self.variables[variables],
self.samples[samples],
skip_stats = skip_stats
)
# if self does not have interventions or missing, the subset can't.
if skip_stats:
d._has_interventions = False
d._has_missing = False
return d
def flume_hm_res(X,Y):
hopperlen = 4.7
hmax = 1.9
hoppertop = 3.3
topangle = 17.0*np.pi/180.0
flumeangle = 31.0*np.pi/180.0
m0 = 0.61
x0 = -hopperlen
x2 = -hmax*np.cos(0.5*np.pi - flumeangle)
x1 = x2 - hoppertop*np.cos(flumeangle-topangle)
x3 = 0.0
y2 = hmax*np.sin(0.5*np.pi - flumeangle)
y1 = y2 - hoppertop*np.sin(flumeangle-topangle)
xm1 = x1 - y1*np.tan(0.5*np.pi - flumeangle)
slopem1 = y1/(x1-x0)
slope0 = y1/(x1-x0)
slope1 = (y2-y1)/(x2-x1)
slope2 = -y2/(x3-x2)
yind = np.where((Y[:,0]<=2.0)&(Y[:,0]>=0.0))[0]
xm1ind = np.where((X[0,:]>=xm1)&(X[0,:]<x0))[0]
x0ind = np.where((X[0,:]>=x0)&(X[0,:]<x1))[0]
x1ind = np.where((X[0,:]>=x1)&(X[0,:]<=x3))[0]
Z=np.zeros(np.shape(X))
#Z[np.ix_(yind,x1ind)] = m0*flume_eta(X[np.ix_(yind,x1ind)],Y[np.ix_(yind,x1ind)])
Z[np.ix_(yind,x1ind)] = m0
Z[np.ix_(yind,x0ind)] = m0*flume_eta(X[np.ix_(yind,x0ind)],Y[np.ix_(yind,x0ind)])/flume_eta_res(X[np.ix_(yind,x0ind)],Y[np.ix_(yind,x0ind)])
return Z
def test_chol_add_remove():
N = 5
X = np.random.randn(10,N)
A = X.T.dot(X)
L = np.linalg.cholesky(A)
Am = A[:-1,:-1]
bm = A[:-1,-1]
cm = A[-1,-1]
Lm = np.linalg.cholesky(Am)
# Get chol by adding row
assert np.allclose(L, chol_add_row(Lm, bm, cm))
# Now get chol by removing a row
def to_range(start, stop):
return np.setdiff1d(np.arange(N), np.arange(start,stop))
assert np.allclose(
np.linalg.cholesky(A[np.ix_(to_range(4,5),
to_range(4,5))]),
chol_remove_row(L,4,5))
assert np.allclose(
np.linalg.cholesky(A[np.ix_(to_range(1,3),
to_range(1,3))]),
chol_remove_row(L,1,3))
def get_corr_pred( self, sctx, u, tn, tn1 ):
ndofs = self.domain.n_dofs
#self.K.data[::] = 0.0
self.K = sparse.lil_matrix((ndofs, ndofs), float_ )
self.F_int[:] = 0.0
e_arr_size = self.e_arr_size
for elem in sctx.sdomain.elements:
e_id = elem.id_number
ix = elem.get_dof_map()
sctx.elem = elem
sctx.elem_state_array = sctx.state_array[ e_id*e_arr_size : (e_id+1)*e_arr_size ]
sctx.X = elem.get_X_mtx()
f, k = self.fets_eval.get_corr_pred( sctx, u[ix_(ix)], tn, tn1 )
#self.K_temp.data[:][:] = 0.
self.K_temp = sparse.lil_matrix((ndofs, ndofs), float_ )
a = 0
for i in ix:
self.K_temp.rows[i] = ix
self.K_temp.data[i][:] = k[a][:]
a =+1
#print K_temp
self.K = self.K + self.K_temp
self.F_int[ ix_(ix) ] += f
return self.F_int, self.K
def _split(estimator, X, y, indices, train_indices=None):
"""Create subset of dataset."""
if hasattr(estimator, 'kernel') and callable(estimator.kernel):
# cannot compute the kernel values with custom function
raise ValueError("Cannot use a custom kernel function. "
"Precompute the kernel matrix instead.")
if not hasattr(X, "shape"):
if getattr(estimator, "_pairwise", False):
raise ValueError("Precomputed kernels or affinity matrices have "
"to be passed as arrays or sparse matrices.")
X_subset = [X[idx] for idx in indices]
else:
if getattr(estimator, "_pairwise", False):
# X is a precomputed square kernel matrix
if X.shape[0] != X.shape[1]:
raise ValueError("X should be a square kernel matrix")
if train_indices is None:
X_subset = X[np.ix_(indices, indices)]
else:
X_subset = X[np.ix_(indices, train_indices)]
else:
X_subset = X[safe_mask(X, indices)]
if y is not None:
y_subset = y[safe_mask(y, indices)]
else:
y_subset = None
return X_subset, y_subset
def __init__(self, skim_dict, orig_zones, dest_zones, transpose=False):
omx_shape = skim_dict.skim_info['omx_shape']
logger.info("init AccessibilitySkims with %d dest zones %d orig zones omx_shape %s" %
(len(dest_zones), len(orig_zones), omx_shape, ))
assert len(orig_zones) <= len(dest_zones)
assert np.isin(orig_zones, dest_zones).all()
assert len(np.unique(orig_zones)) == len(orig_zones)
assert len(np.unique(dest_zones)) == len(dest_zones)
self.skim_dict = skim_dict
self.transpose = transpose
if omx_shape[0] == len(orig_zones):
# no slicing required
self.slice_map = None
else:
# 2-d boolean slicing in numpy is a bit tricky
# data = data[orig_map, dest_map] # <- WRONG!
# data = data[orig_map, :][:, dest_map] # <- RIGHT
# data = data[np.ix_(orig_map, dest_map)] # <- ALSO RIGHT
skim_index = list(range(omx_shape[0]))
orig_map = np.isin(skim_index, skim_dict.offset_mapper.map(orig_zones))
dest_map = np.isin(skim_index, skim_dict.offset_mapper.map(dest_zones))
if not dest_map.all():
# not using the whole skim matrix
logger.info("%s skim zones not in dest_map: %s" %
((~dest_map).sum(), np.ix_(~dest_map)))
self.slice_map = np.ix_(orig_map, dest_map)
def _safe_split(estimator, X, y, indices, train_indices=None):
"""Create subset of dataset and properly handle kernels."""
from ..gaussian_process.kernels import Kernel as GPKernel
if (hasattr(estimator, 'kernel') and callable(estimator.kernel) and
not isinstance(estimator.kernel, GPKernel)):
# cannot compute the kernel values with custom function
raise ValueError("Cannot use a custom kernel function. "
"Precompute the kernel matrix instead.")
if not hasattr(X, "shape"):
if getattr(estimator, "_pairwise", False):
raise ValueError("Precomputed kernels or affinity matrices have "
"to be passed as arrays or sparse matrices.")
X_subset = [X[index] for index in indices]
else:
if getattr(estimator, "_pairwise", False):
# X is a precomputed square kernel matrix
if X.shape[0] != X.shape[1]:
raise ValueError("X should be a square kernel matrix")
if train_indices is None:
X_subset = X[np.ix_(indices, indices)]
else:
X_subset = X[np.ix_(indices, train_indices)]
else:
X_subset = safe_indexing(X, indices)
if y is not None:
y_subset = safe_indexing(y, indices)
else:
y_subset = None
return X_subset, y_subset
def patch(data, rows, cols = None):
"""
data = data matrix, 1D or 2D array (matrix)
rows = iterator of rows (list) to select, None means selecting all rows
cols = iterator of cols (list) to select, None means selecting all cols
return np.array (of the patch shape), but the DIM of return should be
the same as data (1D or 2D)
if data is a sparse matrix, the return the matrix will be dense np.array
"""
if not sparse.issparse(data):
data = np.asarray(data)
dim = get_dim(data)
if dim == 1:
## ignore cols
return data[rows] if rows is not None else data
elif dim == 2:
nrows, ncols = data.shape
rows = rows if rows is not None else xrange(nrows)
cols = cols if cols is not None else xrange(ncols)
if sparse.issparse(data):
return data.toarray()[np.ix_(rows, cols)]
else:
return data[np.ix_(rows, cols)]
else:
raise RuntimeError('only supports 1D or 2D array')
def penalty_function(vocab_indices, summary_indices, sentence_similarity, config):
"""
This is the penalty function that is described in the paper
Graph-Based Submodular selection for extractive Summarization
Args:
vocab_indices: list
summary_indices: list
sentence_similarity: ndarray
config: dictionary
Some of the methods require some hyper parameters
to be set
This penalises redundancy
Returns: The value of the graph cut function
"""
penalty_lambda = config["penalty_lambda"]
sentence_similartiy_ = np.copy(sentence_similarity)
np.fill_diagonal(sentence_similartiy_, 0.0)
if len(summary_indices) == 0:
fn_value = 0.0
else:
v_not_in_s = list(set(vocab_indices) - set(summary_indices))
rows = v_not_in_s
cols = summary_indices
# USING THE ADVANCED INDEXING OF THE NUMPY ARRAY
fn_value = np.sum(sentence_similarity[np.ix_(rows, cols)]) - \
penalty_lambda * np.sum(sentence_similartiy_[np.ix_(summary_indices, summary_indices)])
return fn_value
def nd_bootstrap(data, iterations, axis=None, strip_tuple_if_one=True):
"""
Bootstrap iterator for several n-dimensional data arrays.
:param data: Iterable containing the data arrays
:param iterations: Number of bootstrap iterations.
:param axis: Bootstrapping is performed along this axis.
"""
shape0 = data[0].shape
if axis is None:
axis = 0
data = [d.ravel() for d in data]
n = len(data[0].shape)
K = len(data)
data0 = []
if axis is not None:
m = data[0].shape[axis]
to = tuple([axis]) + tuple(range(axis)) + tuple(range(axis + 1, n))
fro = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, n))
for i in range(K):
data0.append(data[i].transpose(to))
for i in range(iterations):
idx = np.random.randint(m, size=(m,))
if len(data) == 1 and strip_tuple_if_one:
yield (data0[0][np.ix_(idx), ...].squeeze().
transpose(fro).reshape(shape0))
else:
yield tuple(a[np.ix_(idx), ...].squeeze().
transpose(fro).reshape(shape0) for a in data0)
def runTest(self):
F=lambda x,y: 100.0*((x>=0.4)&(x<=0.6)&(y>=0.4)&(y<=0.6))
G=lambda x,y: (y==0)*1.0+(y==1)*(-1.0)
a=fasm.AssemblerElement(self.mesh,felem.ElementTriP1())
dudv=lambda du,dv: du[0]*dv[0]+du[1]*dv[1]
K=a.iasm(dudv)
uv=lambda u,v: u*v
B=a.fasm(uv)
fv=lambda v,x: F(x[0],x[1])*v
f=a.iasm(fv)
gv=lambda v,x: G(x[0],x[1])*v
g=a.fasm(gv)
D=np.nonzero(self.mesh.p[0,:]==0)[0]
I=np.setdiff1d(np.arange(0,self.mesh.p.shape[1]),D)
x=np.zeros(K.shape[0])
x[I]=scipy.sparse.linalg.spsolve(K[np.ix_(I,I)]+B[np.ix_(I,I)],
f[I]+g[I])
self.assertAlmostEqual(np.max(x),1.89635971369,places=2)
def mask_restore(b, passmask, fill_value=0, axis=(0,1)):
"""
Restore the rows and/or columns filtered out using passmask to produce b.
"""
passmask = np.asarray(passmask, dtype=bool)
n = len(passmask)
fill_if = ~passmask
if b.ndim == 1:
a = np.zeros(n, dtype=b.dtype)
a[passmask] = b
if fill_value: a[fill_if] = fill_value
return a
elif b.ndim == 2:
if axis == (0,1):
a = np.zeros((n, n), dtype=b.dtype)
a[np.ix_(passmask, passmask)] = b
if fill_value: a[np.ix_(fill_if, fill_if)] = fill_value
return a
elif axis == 0:
a = np.zeros((n, b.shape[1]), dtype=b.dtype)
a[passmask, :] = b
if fill_value: a[fill_if, :] = fill_value
return a
elif axis == 1:
a = np.zeros((b.shape[0], n), dtype=b.dtype)
a[:, passmask] = b
if fill_value: a[:, fill_if] = fill_value
return a
else:
raise ValueError("'axis' must be 0, 1 or (0,1)")
raise TypeError("Input array must be rank 1 or 2")
def get_corr_pred( self, sctx, u, du, tn, tn1 ):
n_ip_arr, ip_coords_arr, ip_weights_arr = self.ip_scheme
self.F_int[:] = 0.0
self.k_arr[...] = 0.0
B_mtx_grid = None
J_det_grid = None
ip_offset = 0
k_list = []
for e_id, ( elem, n_ip ) in enumerate( zip( self.sdomain.elements, n_ip_arr ) ):
ip_coords = ip_coords_arr[ ip_offset : ip_offset + n_ip ]
ip_weights = ip_weights_arr[ ip_offset : ip_offset + n_ip ]
ix = elem.get_dof_map()
sctx.elem = elem
sctx.elem_state_array = self.state_array[ ip_offset : ip_offset + n_ip ].flatten()
sctx.X = elem.get_X_mtx()
if self.cache_geo_matrices:
B_mtx_grid = self.B_mtx_grid[ e_id, ... ]
J_det_grid = self.J_det_grid[ e_id, ... ]
f, k = self.fets_eval.get_corr_pred( sctx, u[ix_( ix )], du[ix_( ix )],
tn, tn1,
B_mtx_grid = B_mtx_grid,
J_det_grid = J_det_grid,
ip_coords = ip_coords,
ip_weights = ip_weights )
self.k_arr[ e_id ] = k
self.F_int[ ix_( ix ) ] += f
ip_offset += n_ip
return self.F_int, SysMtxArray( mtx_arr = self.k_arr, dof_map_arr = self.sdomain.elem_dof_map )
def flume_theta(X,Y):
"""
angle theta in flume
"""
deg2rad = np.pi/180.0
flumelen = 78.0
flumerad = 10.0
theta1 = 31.0
theta2 = 2.5
D2 = flumelen + flumerad*(theta1 - theta2)*deg2rad
yind = np.where((Y[:,0]<=20.0)&(Y[:,0]>=-20.0))[0]
x1ind = np.where(X[0,:]<=flumelen)[0]
x2ind = np.where((X[0,:]>flumelen)&(X[0,:]<D2))[0]
x3ind = np.where(X[0,:]>=D2)[0]
Z=np.zeros(np.shape(X))
Z[np.ix_(yind,x1ind)] = theta1
Z[np.ix_(yind,x3ind)] = theta2
Z[np.ix_(yind,x2ind)] = theta1 - (X[np.ix_(yind,x2ind)]-flumelen)/(deg2rad*flumerad)
Z = deg2rad*Z
return Z
def flume_eta(X,Y):
hopperlen = 4.7
hmax = 1.9
hoppertop = 3.3
topangle = 17.0*np.pi/180.0
flumeangle = 31.0*np.pi/180.0
backangle = (90.0-flumeangle)*np.pi/180.0
x0 = 0.0
x1 = (x0-hopperlen)*np.cos(flumeangle)
yind = np.where((Y[:,0]<=2.0)&(Y[:,0]>=0.0))[0]
x1ind = np.where(X[0,:]>=x1)[0]
x2ind = np.where(X[0,:]<x1)[0]
Z=np.zeros(np.shape(X))
elev = bed_curve(x0*np.ones(np.shape(X)),0.0*np.ones(np.shape(Y)))
Z[np.ix_(yind,x1ind)] = hmax + (x0-X[np.ix_(yind,x1ind)])*np.sin(topangle) + elev[0,0]
elev = bed_curve(x0*np.ones(np.shape(X)),0.0*np.ones(np.shape(Y)))
etax1 = hmax + (x0-x1)*np.sin(topangle) + elev[0,0]
Z[np.ix_(yind,x2ind)] = etax1 - (x1-X[np.ix_(yind,x2ind)])*np.sin(backangle)
return Z
def _steadystate_lu(L, use_rcm=True, use_umfpack=False):
"""
Find the steady state(s) of an open quantum system by computing the
LU decomposition of the underlying matrix.
"""
if settings.debug:
print('Starting LU solver...')
dims=L.dims[0]
weight=np.abs(L.data.max())
n = prod(L.dims[0][0])
b = np.zeros(n ** 2, dtype=complex)
b[0] = weight
L = L.data.tocsc() + sp.csc_matrix((weight*np.ones(n),
(np.zeros(n), [nn * (n + 1) for nn in range(n)])),
shape=(n ** 2, n ** 2))
L.sort_indices()
use_solver(assumeSortedIndices=True, useUmfpack=use_umfpack)
if use_rcm:
perm = symrcm(L)
L = sparse_permute(L,perm,perm)
b = b[np.ix_(perm,)]
solve = factorized(L)
v = solve(b)
if use_rcm:
rev_perm = np.argsort(perm)
v = v[np.ix_(rev_perm,)]
data = vec2mat(v)
data = 0.5 * (data + data.conj().T)
return Qobj(data, dims=dims, isherm=True)
def get_aligned_new_and_old_datasets():
'''
Returns the old and new matrices only where they overlap
(both rows and columns), so that the data comparison tests don't error out.
This should only error out if there is no overlap whatsoever.
'''
# Align the rows
new_row_ids = [x for x in res_count_dataset[0]]
old_row_ids = [x for x in orig_count_dataset[0]]
common_rows = list(set(new_row_ids).intersection(set(old_row_ids)))
new_matrix_row_inds = [new_row_ids.index(x) for x in common_rows]
old_matrix_row_inds = [old_row_ids.index(x) for x in common_rows]
# Align the columns
new_col_tuples = [tuple(x) for x in res_count_dataset[1]]
old_col_tuples = [tuple(x) for x in orig_count_dataset[1]]
common_cols = list(set(new_col_tuples).intersection(set(old_col_tuples)))
new_matrix_col_inds = [new_col_tuples.index(x) for x in common_cols]
old_matrix_col_inds = [old_col_tuples.index(x) for x in common_cols]
# Get the new aligned matrices
aligned_old_mat = orig_count_dataset[2][np.ix_(old_matrix_row_inds, old_matrix_col_inds)]
aligned_new_mat = res_count_dataset[2][np.ix_(new_matrix_row_inds, new_matrix_col_inds)]
print "Original matrix shape: ", orig_count_dataset[2].shape
print "New matrix shape: ", res_count_dataset[2].shape
print "Aligned matrix shape: ", (len(common_rows), len(common_cols))
return [common_rows, common_cols, aligned_new_mat], [common_rows, common_cols, aligned_old_mat]
def multiClassSVM(distances, trainingIndice, testingIndice, semanticLabels, kernelType):
distances = distances ** 2
trainDistance = distances[np.ix_(trainingIndice, trainingIndice)]
gamma = 1.0 / np.mean(trainDistance)
kernelParam = []
kernelParam.append(gamma)
tempList = []
tempList.append(kernelType)
baseKernel = constructBaseKernels(tempList, kernelParam, distances)
trainGramMatrix = baseKernel[0][np.ix_(trainingIndice, trainingIndice)]
testGramMatrix = baseKernel[0][np.ix_(testingIndice, trainingIndice)]
trainLabels = [semanticLabels[i] for i in trainingIndice]
testLabels = [semanticLabels[i] for i in testingIndice]
clf = SVC(kernel = "precomputed")
clf.fit(trainGramMatrix, trainLabels)
SVMResults = clf.predict(testGramMatrix)
correct = sum(1.0 * (SVMResults == testLabels))
accuracy = correct / len(testLabels)
return accuracy
def load_weight_files(weights_files, genes, patients, typeToGeneIndex, typeToPatientIndex, masterGeneToIndex, masterPatientToIndex):
# Master matrix of all weights
P = np.zeros((len(genes), len(patients)))
for i, weights_file in enumerate(weights_files):
# Load the weights matrix for this cancer type and update the entries appropriately.
# Note that since genes/patients can be measured in multiple types, we need to map
# each patient to the "master" index.
type_P = np.load(weights_file)
ty_genes = set(typeToGeneIndex[i].keys()) & genes
ty_gene_indices = [ typeToGeneIndex[i][g] for g in ty_genes ]
master_gene_indices = [ masterGeneToIndex[g] for g in ty_genes ]
ty_patients = set(typeToPatientIndex[i].keys()) & patients
ty_patient_indices = [ typeToPatientIndex[i][p] for p in ty_patients ]
master_patient_indices = [ masterPatientToIndex[p] for p in ty_patients ]
master_mesh = np.ix_(master_gene_indices, master_patient_indices)
ty_mesh = np.ix_(ty_gene_indices, ty_patient_indices)
if np.any( P[master_mesh] > 0 ):
raise ValueError("Different weights for same gene-patient pair")
else:
P[ master_mesh ] = type_P[ ty_mesh ]
# Set any zero entries to the minimum (pseudocount). The only reason for zeros is if
# a gene wasn't mutated at all in a particular dataset.
P[P == 0] = np.min(P[P > 0])
return dict( (g, P[masterGeneToIndex[g]]) for g in genes )
def classify_binomial(x, data, counts, y):
classes, y = np.unique(y, return_inverse=True)
max_label = None
max = None
for class_label in np.nditer(classes):
class_examples = data[np.ix_(y == class_label)]
class_counts = counts[np.ix_(y == class_label)]
total_class_counts = sum(class_counts)
alfas = (class_examples.sum(axis=0) + 0.01)/(total_class_counts + 0.01)
prior = len(class_examples) / len(data)
membership = getMembershipBinomial(x,alfas, prior, class_counts, total_class_counts)
if(max_label is None):
max_label = class_label
max = membership
else:
if(class_label == 0):
if membership>max:
max = membership
max_label = class_label
else:
if membership>(max+8.5):
max = membership
max_label = class_label
return max_label
def conditional(self, in_dims, out_dims):
conditionals = []
for k, (weight_k, mean_k, covar_k) in enumerate(self):
conditionals.append(conditional(mean_k, covar_k,
in_dims, out_dims,
self.covariance_type))
cond_weights = lambda v: [(weight_k * Gaussian(mean_k[in_dims].reshape(-1,),
covar_k[ix_(in_dims, in_dims)]).normal(v.reshape(-1,)))
for k, (weight_k, mean_k, covar_k) in enumerate(self)]
def res(v):
gmm = GMM(n_components=self.n_components,
covariance_type=self.covariance_type,
random_state=self.random_state, thresh=self.thresh,
min_covar=self.min_covar, n_iter=self.n_iter, n_init=self.n_init,
params=self.params, init_params=self.init_params)
gmm.weights_ = cond_weights(v)
means_covars = [f(v) for f in conditionals]
gmm.means_ = array([mc[0] for mc in means_covars]).reshape(self.n_components,
-1)
gmm._set_covars(array([mc[1] for mc in means_covars]))
return gmm
return res
self.in_dims = array(in_dims)
self.out_dims = array(out_dims)
means = zeros((self.n_components, len(out_dims)))
covars = zeros((self.n_components, len(out_dims), len(out_dims)))
weights = zeros((self.n_components,))
sig_in = []
inin_inv = []
out_in = []
mu_in = []
for k, (weight_k, mean_k, covar_k) in enumerate(self):
sig_in.append(covar_k[ix_(in_dims, in_dims)])
inin_inv.append(matrix(sig_in).I)
out_in.append(covar_k[ix_(out_dims, in_dims)])
mu_in.append(mean_k[in_dims].reshape(-1, 1))
means[k, :] = (mean_k[out_dims] +
(out_in *
inin_inv *
(value - mu_in)).T)
covars[k, :, :] = (covar_k[ix_(out_dims, out_dims)] -
out_in *
inin_inv *
covar_k[ix_(in_dims, out_dims)])
weights[k] = weight_k * Gaussian(mu_in.reshape(-1,),
sig_in).normal(value.reshape(-1,))
weights /= sum(weights)
def p(value):
# hard copy of useful matrices local to the function
pass
return p
def comp(self,mean,var,covar,resp):
# Store the indices to the missing and observed responses.
miss=numpy.isnan(resp)
obs=numpy.logical_not(miss)
if miss.all():
return mean,var,covar
# Store the size of the model.
numresp,numpred=numpy.size(self.gain)
kalmgain=numpy.eye(numresp)
josgain=numpy.eye(numresp)
# Fill in the Kalman and Joseph gain matrices.
ind=numpy.ix_(miss,obs)
kalmgain[ind]=linalg.solve(self.noise[numpy.ix_(obs,obs)],
self.noise[ind].transpose()).transpose()
josgain[:,obs]=josgain[:,obs]-kalmgain[:,obs]
# Compute the predictor/response co-variance.
covar=covar.dot(josgain.transpose())
# Condition the response mean/variance on the observations.
mean=josgain.dot(mean)+numpy.dot(kalmgain[:,obs],resp[obs])
var=numpy.dot(josgain,numpy.dot(var,josgain.transpose()))
return mean,var,covar
def dctt1(a):
""" dct Discrete cosine transform.
y = dct(a) returns the discrete cosine transform of a.
The vector y is the same size as `a` and contains the
discrete cosine transform coefficients.
"""
if len(a.shape)==1:
a = a.reshape(a.size,1)
n,m = a.shape
aa = a[:,:]
#Compute weights to multiply DFT coefficients
ww = arrayexp(n)
if n%2 == 1:
y = np.zeros([2*n,m])
y[:n,:] = aa
y[n:2*n,:] = np.flipud(aa)
# Compute the FFT and keep the appropriate portion:
yy = np.fft.fft(y,axis=0)
yy = yy[:n,:]
else:
# Re-order the elements of the columns of x
y = np.concatenate((aa[np.ix_(range(0,n,2))],\
aa[np.ix_(range(1,n,2)[::-1])]), axis=0)
yy = np.fft.fft(y,axis=0)
ww = 2*ww # Double the weights for even-length case
wy = np.empty([n,m], complex)
for j in range(m):
wy[:,j] = ww
# Multiply FFT by weights:
b = np.multiply(wy,yy)
return b[:n,:m].real
def normal_eq_comb(AtA, AtB, PassSet = None):
num_cholesky = 0
num_eq = 0
if AtB.size == 0:
Z = np.zeros([])
elif (PassSet is None) or np.all(PassSet):
Z = nla.solve(AtA, AtB)
num_cholesky = 1
num_eq = AtB.shape[1]
else:
Z = np.zeros(AtB.shape) #(n, k)
if PassSet.shape[1] == 1:
if np.any(PassSet):
cols = np.nonzero(PassSet)[0]
Z[cols] = nla.solve(AtA[np.ix_(cols, cols)], AtB[cols])
num_cholesky = 1
num_eq = 1
else:
groups = column_group(PassSet)
for g in groups:
cols = np.nonzero(PassSet[:, g[0]])[0]
if cols.size > 0:
ix1 = np.ix_(cols, g)
ix2 = np.ix_(cols, cols)
Z[ix1] = nla.solve(AtA[ix2], AtB[ix1])
num_cholesky += 1
num_eq += len(g)
num_eq += len(g)
return Z, num_cholesky, num_eq
def consgpattern():
"""
binary pattern of the sparse nonlinear constraint gradient
"""
vfdxpat = ( self.vfielddxpattern
if self.vfielddxpattern is not None
else np.ones( (self.Nstates,self.Nstates) ) )
vfdupat = ( self.vfielddupattern
if self.vfielddupattern is not None
else np.ones( (self.Nstates,self.Ninputs) ) )
if( self.Ncons > 0 ):
consdxpat = ( self.consdxpattern
if self.consdxpattern is not None
else np.ones( (self.Ncons,self.Ninputs) ) )
out = np.zeros( ( feuler.Ncons, feuler.N ), dtype=np.int )
for k in range( Nsamples ):
out[ np.ix_( dconsidx[:,k+1], stidx[:,k] ) ] = vfdxpat
out[ np.ix_( dconsidx[:,k+1], uidx[:,k] ) ] = vfdupat
if( self.Ncons > 0 ):
out[ np.ix_( iconsidx[:,k], stidx[:,k+1] ) ] = consdxpat
return out
def _steadystate_direct_sparse(L, use_rcm=True, use_umfpack=False):
"""
Direct solver that uses scipy sparse matrices
"""
if settings.debug:
print('Starting direct solver...')
dims=L.dims[0]
weight=np.abs(L.data.max())
n = prod(L.dims[0][0])
b = np.zeros((n ** 2, 1), dtype=complex)
b[0,0] = weight
L = L.data + sp.csr_matrix((weight*np.ones(n), (np.zeros(n), [nn * (n + 1) for nn in range(n)])),
shape=(n ** 2, n ** 2))
L.sort_indices()
use_solver(assumeSortedIndices=True, useUmfpack=use_umfpack)
if use_rcm:
perm = symrcm(L)
L = sparse_permute(L,perm,perm)
b = b[np.ix_(perm,)]
v = spsolve(L, b)
if use_rcm:
rev_perm = np.argsort(perm)
v = v[np.ix_(rev_perm,)]
data = vec2mat(v)
data = 0.5 * (data + data.conj().T)
return Qobj(data, dims=dims, isherm=True)
print '%3d' % i, '%10.5f' % evals[i]
print
# parameter SEs from fit (psv file)
modprmses = np.sqrt(np.diag(mfmodel['prmcov']))
parms = info.shape[0]
prmidx = np.array(range(parms)) + 1
freeparm = 1 - mfmodel['prmstat'][0:parms]
uparm = prmidx * freeparm
pscore = parmscores[uparm > 0]
uparm = uparm[uparm > 0] - 1
modprmses = modprmses[modprmses > 0]
infosub = info[np.ix_(uparm, uparm)]
covmat = np.linalg.inv(infosub)
parmses = np.sqrt(np.diag(covmat))
print " pno modse Qse Score"
for i in range(len(parmses)):
print '%5d' % (
uparm[i] + 1
), '%10.5f' % modprmses[i], '%10.5f' % parmses[i], '%10.5f' % pscore[i][0]
# ## G matrix for various Pu models $$ \frac{d \mu}{d x}$$
# In[14]~
prmcov = mfmodel['prmcov']
prmvec = mfmodel['prmvec']
def automatic_community_detector_downsample(G,
method = 'un_louvain',
downsample_level = 1,
scale = 'log',
figsize = (15,15),
dpi = 100.,
ylabel='Presynaptic Neuron ID',
xlabel='Postsynaptic Neuron ID',
xticklabels = None,
yticklabels = None,
title='Community-Ordered Connectivity Matrix (Log Scale)',
vmax = None,
export_format = None,
figname='example_automatic_community'):
"""Automatically detects communities for a large networkx graph with downsampling for the adjacency matrix.
# Arguments:
G (network.Graph): A networkx graph or subclass object, including flybrainlab.graph.NeuronGraph.
method (str): Method to use. One of un_louvain, label_propagation, leiden, louvain, walktrap or infomap.
downsample_level (int): A downsampling level to use.
scale (str): 'linear' or 'log'. Use linear or log scale to cluster.
figsize (tuple): size of the figure.
dpi (float): dpi of the figure.
xlabel (str): Name of the x label.
ylabel (str): Name of the y label.
xticklabels (list): x tick labels to have.
yticklabels (list): y tick labels to have.
title (str): Title for the diagram.
vmax (float): Maximum value for the diagram.
export_format (str): if specified, file format to export the diagram.
figname (str): Name for the output figure.
# Returns:
np.ndarray: community-ordered connectivity matrix in linear scale
list: a list of list of node ids for each group member
"""
Gun, all_list_nodes, all_nodes = community_detection(G, method = method)
all_pre_nodes = [i for i in all_list_nodes]
all_post_nodes = [i for i in all_list_nodes]
A = nx.adjacency_matrix(G).todense()[np.ix_(all_pre_nodes,all_post_nodes)]
B = A[::downsample_level,::downsample_level].copy()
if xticklabels is None:
if isinstance(G, NeuronGraph):
xticklabels = sum(nodes_to_unames(G, all_nodes),[])[::downsample_level]
else:
xticklabels = sum(all_nodes, [])[::downsample_level]
if yticklabels is None:
if isinstance(G, NeuronGraph):
yticklabels = sum(nodes_to_unames(G, all_nodes),[])[::downsample_level]
else:
yticklabels = sum(all_nodes, [])[::downsample_level]
if scale == 'log':
Bd = np.log(1.+B)
else:
Bd = B
if title == 'Community-Ordered Connectivity Matrix (Log Scale)':
title = 'Community-Ordered Connectivity Matrix'
sizes = np.round(np.cumsum([len(i) for i in all_nodes]) / downsample_level)
gen_heatmap(Bd, figsize = figsize, dpi = dpi, xlabel = xlabel, ylabel = ylabel,
xticklabels = xticklabels, yticklabels = yticklabels,
hlines = sizes, vlines = sizes, title = title, vmax = vmax,
export_format = export_format, figname = figname)
return B, all_nodes
def dp2():
n = 3
a = np.ones((n, ) * 5)
i = np.random.randint(0, n, size=thesize)
g = a[np.ix_(i, i, i, i, i)]
def __getitem__(self, index):
# print("GETITEM: ", index)
indices, categories = index
# translate category names into ranges
cat_rgs = {}
for category in categories:
splt = category.split(":")
if len(splt) == 2:
if splt[1][-3:] == "hPa":
pressure2index = {
v: k
for k, v in self.config["vbls"][splt[0]]
["index2pressure"].items()
}
idx = pressure2index[int(splt[1][:-3])]
else:
idx = int(splt[1])
offset = self.config["vbls"][splt[0]]["offset"]
n_levels = 1 if self.config["vbls"][
splt[0]]["levels"] is None else len(
self.config["vbls"][splt[0]]["levels"])
assert n_levels > idx, "invalid level index!"
cat_rgs[category] = (
offset + idx, offset + idx,
self.config["vbls"][splt[0]]["type"] == "temp")
else:
offset = self.config["vbls"][category]["offset"]
n_levels = 1 if self.config["vbls"][category][
"levels"] is None else len(
self.config["vbls"][category]["levels"])
cat_rgs[category] = (
offset, offset + n_levels - 1,
self.config["vbls"][splt[0]]["type"] == "temp"
) # CHECK: Indices are inclusive right?
if isinstance(indices, slice):
# special instance - we can return a view!
results_dict = {
cat_name: self.fo[indices, slice(rg[0], rg[1] + 1, 1)]
if rg[2] else self.fo[slice(rg[0], rg[1] + 1, 1)]
for cat_name, rg in cat_rgs.items()
}
else:
results_dict = {}
for cat_name, rg in cat_rgs.items():
if indices is not None:
results_dict[cat_name] = self.fo[np.ix_(
indices, np.arange(rg[0], rg[1] + 1))]
else:
if rg[2]:
# print("BRANCH TMP NON INDICES:", rg)
results_dict[cat_name] = self.fo[
slice(None, None),
slice(rg[0], rg[1] + 1)]
# self.fo[np.arange(self.fo.shape[0]), np.arange(rg[0], rg[1] + 1)]
else:
results_dict[cat_name] = self.fo[slice(
rg[0], rg[1] + 1)]
# np.arange(rg[0], rg[1] + 1)
return results_dict
def compute_h(self):
self.h = (self.nu - self.b)/self.n * self.K[np.ix_(self.full_id, self.train_id)].sum(axis = 1) + \
(self.nl + self.b)/self.n * self.K[np.ix_(self.full_id, self.pool_id)].sum(axis = 1)
def R_score(self, ids):
R = 0.5 * self.K[np.ix_(ids, ids)].sum() + \
(self.nu - self.b)/self.n * self.K[np.ix_(ids, self.train_id)].sum() + \
(self.nl + self.b)/self.n * self.K[np.ix_(ids, self.pool_id)].sum()
return R
def update_h(self):
self.h = self.h - (self.b/self.n) * self.K[np.ix_(self.full_id, self.full_id)].sum(axis = 1) + \
((self.nu - self.nl - 4*self.b)/self.n) * self.K[np.ix_(self.full_id, self.selec_id)].sum(axis = 1)
map(
str.split,
open(
os.path.join(
EVAL_DIR,
'%s_test_split%d.txt' % (classLabel, 1 + SPLIT_IND)))))
train += [(fixClipName(k), classLabel) for k, v in d.items()
if v == '1']
test += [(fixClipName(k), classLabel) for k, v in d.items()
if v == '2']
return (train, test)
splits = map(read_split, range(3))
slice_kernel = lambda inds1, inds2: all_k[np.ix_(map(allClips.index, inds1),
map(allClips.index, inds2))]
REG_C = 1.0
def svm_train_test(train_k, test_k, ytrain, REG_C):
model = SVC(kernel='precomputed', C=REG_C, max_iter=10000)
model.fit(train_k, ytrain)
flatten = lambda ls: list(itertools.chain(*ls))
train_conf, test_conf = map(
flatten, map(model.decision_function, [train_k, test_k]))
return train_conf, test_conf
def one_vs_rest(SPLIT_IND):
calc_accuracy = lambda chosen, true: sum(
def _pyramid_single_interpolation(G, ca, pe, keep_inds, h_filter, **kwargs):
r"""Synthesize a single level of the graph pyramid transform.
Parameters
----------
G : Graph
Graph structure on which the signal resides.
ca : ndarray
Coarse approximation of the signal on a reduced graph.
pe : ndarray
Prediction error that was made when forming the current coarse approximation.
keep_inds : ndarray
The indices of the vertices to keep when downsampling the graph and signal.
h_filter : lambda expression
The filter in use at this level.
use_landweber : bool
To use the Landweber iteration approximation in the least squares synthesis.
Default is False.
reg_eps : float
Interpolation parameter. Default is 0.005.
landweber_its : int
Number of iterations in the Landweber approximation for least squares synthesis.
Default is 50.
landweber_tau : float
Parameter for the Landweber iteration. Default is 1.
Returns
-------
finer_approx :
Coarse approximation of the signal on a higher resolution graph.
"""
nb_ind = keep_inds.shape
N = G.N
reg_eps = float(kwargs.pop('reg_eps', 0.005))
use_landweber = bool(kwargs.pop('use_landweber', False))
landweber_its = int(kwargs.pop('landweber_its', 50))
landweber_tau = float(kwargs.pop('landweber_tau', 1.))
# index matrix (nb_ind x N) of keep_inds, S_i,j = 1 iff keep_inds[i] = j
S = sparse.csr_matrix(([1] * nb_ind, (range(nb_ind), keep_inds)),
shape=(nb_ind, N))
if use_landweber:
x = np.zeros(N)
z = np.concatenate((ca, pe), axis=0)
green_kernel = filters.Filter(G, lambda x: 1. / (x + reg_eps))
PhiVlt = _analysis(green_kernel, S.T, **kwargs).T
filt = filters.Filter(G, h_filter, **kwargs)
for iteration in range(landweber_its):
h_filtered_sig = _analysis(filt, x, **kwargs)
x_bar = h_filtered_sig[keep_inds]
y_bar = x - interpolate(G, x_bar, keep_inds, **kwargs)
z_delt = np.concatenate((x_bar, y_bar), axis=0)
z_delt = z - z_delt
alpha_new = PhiVlt * z_delt[nb_ind:]
x_up = sparse.csr_matrix((z_delt, (range(nb_ind), [1] * nb_ind)),
shape=(N, 1))
reg_L = G.L + reg_esp * sparse.eye(N)
elim_inds = np.setdiff1d(np.arange(N, dtype=int), keep_inds)
L_red = reg_L[np.ix_(keep_inds, keep_inds)]
L_in_out = reg_L[np.ix_(keep_inds, elim_inds)]
L_out_in = reg_L[np.ix_(elim_inds, keep_inds)]
L_comp = reg_L[np.ix_(elim_inds, elim_inds)]
next_term = L_red * alpha_new - L_in_out * linalg.spsolve(
L_comp, L_out_in * alpha_new)
next_up = sparse.csr_matrix((next_term, (keep_inds, [1] * nb_ind)),
shape=(N, 1))
x += landweber_tau * _analysis(filt, x_up - next_up, **
kwargs) + z_delt[nb_ind:]
finer_approx = x
else:
# When the graph is small enough, we can do a full eigendecomposition
# and compute the full analysis operator T_a
H = G.U * sparse.diags(h_filter(G.e), 0) * G.U.T
Phi = G.U * sparse.diags(1. / (reg_eps + G.e), 0) * G.U.T
Ta = np.concatenate(
(S * H, sparse.eye(G.N) - Phi[:, keep_inds] *
linalg.spsolve(Phi[np.ix_(keep_inds, keep_inds)], S * H)),
axis=0)
finer_approx = linalg.spsolve(Ta.T * Ta,
Ta.T * np.concatenate((ca, pe), axis=0))
print(20 * "-")
'''
花式索引
前一个取出行，后一个按位取出对应列
与索引器不同
索引器
'''
arr_new = np.arange(32).reshape((8, 4))
print(arr_new, "\n")
print(arr_new[[0, 3, 5]], "\n") # 输出对应行
print(arr_new[[0, 3, 5], [0, 3, 2]], '\n') # 对应行取出列的元素 [0][0] [3][3] [5][2]
x = arr_new[np.ix_([0, 3, 5], [0, 3, 2])] # ix_生成一个索引器（用于取出特定行列的数据）
print(x)
print(30 * '-')
'''
一些方法
'''
A = np.arange(12).reshape((3, 4))
print(A, '\n')
for row in A:
print(row) # 迭代每一行
print('打住')
for column in A.T:
print(column) # 迭代每一列
for item in A.flat: # 把A转变成一行的序列
print(item) # 迭代每一项
None:10]) # E: list[numpy.ndarray[Any, numpy.dtype[Any]]]
reveal_type(np.index_exp[0:1]) # E: Tuple[builtins.slice]
reveal_type(np.index_exp[0:1,
None:3]) # E: Tuple[builtins.slice, builtins.slice]
reveal_type(
np.index_exp[0, 0:1, ..., [0, 1, 3]]
) # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]]
reveal_type(np.s_[0:1]) # E: builtins.slice
reveal_type(np.s_[0:1, None:3]) # E: Tuple[builtins.slice, builtins.slice]
reveal_type(
np.s_[0, 0:1, ..., [0, 1, 3]]
) # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]]
reveal_type(np.ix_(
AR_LIKE_b)) # E: tuple[numpy.ndarray[Any, numpy.dtype[numpy.bool_]]]
reveal_type(
np.ix_(AR_LIKE_i,
AR_LIKE_f)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{double}]]]
reveal_type(
np.ix_(AR_i8)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int64}]]]
reveal_type(np.fill_diagonal(AR_i8, 5)) # E: None
reveal_type(
np.diag_indices(4)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]
reveal_type(np.diag_indices(
2, 3)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]
reveal_type(np.diag_indices_from(
AR_i8)) # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]
def misorientation(q1, q2, *args):
"""
sym is a tuple (crystal_symmetry, *sample_symmetry)
generally coded, may split up special cases for no symmetry or crystal/sample only...
"""
if not isinstance(q1, ndarray) or not isinstance(q2, ndarray):
raise RuntimeError, "quaternion args are not of type `numpy ndarray'"
if q1.ndim != 2 or q2.ndim != 2:
raise RuntimeError, "quaternion args are the wrong shape; must be 2-d (columns)"
if q1.shape[1] != 1:
raise RuntimeError(
"first argument should be a single quaternion, got shape %s" %
(q1.shape, ))
if len(args) == 0:
# no symmetries; use identity
sym = (c_[1., 0, 0, 0].T, c_[1., 0, 0, 0].T)
else:
sym = args[0]
if len(sym) == 1:
if not isinstance(sym[0], ndarray):
raise RuntimeError, "symmetry argument is not an numpy array"
else:
# add triclinic sample symmetry (identity)
sym += (c_[1., 0, 0, 0].T, )
elif len(sym) == 2:
if not isinstance(sym[0], ndarray) or not isinstance(
sym[1], ndarray):
raise RuntimeError, "symmetry arguments are not an numpy arrays"
elif len(sym) > 2:
raise RuntimeError, "symmetry argument has %d entries; should be 1 or 2" % (
len(sym))
# set some lengths
n = q2.shape[1] # length of misorientation list
m = sym[0].shape[1] # crystal (right)
p = sym[1].shape[1] # sample (left)
# tile q1 inverse
q1i = quatProductMatrix(invertQuat(q1), mult='right').squeeze()
# convert symmetries to (4, 4) qprod matrices
rsym = quatProductMatrix(sym[0], mult='right')
lsym = quatProductMatrix(sym[1], mult='left')
# Do R * Gc, store as
# [q2[:, 0] * Gc[:, 0:m], ..., q2[:, n-1] * Gc[:, 0:m]]
q2 = dot(rsym, q2).transpose(2, 0, 1).reshape(m * n, 4).T
# Do Gs * (R * Gc), store as
# [Gs[:, 0:p] * q[:, 0] * Gc[:, 0], ... , Gs[:, 0:p] * q[:, 0] * Gc[:, m-1], ...
# Gs[:, 0:p] * q[:, n-1] * Gc[:, 0], ... , Gs[:, 0:p] * q[:, n-1] * Gc[:, m-1]]
q2 = dot(lsym, q2).transpose(2, 0, 1).reshape(p * m * n, 4).T
# Calculate the class misorientations for full symmetrically equivalent
# classes for q1 and q2. Note the use of the fact that the application
# of the symmetry groups is an isometry.
eqvMis = fixQuat(dot(q1i, q2))
# Reshape scalar comp columnwise by point in q2 (and q1, if applicable)
sclEqvMis = eqvMis[0, :].reshape(n, p * m).T
# Find misorientation closest to origin for each n equivalence classes
# - fixed quats so garaunteed that sclEqvMis is nonnegative
qmax = sclEqvMis.max(0)
# remap indices to use in eqvMis
qmaxInd = (sclEqvMis == qmax).nonzero()
qmaxInd = c_[qmaxInd[0], qmaxInd[1]]
eqvMisColInd = sort(qmaxInd[:, 0] + qmaxInd[:, 1] * p * m)
# store Rmin in q
mis = eqvMis[ix_(range(4), eqvMisColInd)]
angle = 2 * arccosSafe(qmax)
return angle, mis
def crop_image1(img, tol=7):
# img is image data
# tol is tolerance
mask = img > tol
return img[np.ix_(mask.any(1), mask.any(0))]
def kron_reduction(G, ind):
r"""Compute the Kron reduction.
This function perform the Kron reduction of the weight matrix in the
graph *G*, with boundary nodes labeled by *ind*. This function will
create a new graph with a weight matrix Wnew that contain only boundary
nodes and is computed as the Schur complement of the original matrix
with respect to the selected indices.
Parameters
----------
G : Graph or sparse matrix
Graph structure or weight matrix
ind : list
indices of the nodes to keep
Returns
-------
Gnew : Graph or sparse matrix
New graph structure or weight matrix
References
----------
See :cite:`dorfler2013kron`
"""
if isinstance(G, graphs.Graph):
if G.lap_type != 'combinatorial':
msg = 'Unknown reduction for {} Laplacian.'.format(G.lap_type)
raise NotImplementedError(msg)
if G.is_directed():
msg = 'This method only work for undirected graphs.'
raise NotImplementedError(msg)
L = G.L
else:
L = G
N = np.shape(L)[0]
ind_comp = np.setdiff1d(np.arange(N, dtype=int), ind)
L_red = L[np.ix_(ind, ind)]
L_in_out = L[np.ix_(ind, ind_comp)]
L_out_in = L[np.ix_(ind_comp, ind)].tocsc()
L_comp = L[np.ix_(ind_comp, ind_comp)].tocsc()
Lnew = L_red - L_in_out.dot(linalg.spsolve(L_comp, L_out_in))
# Make the laplacian symmetric if it is almost symmetric!
if np.abs(Lnew - Lnew.T).sum() < np.spacing(1) * np.abs(Lnew).sum():
Lnew = (Lnew + Lnew.T) / 2.
if isinstance(G, graphs.Graph):
# Suppress the diagonal ? This is a good question?
Wnew = sparse.diags(Lnew.diagonal(), 0) - Lnew
Snew = Lnew.diagonal() - np.ravel(Wnew.sum(0))
if np.linalg.norm(Snew, 2) >= np.spacing(1000):
Wnew = Wnew + sparse.diags(Snew, 0)
# Removing diagonal for stability
Wnew = Wnew - Wnew.diagonal()
coords = G.coords[ind, :] if len(G.coords.shape) else np.ndarray(None)
Gnew = graphs.Graph(Wnew,
coords=coords,
lap_type=G.lap_type,
plotting=G.plotting)
else:
Gnew = Lnew
return Gnew
B[e, p, q] = Bpq
# se ensamblan la matriz de rigidez del elemento y el vector de
# fuerzas nodales equivalentes del elemento
Ke16 += Bpq.T @ De[mat[e]] @ Bpq * det_Je[p, q] * te[
mat[e]] * w_gl[p] * w_gl[q]
fe += Npq[:, :8].T @ be[mat[e]] * det_Je[p, q] * te[
mat[e]] * w_gl[p] * w_gl[q]
# se determina si hay puntos con jacobiano negativo, en caso tal se termina
# el programa y se reporta
if np.any(det_Je <= 0):
raise Exception(f'Hay puntos con det_Je negativo en el elemento {e+1}')
# se condensan los GDL jerárquicos u5, v5, u6, v6
Krr = Ke16[np.ix_(r_, r_)]
Ker[e] = Ke16[np.ix_(e_, r_)]
Kre = Ke16[np.ix_(r_, e_)]
inv_Kee[e] = np.linalg.inv(Ke16[np.ix_(e_, e_)])
Ke = Krr - Kre @ inv_Kee[e] @ Ker[e]
# y se añaden la matriz de rigidez del elemento y el vector de fuerzas
# nodales del elemento a sus respectivos arreglos de la estructura
idx[e] = gdl[LaG[e]].flatten() # se obtienen los grados de libertad
K[np.ix_(idx[e], idx[e])] += Ke
f[np.ix_(idx[e])] += fe
# %% Muestro la configuración de la matriz K (K es rala)
plt.figure()
plt.spy(K)
plt.title('Los puntos representan los elementos diferentes de cero')
def compute_symmetric_double_LOO(self):
#bevals_col = np.multiply(self.evals2, self.newevals2).T
#multiplyright = self.U.T * self.Y.T
#I = np.mat(np.identity(2))
G = np.multiply(
(self.newevals1.T - (1. / self.regparam1)), self.V) * self.V.T + (
1. / self.regparam1) * np.mat(np.identity(self.K1.shape[0]))
#G2 = np.multiply((self.newevals2.T-(1./self.regparam)), self.U) * self.U.T + (1./self.regparam) * np.mat(np.identity(self.K2.shape[0]))
GY = G * self.Y
#YG2 = self.Y * G2
GYG = GY * G
#A2 = G2 * self.Y.T
i, j = 2, 4
inds = [i, j]
#A = self.U[inds]
#right = multiplyright - A.T * self.Y.T[inds]
#RQY = A * np.multiply(bevals_col.T, right)
#B = np.multiply(bevals_col.T, A.T)
#HO_col = (la.inv(I - A * B) * RQY).T
#HO_col = (self.Y.T[inds]-la.inv(G2[np.ix_(inds, inds)]) * A2[inds]).T
#print HO_col.shape
#bevals_row = np.multiply(self.evals1, self.newevals1).T
#multiplyright = self.V.T * HO_col
#A = self.V[inds]
#right = multiplyright - A.T * HO_col[inds]
#RQY = A * np.multiply(bevals_row.T, right)
#B = np.multiply(bevals_col.T, A.T)
#HO_row = la.inv(I - A * B) * RQY
#A1 = G1[inds] * HO_col
#HO_row = HO_col[inds]-la.inv(G1[np.ix_(inds, inds)]) * A1
#HO_rowr = self.Y[np.ix_(inds, inds)] \
# - YG2[np.ix_(inds, inds)] * la.inv(G2[np.ix_(inds, inds)]) \
# - la.inv(G1[np.ix_(inds, inds)]) * G1Y[np.ix_(inds, inds)] \
# + la.inv(G1[np.ix_(inds, inds)]) * G1YG2[np.ix_(inds, inds)] * la.inv(G2[np.ix_(inds, inds)])
invGii = la.inv(G[np.ix_(inds, inds)])
GYii = GY[np.ix_(inds, inds)]
invGiiGYii = invGii * GYii
HO_rowr = self.Y[np.ix_(inds, inds)] \
- invGiiGYii.T \
- invGiiGYii \
+ invGii * GYG[np.ix_(inds, inds)] * invGii
#II1 = np.mat(np.identity(self.Y.shape[0]))[inds]
#II2 = np.mat(np.identity(self.Y.shape[1]))[:, inds]
#HO_rowr = (II1 - la.inv(G1[np.ix_(inds, inds)]) * G1[inds]) * self.Y * (II2 - G2[:, inds] * la.inv(G2[np.ix_(inds, inds)]))
#print HO_row.shape
results = np.zeros((self.Y.shape[0], self.Y.shape[1]))
cython_two_step_rls_cv.compute_symmetric_double_loo(
G, self.Y, GY, GYG, results, self.Y.shape[0], self.Y.shape[1])
return results
dict = {'Size': i}
start = time.clock()
result = m[:, indexSet][indexSet, :]
result @ result.T
end = time.clock()
mstart = time.clock()
result @ result.T
mend = time.clock()
mtime = mend - mstart
dict['FancyIndexing'] = end - start - mtime
start = time.clock()
result3 = m[np.ix_(indexSet, indexSet)]
result3 @ result3.T
end = time.clock()
mstart = time.clock()
result3 @ result3.T
mend = time.clock()
mtime = mend - mstart
dict['ix'] = end - start - mtime
frame = frame.append(dict, ignore_index=True)
frame.to_csv('intel/timeNPTakeSparse' + str(x) + '.csv', index=False)
names = glob.glob('intel/timeNPTakeSparse[0-9].csv')
def generateSpectralPoints(self, outfile, union=False, ndiimgfile=None):
"""
Main function to generate spectral points
"""
if union and (ndiimgfile is None):
raise RuntimeError(
"You must provdie a gap-filled NDI image to generation spectral points from the union of two point clouds"
)
self.union = union
self.ndiimgfile = ndiimgfile
# get inputs from command line or defaults
nirfile = self.nirfile
swirfile = self.swirfile
rdiff_thresh = float(self.rdiff_thresh)
nrows = int(self.nrows)
ncols = int(self.ncols)
print "Input NIR point cloud: \n\t" + nirfile
print "Input SWIR point cloud: \n\t" + swirfile
print "Output merged point cloud: \n\t" + outfile
print "Range difference threshold: {0:.3f}".format(rdiff_thresh)
print "Info for converting sample and line to shot number: \n" \
+ "\tnumber of rows: {0:d}, number of columns: {1:d}".format(nrows, ncols)
# read points from text file
print "Loading points"
nirpoints = np.genfromtxt(nirfile, dtype=np.float32, usecols=self.ind, \
delimiter=',', skip_header=self.headerlines, \
filling_values=np.nan, usemask=False, \
comments=None)
swirpoints = np.genfromtxt(swirfile, dtype=np.float32, usecols=self.ind, \
delimiter=',', skip_header=self.headerlines, \
filling_values=np.nan, usemask=False, \
comments=None)
# update column index in loaded array
cind = {il: i for i, il in enumerate(self.ind_label)}
# index the columns that are used in searching common points
ind = [cind['range'], cind['sample'], cind['line']]
print "Generating spectral points ..."
intersectout = self.intersectPointClouds(nirpoints[:, ind],
swirpoints[:, ind])
nir_ind = intersectout[0]
swir_ind = intersectout[1]
return_type = intersectout[2]
nu_shotnum = intersectout[3]
num_of_returns = intersectout[4]
return_num = intersectout[5]
# return_type now gives QA flag QA flag is a three-bit integer, from MSB
# to LSB, each bit tells if NDI, NIR or SWIR is measured value (0) or
# synthetic value (1)
return_type[:] = int('000', 2)
# calculate the mean point location
ind = [cind['x'], cind['y'], cind['z'], cind['shot_number'], cind['range'], cind['theta'], \
cind['phi'], cind['sample'], cind['line']]
nreturn = len(return_type)
meanpoints = (nirpoints[np.ix_(nir_ind, ind)] +
swirpoints[np.ix_(swir_ind, ind)]) / 2.0
if self.man_col < self.tot_col:
outpoints = np.hstack((meanpoints[:, 0:3], \
nirpoints[nir_ind, cind['d_I']:cind['d_I']+1], \
swirpoints[swir_ind, cind['d_I']:cind['d_I']+1], \
return_num.reshape((nreturn, 1)), \
num_of_returns.reshape((nreturn, 1)), \
nu_shotnum.reshape((nreturn, 1)), \
meanpoints[:, 4:9], \
nirpoints[nir_ind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
swirpoints[swir_ind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
return_type.reshape((nreturn, 1)) \
))
else:
outpoints = np.hstack((meanpoints[:, 0:3], \
nirpoints[nir_ind, cind['d_I']:cind['d_I']+1], \
swirpoints[swir_ind, cind['d_I']:cind['d_I']+1], \
return_num.reshape((nreturn, 1)), \
num_of_returns.reshape((nreturn, 1)), \
nu_shotnum.reshape((nreturn, 1)), \
meanpoints[:, 4:9], \
return_type.reshape((nreturn, 1)) \
))
prefixstr = "[DWEL Dual-wavelength Point Cloud Data by {0:s}]\n".format("Union" if self.union else "Intersect") \
+"Run made at: "+time.strftime("%c")+"\n"
headerstr = prefixstr + "x,y,z,d_I_nir,d_I_swir,return_number,number_of_returns,shot_number,range,theta,phi,sample,line," \
+ ",".join([ il+"_nir" for il in self.ind_label[self.man_col:self.tot_col] ]) + "," \
+ ",".join([ il+"_swir" for il in self.ind_label[self.man_col:self.tot_col] ]) + "," \
+ "qa,r,g,b"
# format string
if self.man_col < self.tot_col:
col_dtype = self.inferColDataType()
fmtstr = "%.3f "*5 + "%d "*3 + "%.3f "*3 + "%d "*2 \
+ " ".join([ col_dtype[ci] for ci in self.ind[self.man_col:self.tot_col] ]) + " " \
+ " ".join([ col_dtype[ci] for ci in self.ind[self.man_col:self.tot_col] ]) + " " \
+ "%d "*4
else:
col_dtype = self.inferColDataType()
fmtstr = "%.3f "*5 + "%d "*3 + "%.3f "*3 + "%d "*2 \
+ "%d "*4
fmtstr = fmtstr.strip().split(" ")
if not (self.union):
# spectral points by intersect
# exclude zero-hit points from rgb generation
rgbpoints = np.zeros((outpoints.shape[0], 3))
hitmask = outpoints[:, 6].astype(int) > 0
# fix extremely large reflectance values
bound = np.percentile(outpoints[:, 3:5][hitmask, :], 98, axis=0)
bound[bound < self.i_scale] = self.i_scale
outpoints[outpoints[:, 3] > bound[0], 3] = bound[0]
outpoints[outpoints[:, 4] > bound[1], 4] = bound[1]
# generate pseudo-color composite
rgbpoints[hitmask, :] = self.colorComposite(np.hstack((outpoints[:, 4:5][hitmask, :], \
outpoints[:, 3:4][hitmask, :], \
np.zeros_like(outpoints[:, 3:4][hitmask, :]) \
)))
outpoints = np.hstack((outpoints, rgbpoints))
print "Saving dual-wavelength points: " + str(nir_ind.size)
np.savetxt(outfile, outpoints, delimiter=',', fmt=fmtstr, \
header=headerstr.rstrip(), comments='')
else:
# spectral points by union
nir_unpind = intersectout[6]
swir_unpind = intersectout[7]
nir_unp_nu_shotnum = intersectout[8]
swir_unp_nu_shotnum = intersectout[9]
ind = [cind['d_I'], cind['sample'], cind['line']]
print "\tMatching more point pairs between NIR and SWIR for union approach"
unionout = self.unionPointClouds(nirpoints[np.ix_(nir_unpind, ind)], \
swirpoints[np.ix_(swir_unpind, ind)], \
ndiimgfile)
nir2swir_amp = unionout[0]
nir2swir_qa = unionout[1]
swir2nir_amp = unionout[2]
swir2nir_qa = unionout[3]
tmpflag = np.equal(nir2swir_qa, int('111', 2))
if np.greater(nirpoints[nir_unpind[tmpflag], cind['d_I']],
1e-10).any():
warnings.warn(
"Some no-return shots give non-zero NIR reflectance",
RuntimeWarning)
nirpoints[nir_unpind[tmpflag], cind['d_I']] = 0.0
tmpflag = np.equal(swir2nir_qa, int('111', 2))
if np.greater(swirpoints[swir_unpind[tmpflag], cind['d_I']],
1e-10).any():
warnings.warn(
"Some no-return shots give non-zero SWIR reflectance",
RuntimeWarning)
swirpoints[swir_unpind[tmpflag], cind['d_I']] = 0.0
if self.man_col < self.tot_col:
nir2swir_points = np.hstack(( nirpoints[nir_unpind, 0:3], \
nirpoints[nir_unpind, cind['d_I']:cind['d_I']+1], \
nir2swir_amp.reshape((len(nir2swir_amp), 1)), \
np.zeros((len(nir2swir_amp), 3)), \
nirpoints[nir_unpind, :][:, 5:10], \
nirpoints[nir_unpind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
np.zeros((len(nir2swir_amp), self.tot_col-self.man_col)), \
nir2swir_qa.reshape((len(nir2swir_qa), 1)) \
))
swir2nir_points = np.hstack(( swirpoints[swir_unpind, 0:3], \
swir2nir_amp.reshape((len(swir2nir_amp), 1)), \
swirpoints[swir_unpind, cind['d_I']:cind['d_I']+1], \
np.zeros((len(swir2nir_amp), 3)), \
swirpoints[swir_unpind, :][:, 5:10], \
np.zeros((len(swir2nir_amp), self.tot_col-self.man_col)), \
swirpoints[swir_unpind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
swir2nir_qa.reshape((len(swir2nir_qa), 1)) \
))
else:
nir2swir_points = np.hstack(( nirpoints[nir_unpind, 0:3], \
nirpoints[nir_unpind, cind['d_I']:cind['d_I']+1], \
nir2swir_amp.reshape((len(nir2swir_amp), 1)), \
np.zeros((len(nir2swir_amp), 3)), \
nirpoints[nir_unpind, :][:, 5:10], \
nir2swir_qa.reshape((len(nir2swir_qa), 1)) \
))
swir2nir_points = np.hstack(( swirpoints[swir_unpind, 0:3], \
swir2nir_amp.reshape((len(swir2nir_amp), 1)), \
swirpoints[swir_unpind, cind['d_I']:cind['d_I']+1], \
np.zeros((len(swir2nir_amp), 3)), \
swirpoints[swir_unpind, :][:, 5:10], \
swir2nir_qa.reshape((len(swir2nir_qa), 1)) \
))
unionoutpoints = np.vstack(
(outpoints, nir2swir_points, swir2nir_points))
sortind, num_of_returns, return_num, shotind = \
self.updateReturnNum(unionoutpoints[:, [8, 11, 12]].copy())
unionoutpoints = unionoutpoints[sortind, :]
unionoutpoints[:, 5] = return_num
unionoutpoints[:, 6] = num_of_returns
unionoutpoints[:, 7] = shotind
# exclude zero-hit points from rgb generation
rgbpoints = np.zeros((unionoutpoints.shape[0], 3))
hitmask = unionoutpoints[:, 6].astype(int) > 0
# fix extremely large reflectance values
bound = np.percentile(unionoutpoints[:, 3:5][hitmask, :],
98,
axis=0)
bound[bound < self.i_scale] = self.i_scale
unionoutpoints[unionoutpoints[:, 3] > bound[0], 3] = bound[0]
unionoutpoints[unionoutpoints[:, 4] > bound[1], 4] = bound[1]
# generate pseudo-color composite
rgbpoints[hitmask, :] = self.colorComposite(np.hstack((unionoutpoints[:, 4:5][hitmask, :], \
unionoutpoints[:, 3:4][hitmask, :], \
np.zeros_like(unionoutpoints[:, 3:4][hitmask, :]) \
)))
unionoutpoints = np.hstack((unionoutpoints, rgbpoints))
print "Saving dual-wavelength points: " + str(len(unionoutpoints))
np.savetxt(outfile, unionoutpoints, delimiter=',', fmt=fmtstr, \
header=headerstr.rstrip(), comments='')
import scipy.misc
import matplotlib.pyplot as plt
import numpy as np
# Load the Lena array
lena = scipy.misc.lena()
xmax = lena.shape[0]
ymax = lena.shape[1]
def shuffle_indices(size):
'''
Shuffles an array with values 0 - size
'''
arr = np.arange(size)
np.random.shuffle(arr)
return arr
xindices = shuffle_indices(xmax)
np.testing.assert_equal(len(xindices), xmax)
yindices = shuffle_indices(ymax)
np.testing.assert_equal(len(yindices), ymax)
# Plot Lena
plt.imshow(lena[np.ix_(xindices, yindices)])
plt.show()
#----------------------------------------------------------------------
# Calculate K
#----------------------------------------------------------------------
totDof = 2 * len(coords)
K = zeros(shape=(totDof, totDof))
for elem in elems:
elemDofs = getDofs(elem)
sData = getElemShapeData(coords[elem, :])
for iData in sData:
b = getBmatrix(iData.dhdx)
K[ix_(elemDofs,
elemDofs)] += dot(b.transpose(), dot(D, b)) * iData.weight
#----------------------------------------------------------------------
# Solve Ka=f
#----------------------------------------------------------------------
consDof = len(presInds)
C = zeros(shape=(totDof, totDof - consDof))
j = 0
for i in range(totDof):
if i in presInds:
continue
C[i, j] = 1.
def regression(self, i: int, c: list=None, lam=0):
S, P = self.S, self.P
d = [j for j in range(self.p) if j not in c and j != i] + [i]
if c is None or len(c) == 0:
coefs = []
var = S[i, i]
S_inv = None
# use Schur complement when conditioning to keep inverted submatrix small
elif len(c) < self.p / 2 or P is None:
if lam == 0 and np.isclose(np.diag(S[ix_(c, c)]), 0).any():
coefs, var, _, _ = lstsq(S[ix_(c, c)], S[c, i])
var = S[i, i] - S[i, c] @ coefs
S_inv = pinv(S[ix_(c, c)])
else:
try:
S_inv = inv(S[ix_(c, c)] + lam*np.eye(len(c)))
coefs = S_inv @ S[c, i]
var = S[i, i] - S[i, c] @ S_inv @ S[c, i]
except LinAlgError:
coefs, var, _, _ = lstsq(S[ix_(c, c)], S[c, i])
var = S[i, i] - S[i, c] @ coefs
S_inv = pinv(S[ix_(c, c)])
# use Schur complement when marginalizing to keep inverted submatrix small
else:
P_inv = inv(P[ix_(d, d)])
S_inv = P[ix_(c, c)] - P[ix_(c, d)] @ P_inv @ P[ix_(d, c)]
coefs = S_inv @ S[c, i]
var = inv(P[ix_(d, d)])[-1, -1]
# correct the variance to account for the number of degrees of freedom
var = var * self.n / (self.n - len(c) + 1)
return coefs, var, S_inv
def run_dsistudio(subj,
subjdir,
masks_dir,
preproc_dir,
dset_dir,
modality,
shell='10k',
nr_fibers=20000,
trk_output='tracts20k',
suppress_output=False,
preproc_stem=None):
import subprocess
if modality in ('qball', 'qbi'):
stem = 'diff_disco_eddy'
methodnr = '4'
methodname = 'gqi'
param0 = '1.25'
param1 = None
btable_string = 'btable.txt'
elif modality in ('singleshell', 'sgsh'):
stem = 'diff_singleshell_%s' % shell
methodnr = '3'
methodname = 'qbi'
param0 = '0.006'
param1 = '8'
btable_string = 'btable_singleshell_%s.txt' % shell
trk_output = '%s_sgsh_%s' % (trk_output, shell)
elif modality in ('dsi', ):
stem = 'data_raw_disco_eddy-vol_moco_clean_ordered'
methodnr = '0'
methodname = 'dsi'
param0 = '17'
param1 = None
btable_string = 'dsi515_b_table.txt'
elif modality in ('dti', ):
#stem = 'dti_unprocessed'
stem = 'dti'
methodnr = 1
methodname = 'dti'
param0 = None
param1 = None
btable_string = 'btable.txt'
else:
sys.stderr.write('Unrecognized modality %s\n' % modality)
if preproc_stem is not None:
stem = preproc_stem
try:
os.mkdir(preproc_dir)
except OSError:
pass
if os.path.exists(os.path.join(preproc_dir, '%s.nii' % stem)):
gzip_cmd = ('mri_convert %s %s' %
(os.path.join(preproc_dir, '%s.nii' % stem),
os.path.join(preproc_dir, '%s.nii.gz' % stem)))
print gzip_cmd
m = subprocess.call(gzip_cmd, shell=True)
if modality in ('qball', 'qbi', 'singleshell', 'sgsh'):
src_cmd = (
'dsi_studio --action=src --source=%s --b_table=%s --output=%s' %
(os.path.join(preproc_dir, '%s.nii.gz' % stem),
os.path.join(preproc_dir, btable_string),
os.path.join(preproc_dir, '%s.src.gz' % stem)))
elif modality in ('dsi', ):
src_cmd = (
'dsi_studio --action=src --source=%s --b_table=%s --output=%s' %
(os.path.join(preproc_dir, '%s.nii.gz' % stem),
os.path.join(subjdir, btable_string),
os.path.join(preproc_dir, '%s.src.gz' % stem)))
elif modality in ('dti', ):
src_cmd = (
'dsi_studio --action=src --source=%s --output=%s --b_table=%s' %
(os.path.join(preproc_dir, 'dti.nii.gz'),
os.path.join(preproc_dir, '%s.src.gz' % stem),
os.path.join(preproc_dir, btable_string)))
print src_cmd
p = subprocess.call(src_cmd, shell=True)
rec_cmd = ('dsi_studio --action=rec --source=%s --thread=10 --method=%s '
'--num_fiber=8 --odf_order=8 --record_odf=1 --mask=%s' %
(os.path.join(preproc_dir, '%s.src.gz' % stem), methodnr,
os.path.join(dset_dir, 'wm_b0.nii.gz')))
if param0 is not None:
rec_cmd += ' --param0=%s' % param0
if param1 is not None:
rec_cmd += ' --param1=%s' % param1
print rec_cmd
q = subprocess.call(rec_cmd, shell=True)
trk_cmd = (
'dsi_studio --action=trk --source=%s --fiber_count=%i '
'--output=%s' % (
os.path.join(
preproc_dir, '%s.src.gz.%s%s.%sfib.gz' %
(stem, '' if methodname == 'dti' else 'odf8.f8rec.',
methodname,
reduce(lambda x, y: x + y, [
'sh%s.' % p if i == 0 else '%s.' % p
for i, p in enumerate(
(param0, param1)[::-1]) if p is not None
], ''))),
nr_fibers,
#os.path.join(subjdir,subj,'%s.trk'%trk_output),
os.path.join(dset_dir, 'tracks', '%s.%s' % (trk_output, '%s'))))
print trk_cmd
r1 = subprocess.call(trk_cmd % 'txt', shell=True)
r2 = subprocess.call(trk_cmd % 'trk', shell=True)
for parc in ('laus250', ):
ana_cmd = (
'dsi_studio --action=ana --source=%s --tract=%s '
'--export=connectivity --roi=%s' % (
os.path.join(
preproc_dir, '%s.src.gz.%s%s.%sfib.gz' %
(stem, '' if methodname == 'dti' else 'odf8.f8rec.',
methodname,
reduce(lambda x, y: x + y, [
'sh%s.' % p if i == 0 else '%s.' % p
for i, p in enumerate(
(param0, param1)[::-1]) if p is not None
], ''))),
#os.path.join(subjdir,subj,'%s.trk'%trk_output),
os.path.join(dset_dir, 'tracks', '%s.txt' % trk_output),
os.path.join(masks_dir, 'bigmask_%s_wmreg.nii.gz' % parc)))
print ana_cmd
s = subprocess.call(ana_cmd, shell=True)
#==================================================================
#put the resulting matrix back in alphabetical order
from scipy import io as sio
import numpy as np
#fname=os.path.join(subjdir,subj, '%s.trk.connectivity.mat'%trk_output)
fname = os.path.join(dset_dir, 'tracks',
'%s.txt.connectivity.mat' % trk_output)
adj = sio.loadmat(fname)
labnam = adj['name']
med_length = adj['tract_median_length']
adj = adj['connectivity']
adj = adj / med_length
adj[np.isnan(adj)] = 0
labnam = ''.join(map(chr, labnam.flat)).split('\n')[:-1]
stem = len(os.path.commonprefix(labnam))
ord = np.array(map(lambda x: int(x[stem:]) - 1, labnam))
#dsistudio indexing to python indexing
keys = {}
for i, k in enumerate(ord):
keys.update({k: i})
ord = map(keys.get, xrange(len(ord)))
try:
adj = adj[np.ix_(ord, ord)]
except IndexError:
print ord
raise IndexError('The mask has too few ROIs')
np.save(
os.path.join(dset_dir, 'tracks',
'%s_alph_normed_%s.npy' % (trk_output, parc)), adj)
def Entropy_Thompson(data, hyperparameters, cz, cy, B=50, N=10, ploton=False):
x, z, y = data
az, bz, lz, ay, by, theta, lyx, lyz = hyperparameters
top = np.argsort(y)[-N:]
n = x.shape[0]
Dx = euclidean_distances(x.reshape(-1, 1), x.reshape(-1, 1), squared=True)
SIG_z = az**2 * np.exp(-Dx / (2 * lz**2)) + bz**2 * np.identity(n)
Dz = euclidean_distances(z.reshape(-1, 1), z.reshape(-1, 1), squared=True)
SIG_y = ay**2 * np.exp(-Dx * lyx**2 / 2 -
Dz * lyz**2 / 2) + by**2 * np.identity(n)
""" samples from posterior """
def samples(tt, tu, uu, nz, nzy, ztttu, ytt):
mu_cz = np.matmul(
SIG_z[np.ix_(uu, tt + tu)],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], ztttu))
SIG_cz = SIG_z[np.ix_(uu, uu)] - np.matmul(
SIG_z[np.ix_(uu, tt + tu)],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[np.ix_(
tt + tu, uu)]))
ZS = np.zeros((n, nz))
ZS[tt + tu, :] = np.repeat(ztttu.reshape(-1, 1), nz, 1)
ZS[uu, :] = np.random.multivariate_normal(mu_cz, SIG_cz, nz).T
YS = np.zeros((n, nz * nzy))
YS[tt, :] = np.repeat(ytt.reshape(-1, 1), nz * nzy, 1)
for i in range(nz):
sampled_SIG_y = ay**2 * np.exp(
-Dx * lyx**2 / 2 - euclidean_distances(ZS[:, i].reshape(-1, 1),
ZS[:, i].reshape(-1, 1),
squared=True) * lyz**2 /
2) + by**2 * np.identity(n)
sampled_mu_cy = np.matmul(
sampled_SIG_y[np.ix_(tu + uu, tt)],
np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)], ytt))
sampled_SIG_cy = sampled_SIG_y[np.ix_(
tu + uu, tu + uu)] - np.matmul(
sampled_SIG_y[np.ix_(tu + uu, tt)],
np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)],
sampled_SIG_y[np.ix_(tt, tu + uu)]))
YS[tu + uu, i * nzy:(i + 1) * nzy] = np.random.multivariate_normal(
sampled_mu_cy, sampled_SIG_cy, nzy).T
return ZS, YS
def entropy(x):
I = [i for i in range(len(x)) if x[i] > 0 and x[i] < 1]
H = -np.multiply(x[I], np.log(x[I]))
h = sum(H)
return h
r = 1
tt = list(range(r))
tu = []
uu = list(range(r, n))
b = B
History = []
res = 5
ZR = np.linspace(-2, 2, res)
PR = norm.pdf(ZR)
PR = PR / np.sum(PR)
while b > cy:
""" sample from posterior to estimate greedy-N acquisituion function """
ZS, alpha = samples(tt, tu, uu, 2, 1, z[tt + tu], y[tt])
if len(tu) > 0:
""" select best candidates from TU and UU """
itu = tu[np.argmax(alpha[tu, 0])]
iuu = uu[np.argmax(alpha[uu, 1])]
""" integrate over zuu to estimate profit of action z """
mu_uuz = np.matmul(
SIG_z[iuu, tt + tu],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], z[tt + tu]))
SIG_uuz = SIG_z[iuu, iuu] - np.matmul(
SIG_z[iuu, tt + tu],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[tt + tu,
iuu]))
mu_tuy = np.matmul(SIG_y[itu, tt],
np.linalg.solve(SIG_y[np.ix_(tt, tt)], y[tt]))
SIG_tuy = SIG_y[itu, itu] - np.matmul(
SIG_y[itu, tt],
np.linalg.solve(SIG_y[np.ix_(tt, tt)], SIG_y[tt, iuu]))
ZuuR = mu_uuz + ZR * SIG_uuz**0.5
YtuR = mu_tuy + ZR * SIG_tuy**0.5
uuc = uu.copy()
uuc.remove(iuu)
tuc = tu.copy()
tuc.remove(itu)
tuc.append(iuu)
ttc = tt.copy()
ttc.append(itu)
mz = 5
my = 100
P = np.zeros((n, res, res))
for i in range(res):
for j in range(res):
ZS, YS = samples(
ttc, tuc, uuc, mz, 100,
np.concatenate((z[tt + tu], ZuuR[i].reshape(1))),
np.concatenate((y[tt], YtuR[j].reshape(1))))
for k in range(mz * my):
I = [np.argpartition(YS[:, k], -N)[-N:]]
P[I, i, j] += 1
P = P / (mz * my)
p = np.zeros(n)
for i in range(res):
for j in range(res):
p = p + PR[j] * PR[i] * P[:, i, j]
H = entropy(p)
Hz = np.zeros(res)
for i in range(res):
p = np.zeros(n)
for j in range(res):
p = p + PR[j] * P[:, i, j]
Hz[i] = entropy(p)
Hy = np.zeros(res)
for j in range(res):
p = np.zeros(n)
for i in range(res):
p = p + PR[i] * P[:, i, j]
Hy[j] = entropy(p)
DEz = np.dot(H - Hz, PR)
DEy = np.dot(H - Hy, PR)
if ploton:
plt.figure(figsize=(15, 5))
plt.subplot(121)
plt.plot(x[top], z[top], '.', color='red')
plt.plot(x[tu], z[tu], 's', color='black')
plt.plot(x[tt], z[tt], 'd', color='red')
plt.plot([x[iuu], x[iuu]], [np.min(z), np.max(z)], color='red')
plt.plot(x[iuu], z[iuu], 'x', color='red')
plt.plot(x[itu], z[itu], 'x', markersize=25, color='red')
plt.scatter(x, z, 10, y)
plt.xlabel('x')
plt.ylabel('z')
plt.subplot(122)
plt.plot(ZuuR, Hz, label='z')
plt.plot([ZuuR[0], ZuuR[-1]], [H, H])
plt.plot(YtuR, Hy, label='y')
plt.plot([YtuR[0], YtuR[-1]], [H, H])
plt.legend()
plt.xlabel('z/y')
plt.ylabel('expected entropy')
plt.show()
""" reward/cost ratio of different actions """
alphay = DEy / cy
alphaz = DEz / cz
if alphay > alphaz:
tu.remove(itu)
tt.append(itu)
b = b - cy
History.append([itu, 'y'])
else:
uu.remove(iuu)
tu.append(iuu)
b = b - cz
History.append([iuu, 'z'])
else:
iuu = uu[np.argmax(alpha[uu, 0])]
uu.remove(iuu)
tu.append(iuu)
b = b - cz
History.append([iuu, 'z'])
if ploton:
plt.figure(figsize=(15, 5))
plt.subplot(121)
plt.plot(x[top], z[top], '.', color='red')
plt.plot(x[tu], z[tu], 's', color='black')
plt.plot(x[tt], z[tt], 'd', color='red')
if History[-1][1] == 'y':
plt.plot(x[tt[-1]], z[tt[-1]], 'd', color='red', markersize=15)
else:
plt.plot(x[tu[-1]],
z[tu[-1]],
's',
color='black',
linewidth=3,
markersize=15)
plt.scatter(x, z, 10, y)
plt.xlabel('x')
plt.ylabel('z')
plt.subplot(122)
if History[-1][1] == 'y':
plt.plot(x[tt[-1]], z[tt[-1]], 'd', color='red', markersize=15)
else:
plt.plot(x[tu[-1]],
z[tu[-1]],
's',
color='black',
markersize=15)
plt.scatter(x[tt], z[tt], 50, y[tt])
plt.plot(x[tu], z[tu], 's', color='black')
plt.plot(x[uu], np.ones(len(uu)) * np.min(z), 'x')
plt.xlabel('x')
plt.ylabel('z')
plt.show()
print(b)
ZS, YS = samples(tt, tu, uu, 10, 100, z[tt + tu], y[tt])
P = np.zeros(n)
for i in range(1000):
I = [np.argpartition(YS[:, i], -N)[-N:]]
P[I] += 1
P = P / 1000
H = entropy(P)
History.append([H, 'final_entropy'])
return History
def F(self, x: np.ndarray, Ts: float) -> np.ndarray:
F = self._F_mat
F[np.ix_(self._all_idx, self._all_idx)] = F_CT(x[self._all_idx], Ts)
return F.copy()
"""
import os
import pickle
import pygsp
import numpy as np
from scipy import sparse as sp
os.chdir('..')
# One month graph construction
data = pickle.load(open('temporal_citation/1992-02', 'rb'))
auth_idx = data['active_authors'].nonzero()[0]
W = data['W'][np.ix_(auth_idx, auth_idx)]
G = pygsp.graphs.Graph(W)
G.set_coords(iterations=10)
G.plot(show_edges=True)
# Aggregation of several months
N = 366572
W = sp.csr_matrix((N, N))
# active_authors = sp.csr_matrix((N, 1), dtype='bool')
from_year = 2002
to_year = 2004
from_month = 01
to_month = 12
def samples(tt, tu, uu, nz, nzy, ztttu, ytt):
mu_cz = np.matmul(
SIG_z[np.ix_(uu, tt + tu)],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], ztttu))
SIG_cz = SIG_z[np.ix_(uu, uu)] - np.matmul(
SIG_z[np.ix_(uu, tt + tu)],
np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[np.ix_(
tt + tu, uu)]))
ZS = np.zeros((n, nz))
ZS[tt + tu, :] = np.repeat(ztttu.reshape(-1, 1), nz, 1)
ZS[uu, :] = np.random.multivariate_normal(mu_cz, SIG_cz, nz).T
YS = np.zeros((n, nz * nzy))
YS[tt, :] = np.repeat(ytt.reshape(-1, 1), nz * nzy, 1)
for i in range(nz):
sampled_SIG_y = ay**2 * np.exp(
-Dx * lyx**2 / 2 - euclidean_distances(ZS[:, i].reshape(-1, 1),
ZS[:, i].reshape(-1, 1),
squared=True) * lyz**2 /
2) + by**2 * np.identity(n)
sampled_mu_cy = np.matmul(
sampled_SIG_y[np.ix_(tu + uu, tt)],
np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)], ytt))
sampled_SIG_cy = sampled_SIG_y[np.ix_(
tu + uu, tu + uu)] - np.matmul(
sampled_SIG_y[np.ix_(tu + uu, tt)],
np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)],
sampled_SIG_y[np.ix_(tt, tu + uu)]))
YS[tu + uu, i * nzy:(i + 1) * nzy] = np.random.multivariate_normal(
sampled_mu_cy, sampled_SIG_cy, nzy).T
return ZS, YS
def fit_elasticnet(model,
method="coord_descent",
maxiter=100,
alpha=0.,
L1_wt=1.,
start_params=None,
cnvrg_tol=1e-7,
zero_tol=1e-8,
refit=False,
check_step=True,
loglike_kwds=None,
score_kwds=None,
hess_kwds=None):
"""
Return an elastic net regularized fit to a regression model.
Parameters
----------
model : model object
A statsmodels object implementing ``loglike``, ``score``, and
``hessian``.
method :
Only the coordinate descent algorithm is implemented.
maxiter : int
The maximum number of iteration cycles (an iteration cycle
involves running coordinate descent on all variables).
alpha : scalar or array_like
The penalty weight. If a scalar, the same penalty weight
applies to all variables in the model. If a vector, it
must have the same length as `params`, and contains a
penalty weight for each coefficient.
L1_wt : scalar
The fraction of the penalty given to the L1 penalty term.
Must be between 0 and 1 (inclusive). If 0, the fit is
a ridge fit, if 1 it is a lasso fit.
start_params : array_like
Starting values for `params`.
cnvrg_tol : scalar
If `params` changes by less than this amount (in sup-norm)
in one iteration cycle, the algorithm terminates with
convergence.
zero_tol : scalar
Any estimated coefficient smaller than this value is
replaced with zero.
refit : bool
If True, the model is refit using only the variables that have
non-zero coefficients in the regularized fit. The refitted
model is not regularized.
check_step : bool
If True, confirm that the first step is an improvement and search
further if it is not.
loglike_kwds : dict-like or None
Keyword arguments for the log-likelihood function.
score_kwds : dict-like or None
Keyword arguments for the score function.
hess_kwds : dict-like or None
Keyword arguments for the Hessian function.
Returns
-------
A results object.
Notes
-----
The ``elastic net`` penalty is a combination of L1 and L2
penalties.
The function that is minimized is:
-loglike/n + alpha*((1-L1_wt)*|params|_2^2/2 + L1_wt*|params|_1)
where |*|_1 and |*|_2 are the L1 and L2 norms.
The computational approach used here is to obtain a quadratic
approximation to the smooth part of the target function:
-loglike/n + alpha*(1-L1_wt)*|params|_2^2/2
then repeatedly optimize the L1 penalized version of this function
along coordinate axes.
"""
k_exog = model.exog.shape[1]
loglike_kwds = {} if loglike_kwds is None else loglike_kwds
score_kwds = {} if score_kwds is None else score_kwds
hess_kwds = {} if hess_kwds is None else hess_kwds
if np.isscalar(alpha):
alpha = alpha * np.ones(k_exog)
# Define starting params
if start_params is None:
params = np.zeros(k_exog)
else:
params = start_params.copy()
btol = 1e-4
params_zero = np.zeros(len(params), dtype=bool)
init_args = model._get_init_kwds()
# we do not need a copy of init_args b/c get_init_kwds provides new dict
init_args['hasconst'] = False
model_offset = init_args.pop('offset', None)
if 'exposure' in init_args and init_args['exposure'] is not None:
if model_offset is None:
model_offset = np.log(init_args.pop('exposure'))
else:
model_offset += np.log(init_args.pop('exposure'))
fgh_list = [
_gen_npfuncs(k, L1_wt, alpha, loglike_kwds, score_kwds, hess_kwds)
for k in range(k_exog)
]
for itr in range(maxiter):
# Sweep through the parameters
params_save = params.copy()
for k in range(k_exog):
# Under the active set method, if a parameter becomes
# zero we do not try to change it again.
# TODO : give the user the option to switch this off
if params_zero[k]:
continue
# Set the offset to account for the variables that are
# being held fixed in the current coordinate
# optimization.
params0 = params.copy()
params0[k] = 0
offset = np.dot(model.exog, params0)
if model_offset is not None:
offset += model_offset
# Create a one-variable model for optimization.
model_1var = model.__class__(model.endog,
model.exog[:, k],
offset=offset,
**init_args)
# Do the one-dimensional optimization.
func, grad, hess = fgh_list[k]
params[k] = _opt_1d(func,
grad,
hess,
model_1var,
params[k],
alpha[k] * L1_wt,
tol=btol,
check_step=check_step)
# Update the active set
if itr > 0 and np.abs(params[k]) < zero_tol:
params_zero[k] = True
params[k] = 0.
# Check for convergence
pchange = np.max(np.abs(params - params_save))
if pchange < cnvrg_tol:
break
# Set approximate zero coefficients to be exactly zero
params[np.abs(params) < zero_tol] = 0
if not refit:
results = RegularizedResults(model, params)
return RegularizedResultsWrapper(results)
# Fit the reduced model to get standard errors and other
# post-estimation results.
ii = np.flatnonzero(params)
cov = np.zeros((k_exog, k_exog))
init_args = dict([(k, getattr(model, k, None)) for k in model._init_keys])
if len(ii) > 0:
model1 = model.__class__(model.endog, model.exog[:, ii], **init_args)
rslt = model1.fit()
params[ii] = rslt.params
cov[np.ix_(ii, ii)] = rslt.normalized_cov_params
else:
# Hack: no variables were selected but we need to run fit in
# order to get the correct results class. So just fit a model
# with one variable.
model1 = model.__class__(model.endog, model.exog[:, 0], **init_args)
rslt = model1.fit(maxiter=0)
# fit may return a results or a results wrapper
if issubclass(rslt.__class__, wrap.ResultsWrapper):
klass = rslt._results.__class__
else:
klass = rslt.__class__
# Not all models have a scale
if hasattr(rslt, 'scale'):
scale = rslt.scale
else:
scale = 1.
# The degrees of freedom should reflect the number of parameters
# in the refit model, not including the zeros that are displayed
# to indicate which variables were dropped. See issue #1723 for
# discussion about setting df parameters in model and results
# classes.
p, q = model.df_model, model.df_resid
model.df_model = len(ii)
model.df_resid = model.nobs - model.df_model
# Assuming a standard signature for creating results classes.
refit = klass(model, params, cov, scale=scale)
refit.regularized = True
refit.method = method
refit.fit_history = {'iteration': itr + 1}
# Restore df in model class, see issue #1723 for discussion.
model.df_model, model.df_resid = p, q
return refit
def create_new_grid(self):
# GRID DEFINITION IN DEGREES
self.lon0_deg = lon0_deg
self.lon1_deg = lon1_deg
self.lat0_deg = lat0_deg
self.lat1_deg = lat1_deg
self.dlon_deg = dlon_deg
self.dlat_deg = dlat_deg
# GRID DEFINITION IN RADIANS
self.lon0_rad = self.lon0_deg/180*np.pi
self.lon1_rad = self.lon1_deg/180*np.pi
self.lat0_rad = self.lat0_deg/180*np.pi
self.lat1_rad = self.lat1_deg/180*np.pi
self.dlon_rad_1D = self.dlon_deg/180*np.pi
self.dlat_rad_1D = self.dlat_deg/180*np.pi
# NUMBER OF GRID POINTS IN EACH DIMENSION
self.nz = nz
self.nzs = nzs
self.nx = nx
self.nxs = nxs
self.ny = ny
self.nys = nys
self.nb = nb
# INDEX ARRAYS
self.i = np.arange((self.nb),(self.nx +self.nb))
self.i_s = np.arange((self.nb),(self.nxs+self.nb))
self.j = np.arange((self.nb),(self.ny +self.nb))
self.js = np.arange((self.nb),(self.nys+self.nb))
self.k = np.arange(self.nz)
self.ii, self.jj = np.ix_(self.i ,self.j)
self.iis, self.jjs = np.ix_(self.i_s ,self.js)
# 2D MATRIX OF LONGITUDES AND LATITUDES IN DEGREES
self.lon_deg = np.full(
(self.nx +2*self.nb,self.ny+2*self.nb,1),
np.nan, dtype=wp)
self.lat_deg = np.full(
(self.nx +2*self.nb,self.ny+2*self.nb,1),
np.nan, dtype=wp)
self.lon_is_deg = np.full(
(self.nxs+2*self.nb,self.ny+2*self.nb,1),
np.nan, dtype=wp)
self.lat_is_deg = np.full(
(self.nxs+2*self.nb,self.ny+2*self.nb,1),
np.nan, dtype=wp)
self.lon_js_deg = np.full(
(self.nx+2*self.nb,self.nys+2*self.nb,1),
np.nan, dtype=wp)
self.lat_js_deg = np.full(
(self.nx+2*self.nb,self.nys+2*self.nb,1),
np.nan, dtype=wp)
self.dlon_rad = np.full(
(self.nx +2*self.nb,self.nys+2*self.nb,1),
self.dlon_rad_1D, dtype=wp)
self.dlat_rad = np.full(
(self.nxs+2*self.nb,self.ny +2*self.nb,1),
self.dlat_rad_1D, dtype=wp)
for j in range(self.nb, self.ny+self.nb):
self.lon_deg[self.ii,j,0] = (self.lon0_deg +
(self.ii-self.nb+0.5)*self.dlon_deg)
self.lon_is_deg[self.iis,j,0] = (self.lon0_deg +
(self.iis-self.nb)*self.dlon_deg)
for j_s in range(self.nb, self.nys+self.nb):
self.lon_js_deg[self.ii,j_s,0] = (self.lon0_deg +
(self.ii-self.nb+0.5)*self.dlon_deg)
for i in range(self.nb, self.nx+self.nb):
self.lat_deg[i,self.jj,0] = (self.lat0_deg +
(self.jj-self.nb+0.5)*self.dlat_deg)
self.lat_js_deg[i,self.jjs,0] = (self.lat0_deg +
(self.jjs-self.nb)*self.dlat_deg)
for i_s in range(self.nb, self.nxs+self.nb):
self.lat_is_deg[i_s,self.jj,0] = (self.lat0_deg +
(self.jj-self.nb+0.5)*self.dlat_deg)
# 2D MATRIX OF LONGITUDES AND LATITUDES IN RADIANS
self.lon_rad = self.lon_deg/180*np.pi
self.lat_rad = self.lat_deg/180*np.pi
self.lon_is_rad = self.lon_is_deg/180*np.pi
self.lat_is_rad = self.lat_is_deg/180*np.pi
self.lon_js_rad = self.lon_js_deg/180*np.pi
self.lat_js_rad = self.lat_js_deg/180*np.pi
# 2D MATRIX OF GRID SPACING IN METERS
self.dx = np.full(
(self.nx +2*self.nb,self.ny +2*self.nb,1),
np.nan, dtype=wp)
self.dxjs = np.full(
(self.nx +2*self.nb,self.nys+2*self.nb,1),
np.nan, dtype=wp)
self.dyis = np.full(
(self.nxs+2*self.nb,self.ny +2*self.nb,1),
np.nan, dtype=wp)
self.dx[self.ii,self.jj,0] = (
np.cos( self.lat_rad[self.ii,self.jj,0] ) *
self.dlon_rad_1D*con_rE )
self.dxjs[self.ii,self.jjs,0] = (
np.cos( self.lat_js_rad[self.ii,self.jjs,0] ) *
self.dlon_rad_1D*con_rE )
self.dyis[self.iis,self.jj,0] = self.dlat_rad_1D*con_rE
self.dx = self.exchange_BC(self.dx)
self.dxjs = self.exchange_BC(self.dxjs)
self.dyis = self.exchange_BC(self.dyis)
self.dy = self.dlat_rad*con_rE
self.A = np.full( (self.nx+2*self.nb,self.ny+2*self.nb, 1),
np.nan, dtype=wp)
for i in self.i:
for j in self.j:
self.A[i,j,0] = lat_lon_recangle_area(self.lat_rad[i,j,0],
self.dlon_rad_1D, self.dlat_rad_1D)
self.A = self.exchange_BC(self.A)
print('fraction of earth covered: ' +
str(np.round(np.sum(
self.A[self.ii,self.jj,0])/(4*np.pi*con_rE**2),2)))
# CORIOLIS FORCE
self.corf = np.full(
(self.nx +2*self.nb, self.ny +2*self.nb,1),
np.nan, dtype=wp)
self.corf_is = np.full(
(self.nxs+2*self.nb, self.ny +2*self.nb,1),
np.nan, dtype=wp)
self.corf[self.ii,self.jj,0] = 2*con_omega*np.sin(
self.lat_rad[self.ii,self.jj,0])
self.corf_is[self.iis,self.jj,0] = 2*con_omega*np.sin(
self.lat_is_rad[self.iis,self.jj,0])
# SIGMA LEVELS
self.level = np.arange(0,self.nz )
self.levels = np.arange(0,self.nzs)
# will be set in load_profile of IO
self.sigma_vb = np.full( self.nzs, np.nan, dtype=wp)
self.dsigma = np.full( self.nz , np.nan, dtype=wp)
set_up_sigma_levels(self)
self.dsigma = np.expand_dims(
np.expand_dims(self.dsigma , 0),0)
self.sigma_vb = np.expand_dims(
np.expand_dims(self.sigma_vb , 0),0)
# TIME STEP
mindx = np.nanmin(self.dx)
self.CFL = CFL
self.i_out_nth_hour = i_out_nth_hour
self.nc_output_count = 0
self.i_sim_n_days = i_sim_n_days
self.dt = int(self.CFL*mindx/400)
while i_out_nth_hour*3600 % self.dt > 0:
self.dt -= 1
self.nts = i_sim_n_days*3600*24/self.dt
self.ts = 0
self.i_out_nth_ts = int(self.i_out_nth_hour*3600 / self.dt)
self.i_restart_nth_day = i_restart_nth_day
self.i_restart_nth_ts = int(self.i_restart_nth_day*24/ \
self.i_out_nth_hour*self.i_out_nth_ts)
self.sim_time_sec = 0
self.GMT = GMT_initialization
# TIMER
self.timer = Timer()
# NUMERICAL DIFUSION
self.UVFLX_dif_coef = np.zeros((1,1,nz), dtype=wp)
self.POTT_dif_coef = np.zeros((1,1,nz), dtype=wp)
self.moist_dif_coef = np.zeros((1,1,nz), dtype=wp)
# vert_reduce for 32 vertical levels:
# 1.0 : uppermost level has 40% of lowest
# 1.5 : uppermost level has 23% of lowest
# 2.0 : uppermost level has 10% of lowest
# 3.0 : uppermost level has 5% of lowest
vert_reduce = .0
self.UVFLX_dif_coef[0,0,self.k] = (UVFLX_dif_coef *
np.exp(-vert_reduce*(nz-self.k-1)/nz))
vert_reduce = 1.5
self.POTT_dif_coef[0,0,self.k] = (POTT_dif_coef *
np.exp(-vert_reduce*(nz-self.k-1)/nz))
self.moist_dif_coef[0,0,self.k] = (moist_dif_coef *
np.exp(-vert_reduce*(nz-self.k-1)/nz))
self.copy_to_gpu()
def load_factor(self,
file_name,
file_dir=None,
factor_names=None,
dates=None,
ids=None,
idx=None,
df=True,
h5_style=True):
"""" 读取单因子数据
paramters:
==========
idx: DataFrame
返回data.align(idx, how='right')[0]
dates: list or array-like
dates终将把每个元素转换成int格式
ids: list or array-like
ids终将把每个元素转换成string格式
df: bool
是否已DataFrame格式返回. True by default.
"""
file_shape = self.list_file_shape(file_name, file_dir)
all_factors = self.list_file_factors(file_name, file_dir)
all_ids = self.list_file_ids(file_name, file_dir)
all_dates = self.list_file_dates(file_name, file_dir)
if factor_names is None:
factor_idx = (0, file_shape[2])
factor_names = all_factors
else:
temp_factors = np.sort(factor_names)
factor_idx = (all_factors.index(temp_factors[0]),
all_factors.index(temp_factors[-1]))
if dates is None:
date_idx = (0, file_shape[0])
dates = all_dates
else:
dates = np.sort(
Datetime2MatlabDatetime(pd.DatetimeIndex(dates).values))
date_idx = (np.searchsorted(all_dates, dates[0]),
np.searchsorted(all_dates, dates[-1], side='right'))
if ids is None:
ids_idx = (0, file_shape[1])
ids = all_ids
else:
ids = np.sort(ids).astype('int')
ids_idx = (np.searchsorted(all_ids, ids[0]),
np.searchsorted(all_ids, ids[-1], side='right'))
factor_data = self._read_raw(self.abs_factor_path(file_dir, file_name),
date_idx, ids_idx, factor_idx)
date_idx2 = np.in1d(all_dates[date_idx[0]:date_idx[-1]], dates)
ids_idx2 = np.in1d(all_ids[ids_idx[0]:ids_idx[-1]], ids)
factor_idx2 = np.in1d(all_factors[factor_idx[0]:factor_idx[-1]],
factor_names)
factor_data = factor_data[np.ix_(date_idx2, ids_idx2, factor_idx2)]
if df:
if h5_style:
ids_str_func = np.frompyfunc(intcode_to_tradecode, 1, 1)
ids_str = ids_str_func(ids)
datetimes = MatlabDatetime2Datetime(dates)
df = self.arr3d2df(factor_data, datetimes, ids_str,
factor_names)
else:
df = self.arr3d2df(factor_data, dates, ids, factor_names)
if idx is not None:
df = df.reindex(idx.index)
return df
return factor_data
def uspatial2spin(cc, moidx, mo_coeff):
'''Convert the results of an unrestricted mean-field calculation to spin-orbital form.
Spin-orbital ordering is determined by orbital energy without regard for spin.
Returns:
fock : (nso,nso) ndarray
The Fock matrix in the basis of spin-orbitals
so_coeff : (nao, nso) ndarray
The matrix of spin-orbital coefficients in the AO basis
spin : (nso,) ndarary
The spin (0 or 1) of each spin-orbital
'''
dm = cc._scf.make_rdm1(cc.mo_coeff, cc.mo_occ)
fockao = cc._scf.get_hcore() + cc._scf.get_veff(cc.mol, dm)
fockab = list()
for a in range(2):
fockab.append(
reduce(numpy.dot, (mo_coeff[a].T, fockao[a], mo_coeff[a])))
nocc = cc.nocc
nao = cc.mo_coeff[0].shape[0]
nmo = cc.nmo
nvir = nmo - nocc
so_coeff = np.zeros((nao, nmo), dtype=mo_coeff[0].dtype)
nocc_a = int(sum(cc.mo_occ[0] * moidx[0]))
nocc_b = int(sum(cc.mo_occ[1] * moidx[1]))
nmo_a = fockab[0].shape[0]
nmo_b = fockab[1].shape[0]
nvir_a = nmo_a - nocc_a
nvir_b = nmo_b - nocc_b
oa = range(0, nocc_a)
ob = range(nocc_a, nocc)
va = range(nocc, nocc + nvir_a)
vb = range(nocc + nvir_a, nmo)
spin = np.zeros(nmo, dtype=int)
spin[oa] = 0
spin[ob] = 1
spin[va] = 0
spin[vb] = 1
so_coeff[:, oa] = mo_coeff[0][:, :nocc_a]
so_coeff[:, ob] = mo_coeff[1][:, :nocc_b]
so_coeff[:, va] = mo_coeff[0][:, nocc_a:nmo_a]
so_coeff[:, vb] = mo_coeff[1][:, nocc_b:nmo_b]
fock = np.zeros((nmo, nmo), dtype=fockab[0].dtype)
fock[np.ix_(oa, oa)] = fockab[0][:nocc_a, :nocc_a]
fock[np.ix_(oa, va)] = fockab[0][:nocc_a, nocc_a:]
fock[np.ix_(va, oa)] = fockab[0][nocc_a:, :nocc_a]
fock[np.ix_(va, va)] = fockab[0][nocc_a:, nocc_a:]
fock[np.ix_(ob, ob)] = fockab[1][:nocc_b, :nocc_b]
fock[np.ix_(ob, vb)] = fockab[1][:nocc_b, nocc_b:]
fock[np.ix_(vb, ob)] = fockab[1][nocc_b:, :nocc_b]
fock[np.ix_(vb, vb)] = fockab[1][nocc_b:, nocc_b:]
# Do not sort because it's different to the orbital ordering generated by
# get_frozen_mask function in AO-direct vvvv contraction
# idxo = np.diagonal(fock[:nocc,:nocc]).argsort()
# idxv = nocc + np.diagonal(fock[nocc:,nocc:]).argsort()
# idx = np.concatenate((idxo,idxv))
# spin = spin[idx]
# so_coeff = so_coeff[:,idx]
# fock = fock[:, idx][idx]
return fock, so_coeff, spin
