Пример #1
0
    def reduce_system(self, states, inputs, outputs):
        """Returns reduced state space matrices.

        Parameters
        ----------
        states : list
            A list of the state names of the reduced system.
        inputs : list
            A list of the input names of the reduced system.
        outputs : list
            A list of the output names of the reduced system.

        Returns
        -------
        A : ndarray
            State matrix.
        B : ndarray
            Input matrix.
        C : ndarray
            Output matrix.
        D : ndarray
            Feed forward matrix.

        """
        stateIndices = sorted([self.stateNames.index(s) for s in states])
        inputIndices = sorted([self.inputNames.index(s) for s in inputs])
        outputIndices = sorted([self.outputNames.index(s) for s in outputs])

        A = self.A[np.ix_(stateIndices, stateIndices)]
        B = self.B[np.ix_(stateIndices, inputIndices)]
        C = self.C[np.ix_(outputIndices, stateIndices)]
        D = self.D[np.ix_(outputIndices, inputIndices)]

        return A, B, C, D
Пример #2
0
    def impute(self, states):
        S = self.S

        if states.ndim == 1:
            states = states[np.newaxis, :]

        N = states.shape[0]
        mask = self.state.slice_array(states, 'mask').astype(bool)
        feature_mask = self.state.get_feature_mask(mask)

        # first, mean impute
        states_imputed = states.copy()
        Xm = self.state.slice_array(states_imputed, 'observations')
        means = np.tile(self.mean, (N, 1))
        Xm[feature_mask] = means[feature_mask]

        # now condition on observed values, if any
        for i in xrange(N):
            obs_ind = ~feature_mask[i, :]

            # if no features observed or all features observed, don't act
            if obs_ind.sum() == 0 or (~obs_ind).sum() == 0:
                continue

            # otherwise, do the dirty
            A = S[np.ix_(obs_ind, obs_ind)]
            C_T = S[np.ix_(~obs_ind, obs_ind)]
            ctainv = np.dot(C_T, np.linalg.pinv(A))
            mean = np.dot(ctainv, Xm[i, obs_ind])
            Xm[i, ~obs_ind] = mean

        if N == 1:
            return states_imputed.flatten()
        return states_imputed
Пример #3
0
def filter_otu_table_by_min(sample_names, taxon_names, data, lineages, min=1):
    # loop through and remove every otu present at less than min
    # this should be replaced by native QIIME filter code.
    s_vec = []
    s_names = []
    taxonomies = []

    # create list of OTUs to keep
    for otu in range(data.shape[0]):
        if data[otu, :].sum() >= min:
            s_vec.append(otu)
            s_names.append(taxon_names[otu])
            if lineages:
                taxonomies.append(lineages[otu])

    h_vec = []
    h_names = []

    for sample in range(data.shape[1]):
        if data[numpy.ix_(s_vec), sample].sum() >= 1:
            h_vec.append(sample)
            h_names.append(sample_names[sample])

    # slice data
    data = data[numpy.ix_(s_vec, h_vec)]

    return h_names, s_names, data, taxonomies
Пример #4
0
def Barabasi_Albert(m0, m, N):
#   if m > m0:
#       raise ValueError('m must be smaller than or equal to m0')
#   # initial graph
#   Graph = [Node() for _ in range(m0)]
#   for (ix, node) in enumerate(Graph):
#       node.connect(Graph[ix + 1:])
#   degrees = np.array([node.degree for node in Graph])
#   cum_degrees = np.float(np.cumsum(degrees)) / np.sum(degrees)
    K = np.eye(N, dtype=np.bool)
    K[np.ix_(np.arange(m0), np.arange(m0))] = True
    for ix in np.arange(m0, N):
        selected = np.zeros((ix,), dtype=np.bool)
        for conn in np.arange(m):
            free = np.logical_not(selected)
            p = np.array(np.sum(K[..., free], axis=0), dtype=np.float)
            cdf = np.cumsum(p) / np.sum(p)
            r = np.random.uniform()
            link = np.where(np.logical_and(r < cdf,
                                           np.logical_not(r >= cdf)))
            K[ix, free[link]] = True
            K[free[link], ix] = True
            selected[free[link]] = True
    rp = np.random.permutation(N)
    return K[np.ix_(rp, rp)]
Пример #5
0
    def subset(self, variables=None, samples=None):
        """Returns a subset of the dataset (and metadata).
        
        Specify the variables and samples for creating a subset of the data.
        variables and samples should be a list of ids. If not specified, it is
        assumed to be all variables or samples. 

        Some examples:
        
            - d.subset([3], [4])
            - d.subset([3,1,2])
            - d.subset(samples=[5,2,7,1])
        
        Note: order matters! d.subset([3,1,2]) != d.subset([1,2,3])

        """

        variables = variables if variables is not None else range(self.variables.size)
        samples = samples if samples is not None else range(self.samples.size)
        skip_stats = not (self.has_interventions or self.has_missing)
        d = Dataset(
            self.observations[N.ix_(samples,variables)],
            self.missing[N.ix_(samples,variables)],
            self.interventions[N.ix_(samples,variables)],
            self.variables[variables],
            self.samples[samples],
            skip_stats = skip_stats
        )
        
        # if self does not have interventions or missing, the subset can't.
        if skip_stats:
            d._has_interventions = False
            d._has_missing = False

        return d
Пример #6
0
def flume_hm_res(X,Y):

    hopperlen = 4.7
    hmax = 1.9
    hoppertop = 3.3
    topangle = 17.0*np.pi/180.0
    flumeangle = 31.0*np.pi/180.0
    m0 = 0.61

    x0 = -hopperlen
    x2 = -hmax*np.cos(0.5*np.pi - flumeangle)
    x1 = x2 - hoppertop*np.cos(flumeangle-topangle)

    x3 = 0.0
    y2 = hmax*np.sin(0.5*np.pi - flumeangle)
    y1 = y2 - hoppertop*np.sin(flumeangle-topangle)
    xm1 = x1 - y1*np.tan(0.5*np.pi - flumeangle)

    slopem1 = y1/(x1-x0)
    slope0 = y1/(x1-x0)
    slope1 = (y2-y1)/(x2-x1)
    slope2 = -y2/(x3-x2)

    yind =  np.where((Y[:,0]<=2.0)&(Y[:,0]>=0.0))[0]
    xm1ind = np.where((X[0,:]>=xm1)&(X[0,:]<x0))[0]
    x0ind = np.where((X[0,:]>=x0)&(X[0,:]<x1))[0]
    x1ind = np.where((X[0,:]>=x1)&(X[0,:]<=x3))[0]

    Z=np.zeros(np.shape(X))
    #Z[np.ix_(yind,x1ind)] = m0*flume_eta(X[np.ix_(yind,x1ind)],Y[np.ix_(yind,x1ind)])
    Z[np.ix_(yind,x1ind)] = m0
    Z[np.ix_(yind,x0ind)] = m0*flume_eta(X[np.ix_(yind,x0ind)],Y[np.ix_(yind,x0ind)])/flume_eta_res(X[np.ix_(yind,x0ind)],Y[np.ix_(yind,x0ind)])


    return Z
Пример #7
0
def test_chol_add_remove():
    N = 5
    X = np.random.randn(10,N)
    A = X.T.dot(X)
    L = np.linalg.cholesky(A)

    Am = A[:-1,:-1]
    bm = A[:-1,-1]
    cm = A[-1,-1]
    Lm = np.linalg.cholesky(Am)

    # Get chol by adding row
    assert np.allclose(L, chol_add_row(Lm, bm, cm))

    # Now get chol by removing a row
    def to_range(start, stop):
        return np.setdiff1d(np.arange(N), np.arange(start,stop))
    assert np.allclose(
        np.linalg.cholesky(A[np.ix_(to_range(4,5),
                                    to_range(4,5))]),
                           chol_remove_row(L,4,5))

    assert np.allclose(
        np.linalg.cholesky(A[np.ix_(to_range(1,3),
                                    to_range(1,3))]),
                           chol_remove_row(L,1,3))
Пример #8
0
    def get_corr_pred( self, sctx, u, tn, tn1 ):
        ndofs = self.domain.n_dofs
        #self.K.data[::] = 0.0
        self.K = sparse.lil_matrix((ndofs, ndofs), float_ )
        self.F_int[:]    = 0.0
        e_arr_size      = self.e_arr_size

        for elem in sctx.sdomain.elements:
            e_id = elem.id_number
            ix = elem.get_dof_map()
            sctx.elem = elem
            sctx.elem_state_array = sctx.state_array[ e_id*e_arr_size : (e_id+1)*e_arr_size ]
            sctx.X = elem.get_X_mtx()
            f, k = self.fets_eval.get_corr_pred( sctx, u[ix_(ix)], tn, tn1 )
            #self.K_temp.data[:][:] = 0.
            self.K_temp = sparse.lil_matrix((ndofs, ndofs), float_ )
            a = 0
            for i in ix:
                self.K_temp.rows[i] = ix
                self.K_temp.data[i][:] = k[a][:]
                a =+1
                #print K_temp
            self.K = self.K + self.K_temp
            self.F_int[ ix_(ix) ] += f


        return self.F_int, self.K
Пример #9
0
def _split(estimator, X, y, indices, train_indices=None):
    """Create subset of dataset."""
    if hasattr(estimator, 'kernel') and callable(estimator.kernel):
        # cannot compute the kernel values with custom function
        raise ValueError("Cannot use a custom kernel function. "
                         "Precompute the kernel matrix instead.")

    if not hasattr(X, "shape"):
        if getattr(estimator, "_pairwise", False):
            raise ValueError("Precomputed kernels or affinity matrices have "
                             "to be passed as arrays or sparse matrices.")
        X_subset = [X[idx] for idx in indices]
    else:
        if getattr(estimator, "_pairwise", False):
            # X is a precomputed square kernel matrix
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square kernel matrix")
            if train_indices is None:
                X_subset = X[np.ix_(indices, indices)]
            else:
                X_subset = X[np.ix_(indices, train_indices)]
        else:
            X_subset = X[safe_mask(X, indices)]

    if y is not None:
        y_subset = y[safe_mask(y, indices)]
    else:
        y_subset = None

    return X_subset, y_subset
Пример #10
0
    def __init__(self, skim_dict, orig_zones, dest_zones, transpose=False):

        omx_shape = skim_dict.skim_info['omx_shape']
        logger.info("init AccessibilitySkims with %d dest zones %d orig zones omx_shape %s" %
                    (len(dest_zones), len(orig_zones), omx_shape, ))

        assert len(orig_zones) <= len(dest_zones)
        assert np.isin(orig_zones, dest_zones).all()
        assert len(np.unique(orig_zones)) == len(orig_zones)
        assert len(np.unique(dest_zones)) == len(dest_zones)

        self.skim_dict = skim_dict
        self.transpose = transpose

        if omx_shape[0] == len(orig_zones):
            # no slicing required
            self.slice_map = None
        else:
            # 2-d boolean slicing in numpy is a bit tricky
            # data = data[orig_map, dest_map]          # <- WRONG!
            # data = data[orig_map, :][:, dest_map]    # <- RIGHT
            # data = data[np.ix_(orig_map, dest_map)]  # <- ALSO RIGHT

            skim_index = list(range(omx_shape[0]))
            orig_map = np.isin(skim_index, skim_dict.offset_mapper.map(orig_zones))
            dest_map = np.isin(skim_index, skim_dict.offset_mapper.map(dest_zones))

            if not dest_map.all():
                # not using the whole skim matrix
                logger.info("%s skim zones not in dest_map: %s" %
                            ((~dest_map).sum(), np.ix_(~dest_map)))

            self.slice_map = np.ix_(orig_map, dest_map)
Пример #11
0
def _safe_split(estimator, X, y, indices, train_indices=None):
    """Create subset of dataset and properly handle kernels."""
    from ..gaussian_process.kernels import Kernel as GPKernel

    if (hasattr(estimator, 'kernel') and callable(estimator.kernel) and
            not isinstance(estimator.kernel, GPKernel)):
        # cannot compute the kernel values with custom function
        raise ValueError("Cannot use a custom kernel function. "
                         "Precompute the kernel matrix instead.")

    if not hasattr(X, "shape"):
        if getattr(estimator, "_pairwise", False):
            raise ValueError("Precomputed kernels or affinity matrices have "
                             "to be passed as arrays or sparse matrices.")
        X_subset = [X[index] for index in indices]
    else:
        if getattr(estimator, "_pairwise", False):
            # X is a precomputed square kernel matrix
            if X.shape[0] != X.shape[1]:
                raise ValueError("X should be a square kernel matrix")
            if train_indices is None:
                X_subset = X[np.ix_(indices, indices)]
            else:
                X_subset = X[np.ix_(indices, train_indices)]
        else:
            X_subset = safe_indexing(X, indices)

    if y is not None:
        y_subset = safe_indexing(y, indices)
    else:
        y_subset = None

    return X_subset, y_subset
Пример #12
0
def patch(data, rows, cols = None):
	"""
	data = data matrix, 1D or 2D array (matrix) 
	rows = iterator of rows (list) to select, None means selecting all rows
	cols = iterator of cols (list) to select, None means selecting all cols 
	return np.array (of the patch shape), but the DIM of return should be 
	the same as data (1D or 2D)
	if data is a sparse matrix, the return the matrix will be dense np.array
	"""
	if not sparse.issparse(data):
		data = np.asarray(data)
	dim = get_dim(data)
	if dim == 1:
		## ignore cols
		return data[rows] if rows is not None else data
	elif dim == 2:
		nrows, ncols = data.shape
		rows = rows if rows is not None else xrange(nrows)
		cols = cols if cols is not None else  xrange(ncols)
		if sparse.issparse(data):
			return data.toarray()[np.ix_(rows, cols)]
		else:
			return data[np.ix_(rows, cols)]
	else:
		raise RuntimeError('only supports 1D or 2D array') 
def penalty_function(vocab_indices, summary_indices, sentence_similarity, config):
    """
        This is the penalty function that is described in the paper
        Graph-Based Submodular selection for extractive Summarization
    Args:
        vocab_indices: list
        summary_indices: list
        sentence_similarity: ndarray
        config: dictionary
        Some of the methods require some hyper parameters
        to be set

        This penalises redundancy
    Returns: The value of the graph cut function
    """
    penalty_lambda = config["penalty_lambda"]
    sentence_similartiy_ = np.copy(sentence_similarity)
    np.fill_diagonal(sentence_similartiy_, 0.0)

    if len(summary_indices) == 0:
        fn_value = 0.0
    else:
        v_not_in_s = list(set(vocab_indices) - set(summary_indices))
        rows = v_not_in_s
        cols = summary_indices
        # USING THE ADVANCED INDEXING OF THE NUMPY ARRAY
        fn_value = np.sum(sentence_similarity[np.ix_(rows, cols)]) - \
                   penalty_lambda * np.sum(sentence_similartiy_[np.ix_(summary_indices, summary_indices)])

    return fn_value
Пример #14
0
def nd_bootstrap(data, iterations, axis=None, strip_tuple_if_one=True):
    """
    Bootstrap iterator for several n-dimensional data arrays.

    :param data: Iterable containing the data arrays
    :param iterations: Number of bootstrap iterations.
    :param axis: Bootstrapping is performed along this axis.
    """
    shape0 = data[0].shape
    if axis is None:
        axis = 0
        data = [d.ravel() for d in data]

    n = len(data[0].shape)
    K = len(data)
    data0 = []

    if axis is not None:
        m = data[0].shape[axis]
        to = tuple([axis]) + tuple(range(axis)) + tuple(range(axis + 1, n))
        fro = tuple(range(1, axis + 1)) + (0,) + tuple(range(axis + 1, n))
        for i in range(K):
            data0.append(data[i].transpose(to))

        for i in range(iterations):
            idx = np.random.randint(m, size=(m,))
            if len(data) == 1 and strip_tuple_if_one:
                yield (data0[0][np.ix_(idx), ...].squeeze().
                       transpose(fro).reshape(shape0))
            else:
                yield tuple(a[np.ix_(idx), ...].squeeze().
                            transpose(fro).reshape(shape0) for a in data0)
Пример #15
0
    def runTest(self):
        F=lambda x,y: 100.0*((x>=0.4)&(x<=0.6)&(y>=0.4)&(y<=0.6))
        G=lambda x,y: (y==0)*1.0+(y==1)*(-1.0)

        a=fasm.AssemblerElement(self.mesh,felem.ElementTriP1())

        dudv=lambda du,dv: du[0]*dv[0]+du[1]*dv[1]
        K=a.iasm(dudv)

        uv=lambda u,v: u*v
        B=a.fasm(uv)
        
        fv=lambda v,x: F(x[0],x[1])*v
        f=a.iasm(fv)

        gv=lambda v,x: G(x[0],x[1])*v
        g=a.fasm(gv)

        D=np.nonzero(self.mesh.p[0,:]==0)[0]
        I=np.setdiff1d(np.arange(0,self.mesh.p.shape[1]),D)

        x=np.zeros(K.shape[0])
        x[I]=scipy.sparse.linalg.spsolve(K[np.ix_(I,I)]+B[np.ix_(I,I)],
                                         f[I]+g[I])

        self.assertAlmostEqual(np.max(x),1.89635971369,places=2)
Пример #16
0
def mask_restore(b, passmask, fill_value=0, axis=(0,1)):
    """
    Restore the rows and/or columns filtered out using passmask to produce b.

    """
    passmask = np.asarray(passmask, dtype=bool)
    n = len(passmask)
    fill_if = ~passmask

    if b.ndim == 1:
        a = np.zeros(n, dtype=b.dtype)
        a[passmask] = b
        if fill_value: a[fill_if] = fill_value
        return a

    elif b.ndim == 2:
        if axis == (0,1):
            a = np.zeros((n, n), dtype=b.dtype)
            a[np.ix_(passmask, passmask)] = b
            if fill_value: a[np.ix_(fill_if, fill_if)] = fill_value
            return a
        elif axis == 0:
            a = np.zeros((n, b.shape[1]), dtype=b.dtype)
            a[passmask, :] = b
            if fill_value: a[fill_if, :] = fill_value
            return a
        elif axis == 1:
            a = np.zeros((b.shape[0], n), dtype=b.dtype)
            a[:, passmask] = b
            if fill_value: a[:, fill_if] = fill_value
            return a
        else:
            raise ValueError("'axis' must be 0, 1 or (0,1)")

    raise TypeError("Input array must be rank 1 or 2")
Пример #17
0
    def get_corr_pred( self, sctx, u, du, tn, tn1 ):

        n_ip_arr, ip_coords_arr, ip_weights_arr = self.ip_scheme

        self.F_int[:] = 0.0
        self.k_arr[...] = 0.0

        B_mtx_grid = None
        J_det_grid = None

        ip_offset = 0
        k_list = []
        for e_id, ( elem, n_ip ) in enumerate( zip( self.sdomain.elements, n_ip_arr ) ):
            ip_coords = ip_coords_arr[ ip_offset : ip_offset + n_ip ]
            ip_weights = ip_weights_arr[ ip_offset : ip_offset + n_ip ]
            ix = elem.get_dof_map()
            sctx.elem = elem
            sctx.elem_state_array = self.state_array[ ip_offset : ip_offset + n_ip ].flatten()
            sctx.X = elem.get_X_mtx()
            if self.cache_geo_matrices:
                B_mtx_grid = self.B_mtx_grid[ e_id, ... ]
                J_det_grid = self.J_det_grid[ e_id, ... ]
            f, k = self.fets_eval.get_corr_pred( sctx, u[ix_( ix )], du[ix_( ix )],
                                                 tn, tn1,
                                                 B_mtx_grid = B_mtx_grid,
                                                 J_det_grid = J_det_grid,
                                                 ip_coords = ip_coords,
                                                 ip_weights = ip_weights )

            self.k_arr[ e_id ] = k
            self.F_int[ ix_( ix ) ] += f
            ip_offset += n_ip

        return self.F_int, SysMtxArray( mtx_arr = self.k_arr, dof_map_arr = self.sdomain.elem_dof_map )
Пример #18
0
def flume_theta(X,Y):

    """
    angle theta in flume
    """
    deg2rad = np.pi/180.0
    flumelen = 78.0
    flumerad = 10.0
    theta1 = 31.0
    theta2 = 2.5

    D2 = flumelen + flumerad*(theta1 - theta2)*deg2rad


    yind =  np.where((Y[:,0]<=20.0)&(Y[:,0]>=-20.0))[0]
    x1ind = np.where(X[0,:]<=flumelen)[0]
    x2ind = np.where((X[0,:]>flumelen)&(X[0,:]<D2))[0]
    x3ind = np.where(X[0,:]>=D2)[0]

    Z=np.zeros(np.shape(X))
    Z[np.ix_(yind,x1ind)] = theta1
    Z[np.ix_(yind,x3ind)] = theta2
    Z[np.ix_(yind,x2ind)] = theta1 - (X[np.ix_(yind,x2ind)]-flumelen)/(deg2rad*flumerad)
    Z = deg2rad*Z

    return Z
Пример #19
0
def flume_eta(X,Y):

    hopperlen = 4.7
    hmax = 1.9
    hoppertop = 3.3
    topangle = 17.0*np.pi/180.0
    flumeangle = 31.0*np.pi/180.0
    backangle = (90.0-flumeangle)*np.pi/180.0

    x0 = 0.0
    x1 = (x0-hopperlen)*np.cos(flumeangle)
    yind =  np.where((Y[:,0]<=2.0)&(Y[:,0]>=0.0))[0]
    x1ind = np.where(X[0,:]>=x1)[0]
    x2ind = np.where(X[0,:]<x1)[0]

    Z=np.zeros(np.shape(X))
    elev = bed_curve(x0*np.ones(np.shape(X)),0.0*np.ones(np.shape(Y)))
    Z[np.ix_(yind,x1ind)] = hmax + (x0-X[np.ix_(yind,x1ind)])*np.sin(topangle) + elev[0,0]

    elev = bed_curve(x0*np.ones(np.shape(X)),0.0*np.ones(np.shape(Y)))
    etax1 = hmax + (x0-x1)*np.sin(topangle) + elev[0,0]

    Z[np.ix_(yind,x2ind)] = etax1 - (x1-X[np.ix_(yind,x2ind)])*np.sin(backangle)

    return Z
Пример #20
0
def _steadystate_lu(L, use_rcm=True, use_umfpack=False):
    """
    Find the steady state(s) of an open quantum system by computing the
    LU decomposition of the underlying matrix.
    """
    if settings.debug:
        print('Starting LU solver...')
    dims=L.dims[0]
    weight=np.abs(L.data.max())
    n = prod(L.dims[0][0])
    b = np.zeros(n ** 2, dtype=complex)
    b[0] = weight
    L = L.data.tocsc() + sp.csc_matrix((weight*np.ones(n),
                    (np.zeros(n), [nn * (n + 1) for nn in range(n)])),
        shape=(n ** 2, n ** 2))
    
    L.sort_indices()
    use_solver(assumeSortedIndices=True, useUmfpack=use_umfpack)
    if use_rcm:
        perm = symrcm(L)
        L = sparse_permute(L,perm,perm)
        b = b[np.ix_(perm,)]
    
    solve = factorized(L)
    v = solve(b)
    if use_rcm:
        rev_perm = np.argsort(perm)
        v = v[np.ix_(rev_perm,)]
    data = vec2mat(v)
    data = 0.5 * (data + data.conj().T)

    return Qobj(data, dims=dims, isherm=True)
Пример #21
0
def get_aligned_new_and_old_datasets():

    '''
    Returns the old and new matrices only where they overlap
    (both rows and columns), so that the data comparison tests don't error out.
    This should only error out if there is no overlap whatsoever.
    '''
    
    # Align the rows
    new_row_ids = [x for x in res_count_dataset[0]]
    old_row_ids = [x for x in orig_count_dataset[0]]
    common_rows = list(set(new_row_ids).intersection(set(old_row_ids)))

    new_matrix_row_inds = [new_row_ids.index(x) for x in common_rows]
    old_matrix_row_inds = [old_row_ids.index(x) for x in common_rows]

    # Align the columns
    new_col_tuples = [tuple(x) for x in res_count_dataset[1]]
    old_col_tuples = [tuple(x) for x in orig_count_dataset[1]]
    common_cols = list(set(new_col_tuples).intersection(set(old_col_tuples)))

    new_matrix_col_inds = [new_col_tuples.index(x) for x in common_cols]
    old_matrix_col_inds = [old_col_tuples.index(x) for x in common_cols]

    # Get the new aligned matrices
    aligned_old_mat = orig_count_dataset[2][np.ix_(old_matrix_row_inds, old_matrix_col_inds)]
    aligned_new_mat = res_count_dataset[2][np.ix_(new_matrix_row_inds, new_matrix_col_inds)]

    print "Original matrix shape: ", orig_count_dataset[2].shape
    print "New matrix shape: ", res_count_dataset[2].shape
    print "Aligned matrix shape: ", (len(common_rows), len(common_cols))

    return [common_rows, common_cols, aligned_new_mat], [common_rows, common_cols, aligned_old_mat]
Пример #22
0
def multiClassSVM(distances, trainingIndice, testingIndice, semanticLabels, kernelType):

    distances = distances ** 2
    trainDistance = distances[np.ix_(trainingIndice, trainingIndice)]

    gamma = 1.0 / np.mean(trainDistance)
    kernelParam = []
    kernelParam.append(gamma)


    tempList = []
    tempList.append(kernelType)
    baseKernel = constructBaseKernels(tempList, kernelParam, distances)

    trainGramMatrix = baseKernel[0][np.ix_(trainingIndice, trainingIndice)]
    testGramMatrix = baseKernel[0][np.ix_(testingIndice, trainingIndice)]

    trainLabels = [semanticLabels[i] for i in trainingIndice]
    testLabels = [semanticLabels[i] for i in testingIndice]

    clf = SVC(kernel = "precomputed")
    clf.fit(trainGramMatrix, trainLabels)
    SVMResults = clf.predict(testGramMatrix)

    correct = sum(1.0 * (SVMResults == testLabels))
    accuracy = correct / len(testLabels)

    return accuracy
Пример #23
0
def load_weight_files(weights_files, genes, patients, typeToGeneIndex, typeToPatientIndex, masterGeneToIndex, masterPatientToIndex):
    # Master matrix of all weights
    P = np.zeros((len(genes), len(patients)))
    for i, weights_file in enumerate(weights_files):
        # Load the weights matrix for this cancer type and update the entries appropriately.
        # Note that since genes/patients can be measured in multiple types, we need to map
        # each patient to the "master" index.
        type_P                 = np.load(weights_file)

        ty_genes               = set(typeToGeneIndex[i].keys()) & genes
        ty_gene_indices        = [ typeToGeneIndex[i][g] for g in ty_genes ]
        master_gene_indices    = [ masterGeneToIndex[g] for g in ty_genes ]

        ty_patients            = set(typeToPatientIndex[i].keys()) & patients
        ty_patient_indices     = [ typeToPatientIndex[i][p] for p in ty_patients ]
        master_patient_indices = [ masterPatientToIndex[p] for p in ty_patients ]

        master_mesh            = np.ix_(master_gene_indices, master_patient_indices)
        ty_mesh                = np.ix_(ty_gene_indices, ty_patient_indices)

        if np.any( P[master_mesh] > 0 ):
            raise ValueError("Different weights for same gene-patient pair")
        else:
            P[ master_mesh ] = type_P[ ty_mesh  ]

    # Set any zero entries to the minimum (pseudocount). The only reason for zeros is if
    #  a gene wasn't mutated at all in a particular dataset.
    P[P == 0] = np.min(P[P > 0])

    return dict( (g, P[masterGeneToIndex[g]]) for g in genes )
Пример #24
0
def classify_binomial(x, data, counts, y):
    classes, y = np.unique(y, return_inverse=True)
    max_label = None
    max = None
    for class_label in np.nditer(classes):
        class_examples = data[np.ix_(y == class_label)]
        class_counts = counts[np.ix_(y == class_label)]
        total_class_counts = sum(class_counts)
        alfas = (class_examples.sum(axis=0) + 0.01)/(total_class_counts + 0.01)

        prior = len(class_examples) / len(data)
        membership = getMembershipBinomial(x,alfas, prior, class_counts, total_class_counts)
        if(max_label is None):
            max_label = class_label
            max = membership
        else:
            if(class_label == 0):
                if membership>max:
                    max = membership
                    max_label = class_label
            else:
                if membership>(max+8.5):
                    max = membership
                    max_label = class_label
    return max_label
Пример #25
0
    def conditional(self, in_dims, out_dims):
        conditionals = []

        for k, (weight_k, mean_k, covar_k) in enumerate(self):
            conditionals.append(conditional(mean_k, covar_k,
                                            in_dims, out_dims,
                                            self.covariance_type))

        cond_weights = lambda v: [(weight_k * Gaussian(mean_k[in_dims].reshape(-1,),
                                  covar_k[ix_(in_dims, in_dims)]).normal(v.reshape(-1,)))
                                  for k, (weight_k, mean_k, covar_k) in enumerate(self)]

        def res(v):
            gmm = GMM(n_components=self.n_components,
                      covariance_type=self.covariance_type,
                      random_state=self.random_state, thresh=self.thresh,
                      min_covar=self.min_covar, n_iter=self.n_iter, n_init=self.n_init,
                      params=self.params, init_params=self.init_params)
            gmm.weights_ = cond_weights(v)
            means_covars = [f(v) for f in conditionals]
            gmm.means_ = array([mc[0] for mc in means_covars]).reshape(self.n_components,
                                                                       -1)
            gmm._set_covars(array([mc[1] for mc in means_covars]))
            return gmm

        return res

        self.in_dims = array(in_dims)
        self.out_dims = array(out_dims)
        means = zeros((self.n_components, len(out_dims)))
        covars = zeros((self.n_components, len(out_dims), len(out_dims)))
        weights = zeros((self.n_components,))
        sig_in = []
        inin_inv = []
        out_in = []
        mu_in = []
        for k, (weight_k, mean_k, covar_k) in enumerate(self):
            sig_in.append(covar_k[ix_(in_dims, in_dims)])
            inin_inv.append(matrix(sig_in).I)
            out_in.append(covar_k[ix_(out_dims, in_dims)])
            mu_in.append(mean_k[in_dims].reshape(-1, 1))

            means[k, :] = (mean_k[out_dims] +
                           (out_in *
                            inin_inv *
                            (value - mu_in)).T)

            covars[k, :, :] = (covar_k[ix_(out_dims, out_dims)] -
                               out_in *
                               inin_inv *
                               covar_k[ix_(in_dims, out_dims)])
            weights[k] = weight_k * Gaussian(mu_in.reshape(-1,),
                                             sig_in).normal(value.reshape(-1,))
        weights /= sum(weights)

        def p(value):
            # hard copy of useful matrices local to the function
            pass

        return p
Пример #26
0
        def comp(self,mean,var,covar,resp):

            # Store the indices to the missing and observed responses.
            miss=numpy.isnan(resp)
            obs=numpy.logical_not(miss)

            if miss.all():
                return mean,var,covar

            # Store the size of the model.
            numresp,numpred=numpy.size(self.gain)

            kalmgain=numpy.eye(numresp)
            josgain=numpy.eye(numresp)

            # Fill in the Kalman and Joseph gain matrices.
            ind=numpy.ix_(miss,obs)
            kalmgain[ind]=linalg.solve(self.noise[numpy.ix_(obs,obs)],
                self.noise[ind].transpose()).transpose()
            josgain[:,obs]=josgain[:,obs]-kalmgain[:,obs]

            # Compute the predictor/response co-variance.
            covar=covar.dot(josgain.transpose())

            # Condition the response mean/variance on the observations.
            mean=josgain.dot(mean)+numpy.dot(kalmgain[:,obs],resp[obs])
            var=numpy.dot(josgain,numpy.dot(var,josgain.transpose()))

            return mean,var,covar
Пример #27
0
def dctt1(a):
    """ dct  Discrete cosine transform.
    y = dct(a) returns the discrete cosine transform of a.
    The vector y is the same size as `a` and contains the
    discrete cosine transform coefficients.
    """
    if len(a.shape)==1:
        a = a.reshape(a.size,1)
    n,m = a.shape
    aa = a[:,:]
    #Compute weights to multiply DFT coefficients
    ww = arrayexp(n)
    if n%2 == 1:
        y = np.zeros([2*n,m])
        y[:n,:] = aa
        y[n:2*n,:] = np.flipud(aa)
        # Compute the FFT and keep the appropriate portion:
        yy = np.fft.fft(y,axis=0)
        yy = yy[:n,:]
    else:
        # Re-order the elements of the columns of x
        y = np.concatenate((aa[np.ix_(range(0,n,2))],\
                            aa[np.ix_(range(1,n,2)[::-1])]), axis=0)
        yy = np.fft.fft(y,axis=0)
        ww = 2*ww  # Double the weights for even-length case 

    wy = np.empty([n,m], complex)
    for j in range(m):
        wy[:,j]  = ww
    # Multiply FFT by weights:
    b = np.multiply(wy,yy)
    
    return b[:n,:m].real
Пример #28
0
def normal_eq_comb(AtA, AtB, PassSet = None):
    num_cholesky = 0
    num_eq = 0
    if AtB.size == 0:
        Z = np.zeros([])

    elif (PassSet is None) or np.all(PassSet):
        Z = nla.solve(AtA, AtB)
        num_cholesky = 1
        num_eq = AtB.shape[1]

    else:
        Z = np.zeros(AtB.shape) #(n, k)
        if PassSet.shape[1] == 1:
            if np.any(PassSet):
                cols = np.nonzero(PassSet)[0]
                Z[cols] = nla.solve(AtA[np.ix_(cols, cols)], AtB[cols])
                num_cholesky = 1
                num_eq = 1
        else:
            groups = column_group(PassSet)

            for g in groups:
                cols = np.nonzero(PassSet[:, g[0]])[0]

                if cols.size > 0:
                    ix1 = np.ix_(cols, g)
                    ix2 = np.ix_(cols, cols)

                    Z[ix1] = nla.solve(AtA[ix2], AtB[ix1])
                    num_cholesky += 1
                    num_eq += len(g)
                    num_eq += len(g)
    return Z, num_cholesky, num_eq
Пример #29
0
        def consgpattern():
            """
            binary pattern of the sparse nonlinear constraint gradient

            """

            vfdxpat = ( self.vfielddxpattern
                        if self.vfielddxpattern is not None
                        else np.ones( (self.Nstates,self.Nstates) ) )
            vfdupat = ( self.vfielddupattern
                        if self.vfielddupattern is not None
                        else np.ones( (self.Nstates,self.Ninputs) ) )
            if( self.Ncons > 0 ):
                consdxpat = ( self.consdxpattern
                              if self.consdxpattern is not None
                              else np.ones( (self.Ncons,self.Ninputs) ) )

            out = np.zeros( ( feuler.Ncons, feuler.N ), dtype=np.int )

            for k in range( Nsamples ):
                out[ np.ix_( dconsidx[:,k+1], stidx[:,k] ) ] = vfdxpat
                out[ np.ix_( dconsidx[:,k+1], uidx[:,k] ) ] = vfdupat

                if( self.Ncons > 0 ):
                    out[ np.ix_( iconsidx[:,k], stidx[:,k+1] ) ] = consdxpat

            return out
Пример #30
0
def _steadystate_direct_sparse(L, use_rcm=True, use_umfpack=False):
    """
    Direct solver that uses scipy sparse matrices
    """
    if settings.debug:
        print('Starting direct solver...')
    dims=L.dims[0]
    weight=np.abs(L.data.max())
    n = prod(L.dims[0][0])
    b = np.zeros((n ** 2, 1), dtype=complex)
    b[0,0] = weight
    L = L.data + sp.csr_matrix((weight*np.ones(n), (np.zeros(n), [nn * (n + 1) for nn in range(n)])),
                               shape=(n ** 2, n ** 2))
    L.sort_indices()
    use_solver(assumeSortedIndices=True, useUmfpack=use_umfpack)
    if use_rcm:
        perm = symrcm(L)
        L = sparse_permute(L,perm,perm)
        b = b[np.ix_(perm,)]
    
    v = spsolve(L, b)
    if use_rcm:
        rev_perm = np.argsort(perm)
        v = v[np.ix_(rev_perm,)]
    
    data = vec2mat(v)
    data = 0.5 * (data + data.conj().T)
    return Qobj(data, dims=dims, isherm=True)
Пример #31
0
    print '%3d' % i, '%10.5f' % evals[i]
print

# parameter SEs from fit (psv file)
modprmses = np.sqrt(np.diag(mfmodel['prmcov']))

parms = info.shape[0]
prmidx = np.array(range(parms)) + 1
freeparm = 1 - mfmodel['prmstat'][0:parms]
uparm = prmidx * freeparm
pscore = parmscores[uparm > 0]
uparm = uparm[uparm > 0] - 1

modprmses = modprmses[modprmses > 0]

infosub = info[np.ix_(uparm, uparm)]
covmat = np.linalg.inv(infosub)
parmses = np.sqrt(np.diag(covmat))
print "   pno    modse        Qse     Score"
for i in range(len(parmses)):
    print '%5d' % (
        uparm[i] + 1
    ), '%10.5f' % modprmses[i], '%10.5f' % parmses[i], '%10.5f' % pscore[i][0]

# ## G matrix for various Pu models  $$   \frac{d \mu}{d x}$$

# In[14]~

prmcov = mfmodel['prmcov']
prmvec = mfmodel['prmvec']
Пример #32
0
def automatic_community_detector_downsample(G, 
                                            method = 'un_louvain',
                                            downsample_level = 1,
                                            scale = 'log',
                                            figsize = (15,15),
                                            dpi = 100.,
                                            ylabel='Presynaptic Neuron ID',
                                            xlabel='Postsynaptic Neuron ID',
                                            xticklabels = None,
                                            yticklabels = None,
                                            title='Community-Ordered Connectivity Matrix (Log Scale)',
                                            vmax = None,
                                            export_format = None,
                                            figname='example_automatic_community'):
    """Automatically detects communities for a large networkx graph with downsampling for the adjacency matrix.
    
    # Arguments:
        G (network.Graph): A networkx graph or subclass object, including flybrainlab.graph.NeuronGraph.
        method (str): Method to use. One of un_louvain, label_propagation, leiden, louvain, walktrap or infomap.
        downsample_level (int): A downsampling level to use.
        scale (str): 'linear' or 'log'. Use linear or log scale to cluster.
        figsize (tuple): size of the figure.
        dpi (float): dpi of the figure.
        xlabel (str): Name of the x label.
        ylabel (str): Name of the y label.
        xticklabels (list): x tick labels to have.
        yticklabels (list): y tick labels to have.
        title (str): Title for the diagram.
        vmax (float): Maximum value for the diagram.
        export_format (str): if specified, file format to export the diagram.
        figname (str): Name for the output figure.
    
    # Returns:
        np.ndarray: community-ordered connectivity matrix in linear scale
        list: a list of list of node ids for each group member
    """
    Gun, all_list_nodes, all_nodes  = community_detection(G, method = method)

    all_pre_nodes = [i for i in all_list_nodes]
    all_post_nodes = [i for i in all_list_nodes]
    A = nx.adjacency_matrix(G).todense()[np.ix_(all_pre_nodes,all_post_nodes)]
    B = A[::downsample_level,::downsample_level].copy()

    if xticklabels is None:
        if isinstance(G, NeuronGraph):
            xticklabels = sum(nodes_to_unames(G, all_nodes),[])[::downsample_level]
        else:
            xticklabels = sum(all_nodes, [])[::downsample_level]
    if yticklabels is None:
        if isinstance(G, NeuronGraph):
            yticklabels = sum(nodes_to_unames(G, all_nodes),[])[::downsample_level]
        else:
            yticklabels = sum(all_nodes, [])[::downsample_level]
    if scale == 'log':
        Bd = np.log(1.+B)
    else:
        Bd = B
        if title == 'Community-Ordered Connectivity Matrix (Log Scale)':
            title = 'Community-Ordered Connectivity Matrix'

    sizes = np.round(np.cumsum([len(i) for i in all_nodes]) / downsample_level)
    
    gen_heatmap(Bd, figsize = figsize, dpi = dpi, xlabel = xlabel, ylabel = ylabel,
                xticklabels = xticklabels, yticklabels = yticklabels,
                hlines = sizes, vlines = sizes, title = title, vmax = vmax,
                export_format = export_format, figname = figname)
    return B, all_nodes
Пример #33
0
 def dp2():
     n = 3
     a = np.ones((n, ) * 5)
     i = np.random.randint(0, n, size=thesize)
     g = a[np.ix_(i, i, i, i, i)]
Пример #34
0
    def __getitem__(self, index):
        # print("GETITEM: ", index)
        indices, categories = index

        # translate category names into ranges
        cat_rgs = {}
        for category in categories:
            splt = category.split(":")
            if len(splt) == 2:
                if splt[1][-3:] == "hPa":
                    pressure2index = {
                        v: k
                        for k, v in self.config["vbls"][splt[0]]
                        ["index2pressure"].items()
                    }
                    idx = pressure2index[int(splt[1][:-3])]
                else:
                    idx = int(splt[1])
                offset = self.config["vbls"][splt[0]]["offset"]
                n_levels = 1 if self.config["vbls"][
                    splt[0]]["levels"] is None else len(
                        self.config["vbls"][splt[0]]["levels"])
                assert n_levels > idx, "invalid level index!"
                cat_rgs[category] = (
                    offset + idx, offset + idx,
                    self.config["vbls"][splt[0]]["type"] == "temp")
            else:
                offset = self.config["vbls"][category]["offset"]
                n_levels = 1 if self.config["vbls"][category][
                    "levels"] is None else len(
                        self.config["vbls"][category]["levels"])
                cat_rgs[category] = (
                    offset, offset + n_levels - 1,
                    self.config["vbls"][splt[0]]["type"] == "temp"
                )  # CHECK: Indices are inclusive right?

        if isinstance(indices, slice):
            # special instance - we can return a view!
            results_dict = {
                cat_name: self.fo[indices, slice(rg[0], rg[1] + 1, 1)]
                if rg[2] else self.fo[slice(rg[0], rg[1] + 1, 1)]
                for cat_name, rg in cat_rgs.items()
            }
        else:
            results_dict = {}
            for cat_name, rg in cat_rgs.items():
                if indices is not None:
                    results_dict[cat_name] = self.fo[np.ix_(
                        indices, np.arange(rg[0], rg[1] + 1))]
                else:
                    if rg[2]:
                        # print("BRANCH TMP NON INDICES:", rg)
                        results_dict[cat_name] = self.fo[
                            slice(None, None),
                            slice(rg[0], rg[1] + 1)]
                        # self.fo[np.arange(self.fo.shape[0]), np.arange(rg[0], rg[1] + 1)]
                    else:
                        results_dict[cat_name] = self.fo[slice(
                            rg[0], rg[1] + 1)]
                        # np.arange(rg[0], rg[1] + 1)

        return results_dict
Пример #35
0
 def compute_h(self):
     self.h = (self.nu - self.b)/self.n * self.K[np.ix_(self.full_id, self.train_id)].sum(axis = 1) + \
                 (self.nl + self.b)/self.n * self.K[np.ix_(self.full_id, self.pool_id)].sum(axis = 1)
Пример #36
0
 def R_score(self, ids):
     R = 0.5 * self.K[np.ix_(ids, ids)].sum() + \
     (self.nu - self.b)/self.n * self.K[np.ix_(ids, self.train_id)].sum() + \
     (self.nl + self.b)/self.n * self.K[np.ix_(ids, self.pool_id)].sum()
     return R
Пример #37
0
 def update_h(self):
     self.h = self.h - (self.b/self.n) * self.K[np.ix_(self.full_id, self.full_id)].sum(axis = 1) + \
         ((self.nu - self.nl - 4*self.b)/self.n) * self.K[np.ix_(self.full_id, self.selec_id)].sum(axis = 1)
Пример #38
0
            map(
                str.split,
                open(
                    os.path.join(
                        EVAL_DIR,
                        '%s_test_split%d.txt' % (classLabel, 1 + SPLIT_IND)))))
        train += [(fixClipName(k), classLabel) for k, v in d.items()
                  if v == '1']
        test += [(fixClipName(k), classLabel) for k, v in d.items()
                 if v == '2']

    return (train, test)


splits = map(read_split, range(3))
slice_kernel = lambda inds1, inds2: all_k[np.ix_(map(allClips.index, inds1),
                                                 map(allClips.index, inds2))]
REG_C = 1.0


def svm_train_test(train_k, test_k, ytrain, REG_C):
    model = SVC(kernel='precomputed', C=REG_C, max_iter=10000)
    model.fit(train_k, ytrain)

    flatten = lambda ls: list(itertools.chain(*ls))
    train_conf, test_conf = map(
        flatten, map(model.decision_function, [train_k, test_k]))
    return train_conf, test_conf


def one_vs_rest(SPLIT_IND):
    calc_accuracy = lambda chosen, true: sum(
Пример #39
0
def _pyramid_single_interpolation(G, ca, pe, keep_inds, h_filter, **kwargs):
    r"""Synthesize a single level of the graph pyramid transform.

    Parameters
    ----------
    G : Graph
        Graph structure on which the signal resides.
    ca : ndarray
        Coarse approximation of the signal on a reduced graph.
    pe : ndarray
        Prediction error that was made when forming the current coarse approximation.
    keep_inds : ndarray
        The indices of the vertices to keep when downsampling the graph and signal.
    h_filter : lambda expression
        The filter in use at this level.
    use_landweber : bool
        To use the Landweber iteration approximation in the least squares synthesis.
        Default is False.
    reg_eps : float
        Interpolation parameter. Default is 0.005.
    landweber_its : int
        Number of iterations in the Landweber approximation for least squares synthesis.
        Default is 50.
    landweber_tau : float
        Parameter for the Landweber iteration. Default is 1.

    Returns
    -------
    finer_approx :
        Coarse approximation of the signal on a higher resolution graph.

    """
    nb_ind = keep_inds.shape
    N = G.N
    reg_eps = float(kwargs.pop('reg_eps', 0.005))
    use_landweber = bool(kwargs.pop('use_landweber', False))
    landweber_its = int(kwargs.pop('landweber_its', 50))
    landweber_tau = float(kwargs.pop('landweber_tau', 1.))

    # index matrix (nb_ind x N) of keep_inds, S_i,j = 1 iff keep_inds[i] = j
    S = sparse.csr_matrix(([1] * nb_ind, (range(nb_ind), keep_inds)),
                          shape=(nb_ind, N))

    if use_landweber:
        x = np.zeros(N)
        z = np.concatenate((ca, pe), axis=0)
        green_kernel = filters.Filter(G, lambda x: 1. / (x + reg_eps))
        PhiVlt = _analysis(green_kernel, S.T, **kwargs).T
        filt = filters.Filter(G, h_filter, **kwargs)

        for iteration in range(landweber_its):
            h_filtered_sig = _analysis(filt, x, **kwargs)
            x_bar = h_filtered_sig[keep_inds]
            y_bar = x - interpolate(G, x_bar, keep_inds, **kwargs)
            z_delt = np.concatenate((x_bar, y_bar), axis=0)
            z_delt = z - z_delt
            alpha_new = PhiVlt * z_delt[nb_ind:]
            x_up = sparse.csr_matrix((z_delt, (range(nb_ind), [1] * nb_ind)),
                                     shape=(N, 1))
            reg_L = G.L + reg_esp * sparse.eye(N)

            elim_inds = np.setdiff1d(np.arange(N, dtype=int), keep_inds)
            L_red = reg_L[np.ix_(keep_inds, keep_inds)]
            L_in_out = reg_L[np.ix_(keep_inds, elim_inds)]
            L_out_in = reg_L[np.ix_(elim_inds, keep_inds)]
            L_comp = reg_L[np.ix_(elim_inds, elim_inds)]

            next_term = L_red * alpha_new - L_in_out * linalg.spsolve(
                L_comp, L_out_in * alpha_new)
            next_up = sparse.csr_matrix((next_term, (keep_inds, [1] * nb_ind)),
                                        shape=(N, 1))
            x += landweber_tau * _analysis(filt, x_up - next_up, **
                                           kwargs) + z_delt[nb_ind:]

        finer_approx = x

    else:
        # When the graph is small enough, we can do a full eigendecomposition
        # and compute the full analysis operator T_a
        H = G.U * sparse.diags(h_filter(G.e), 0) * G.U.T
        Phi = G.U * sparse.diags(1. / (reg_eps + G.e), 0) * G.U.T
        Ta = np.concatenate(
            (S * H, sparse.eye(G.N) - Phi[:, keep_inds] *
             linalg.spsolve(Phi[np.ix_(keep_inds, keep_inds)], S * H)),
            axis=0)
        finer_approx = linalg.spsolve(Ta.T * Ta,
                                      Ta.T * np.concatenate((ca, pe), axis=0))
Пример #40
0
print(20 * "-")
'''
花式索引
    前一个取出行,后一个按位取出对应列
    与索引器不同
    
索引器
'''
arr_new = np.arange(32).reshape((8, 4))
print(arr_new, "\n")

print(arr_new[[0, 3, 5]], "\n")  # 输出对应行
print(arr_new[[0, 3, 5], [0, 3, 2]], '\n')  # 对应行取出列的元素 [0][0] [3][3] [5][2]

x = arr_new[np.ix_([0, 3, 5], [0, 3, 2])]  # ix_生成一个索引器(用于取出特定行列的数据)
print(x)

print(30 * '-')
'''
一些方法
'''
A = np.arange(12).reshape((3, 4))
print(A, '\n')
for row in A:
    print(row)  # 迭代每一行
print('打住')
for column in A.T:
    print(column)  # 迭代每一列
for item in A.flat:  # 把A转变成一行的序列
    print(item)  # 迭代每一项
Пример #41
0
                     None:10])  # E: list[numpy.ndarray[Any, numpy.dtype[Any]]]

reveal_type(np.index_exp[0:1])  # E: Tuple[builtins.slice]
reveal_type(np.index_exp[0:1,
                         None:3])  # E: Tuple[builtins.slice, builtins.slice]
reveal_type(
    np.index_exp[0, 0:1, ..., [0, 1, 3]]
)  # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]]

reveal_type(np.s_[0:1])  # E: builtins.slice
reveal_type(np.s_[0:1, None:3])  # E: Tuple[builtins.slice, builtins.slice]
reveal_type(
    np.s_[0, 0:1, ..., [0, 1, 3]]
)  # E: Tuple[Literal[0]?, builtins.slice, builtins.ellipsis, builtins.list[builtins.int]]

reveal_type(np.ix_(
    AR_LIKE_b))  # E: tuple[numpy.ndarray[Any, numpy.dtype[numpy.bool_]]]
reveal_type(
    np.ix_(AR_LIKE_i,
           AR_LIKE_f))  # E: tuple[numpy.ndarray[Any, numpy.dtype[{double}]]]
reveal_type(
    np.ix_(AR_i8))  # E: tuple[numpy.ndarray[Any, numpy.dtype[{int64}]]]

reveal_type(np.fill_diagonal(AR_i8, 5))  # E: None

reveal_type(
    np.diag_indices(4))  # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]
reveal_type(np.diag_indices(
    2, 3))  # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]

reveal_type(np.diag_indices_from(
    AR_i8))  # E: tuple[numpy.ndarray[Any, numpy.dtype[{int_}]]]
Пример #42
0
def misorientation(q1, q2, *args):
    """
    sym is a tuple (crystal_symmetry, *sample_symmetry)
    generally coded, may split up special cases for no symmetry or crystal/sample only...
    """
    if not isinstance(q1, ndarray) or not isinstance(q2, ndarray):
        raise RuntimeError, "quaternion args are not of type `numpy ndarray'"

    if q1.ndim != 2 or q2.ndim != 2:
        raise RuntimeError, "quaternion args are the wrong shape; must be 2-d (columns)"

    if q1.shape[1] != 1:
        raise RuntimeError(
            "first argument should be a single quaternion, got shape %s" %
            (q1.shape, ))

    if len(args) == 0:
        # no symmetries; use identity
        sym = (c_[1., 0, 0, 0].T, c_[1., 0, 0, 0].T)
    else:
        sym = args[0]
        if len(sym) == 1:
            if not isinstance(sym[0], ndarray):
                raise RuntimeError, "symmetry argument is not an numpy array"
            else:
                # add triclinic sample symmetry (identity)
                sym += (c_[1., 0, 0, 0].T, )
        elif len(sym) == 2:
            if not isinstance(sym[0], ndarray) or not isinstance(
                    sym[1], ndarray):
                raise RuntimeError, "symmetry arguments are not an numpy arrays"
        elif len(sym) > 2:
            raise RuntimeError, "symmetry argument has %d entries; should be 1 or 2" % (
                len(sym))

    # set some lengths
    n = q2.shape[1]  # length of misorientation list
    m = sym[0].shape[1]  # crystal (right)
    p = sym[1].shape[1]  # sample  (left)

    # tile q1 inverse
    q1i = quatProductMatrix(invertQuat(q1), mult='right').squeeze()

    # convert symmetries to (4, 4) qprod matrices
    rsym = quatProductMatrix(sym[0], mult='right')
    lsym = quatProductMatrix(sym[1], mult='left')

    # Do R * Gc, store as
    # [q2[:, 0] * Gc[:, 0:m], ..., q2[:, n-1] * Gc[:, 0:m]]
    q2 = dot(rsym, q2).transpose(2, 0, 1).reshape(m * n, 4).T

    # Do Gs * (R * Gc), store as
    # [Gs[:, 0:p] * q[:,   0] * Gc[:, 0], ... , Gs[:, 0:p] * q[:,   0] * Gc[:, m-1], ...
    #  Gs[:, 0:p] * q[:, n-1] * Gc[:, 0], ... , Gs[:, 0:p] * q[:, n-1] * Gc[:, m-1]]
    q2 = dot(lsym, q2).transpose(2, 0, 1).reshape(p * m * n, 4).T

    # Calculate the class misorientations for full symmetrically equivalent
    # classes for q1 and q2.  Note the use of the fact that the application
    # of the symmetry groups is an isometry.
    eqvMis = fixQuat(dot(q1i, q2))

    # Reshape scalar comp columnwise by point in q2 (and q1, if applicable)
    sclEqvMis = eqvMis[0, :].reshape(n, p * m).T

    # Find misorientation closest to origin for each n equivalence classes
    #   - fixed quats so garaunteed that sclEqvMis is nonnegative
    qmax = sclEqvMis.max(0)

    # remap indices to use in eqvMis
    qmaxInd = (sclEqvMis == qmax).nonzero()
    qmaxInd = c_[qmaxInd[0], qmaxInd[1]]

    eqvMisColInd = sort(qmaxInd[:, 0] + qmaxInd[:, 1] * p * m)

    # store Rmin in q
    mis = eqvMis[ix_(range(4), eqvMisColInd)]

    angle = 2 * arccosSafe(qmax)

    return angle, mis
Пример #43
0
def crop_image1(img, tol=7):
    # img is image data
    # tol  is tolerance
    mask = img > tol
    return img[np.ix_(mask.any(1), mask.any(0))]
Пример #44
0
def kron_reduction(G, ind):
    r"""Compute the Kron reduction.

    This function perform the Kron reduction of the weight matrix in the
    graph *G*, with boundary nodes labeled by *ind*. This function will
    create a new graph with a weight matrix Wnew that contain only boundary
    nodes and is computed as the Schur complement of the original matrix
    with respect to the selected indices.

    Parameters
    ----------
    G : Graph or sparse matrix
        Graph structure or weight matrix
    ind : list
        indices of the nodes to keep

    Returns
    -------
    Gnew : Graph or sparse matrix
        New graph structure or weight matrix


    References
    ----------
    See :cite:`dorfler2013kron`

    """
    if isinstance(G, graphs.Graph):

        if G.lap_type != 'combinatorial':
            msg = 'Unknown reduction for {} Laplacian.'.format(G.lap_type)
            raise NotImplementedError(msg)

        if G.is_directed():
            msg = 'This method only work for undirected graphs.'
            raise NotImplementedError(msg)

        L = G.L

    else:

        L = G

    N = np.shape(L)[0]
    ind_comp = np.setdiff1d(np.arange(N, dtype=int), ind)

    L_red = L[np.ix_(ind, ind)]
    L_in_out = L[np.ix_(ind, ind_comp)]
    L_out_in = L[np.ix_(ind_comp, ind)].tocsc()
    L_comp = L[np.ix_(ind_comp, ind_comp)].tocsc()

    Lnew = L_red - L_in_out.dot(linalg.spsolve(L_comp, L_out_in))

    # Make the laplacian symmetric if it is almost symmetric!
    if np.abs(Lnew - Lnew.T).sum() < np.spacing(1) * np.abs(Lnew).sum():
        Lnew = (Lnew + Lnew.T) / 2.

    if isinstance(G, graphs.Graph):
        # Suppress the diagonal ? This is a good question?
        Wnew = sparse.diags(Lnew.diagonal(), 0) - Lnew
        Snew = Lnew.diagonal() - np.ravel(Wnew.sum(0))
        if np.linalg.norm(Snew, 2) >= np.spacing(1000):
            Wnew = Wnew + sparse.diags(Snew, 0)

        # Removing diagonal for stability
        Wnew = Wnew - Wnew.diagonal()

        coords = G.coords[ind, :] if len(G.coords.shape) else np.ndarray(None)
        Gnew = graphs.Graph(Wnew,
                            coords=coords,
                            lap_type=G.lap_type,
                            plotting=G.plotting)
    else:
        Gnew = Lnew

    return Gnew
            B[e, p, q] = Bpq

            # se ensamblan la matriz de rigidez del elemento y el vector de
            # fuerzas nodales equivalentes del elemento
            Ke16 += Bpq.T @ De[mat[e]] @ Bpq * det_Je[p, q] * te[
                mat[e]] * w_gl[p] * w_gl[q]
            fe += Npq[:, :8].T @ be[mat[e]] * det_Je[p, q] * te[
                mat[e]] * w_gl[p] * w_gl[q]

    # se determina si hay puntos con jacobiano negativo, en caso tal se termina
    # el programa y se reporta
    if np.any(det_Je <= 0):
        raise Exception(f'Hay puntos con det_Je negativo en el elemento {e+1}')

    # se condensan los GDL jerárquicos u5, v5, u6, v6
    Krr = Ke16[np.ix_(r_, r_)]
    Ker[e] = Ke16[np.ix_(e_, r_)]
    Kre = Ke16[np.ix_(r_, e_)]
    inv_Kee[e] = np.linalg.inv(Ke16[np.ix_(e_, e_)])
    Ke = Krr - Kre @ inv_Kee[e] @ Ker[e]

    # y se añaden la matriz de rigidez del elemento y el vector de fuerzas
    # nodales del elemento a sus respectivos arreglos de la estructura
    idx[e] = gdl[LaG[e]].flatten()  # se obtienen los grados de libertad
    K[np.ix_(idx[e], idx[e])] += Ke
    f[np.ix_(idx[e])] += fe

# %% Muestro la configuración de la matriz K (K es rala)
plt.figure()
plt.spy(K)
plt.title('Los puntos representan los elementos diferentes de cero')
Пример #46
0
    def compute_symmetric_double_LOO(self):

        #bevals_col = np.multiply(self.evals2, self.newevals2).T
        #multiplyright = self.U.T * self.Y.T
        #I = np.mat(np.identity(2))

        G = np.multiply(
            (self.newevals1.T - (1. / self.regparam1)), self.V) * self.V.T + (
                1. / self.regparam1) * np.mat(np.identity(self.K1.shape[0]))
        #G2 = np.multiply((self.newevals2.T-(1./self.regparam)), self.U) * self.U.T + (1./self.regparam) * np.mat(np.identity(self.K2.shape[0]))
        GY = G * self.Y
        #YG2 = self.Y * G2
        GYG = GY * G
        #A2 = G2 * self.Y.T

        i, j = 2, 4
        inds = [i, j]

        #A = self.U[inds]
        #right = multiplyright - A.T * self.Y.T[inds]
        #RQY = A * np.multiply(bevals_col.T, right)
        #B = np.multiply(bevals_col.T, A.T)
        #HO_col = (la.inv(I - A * B) * RQY).T

        #HO_col = (self.Y.T[inds]-la.inv(G2[np.ix_(inds, inds)]) * A2[inds]).T
        #print HO_col.shape

        #bevals_row = np.multiply(self.evals1, self.newevals1).T
        #multiplyright = self.V.T * HO_col

        #A = self.V[inds]
        #right = multiplyright - A.T * HO_col[inds]
        #RQY = A * np.multiply(bevals_row.T, right)
        #B = np.multiply(bevals_col.T, A.T)
        #HO_row = la.inv(I - A * B) * RQY

        #A1 = G1[inds] * HO_col
        #HO_row = HO_col[inds]-la.inv(G1[np.ix_(inds, inds)]) * A1

        #HO_rowr = self.Y[np.ix_(inds, inds)] \
        #    - YG2[np.ix_(inds, inds)] * la.inv(G2[np.ix_(inds, inds)]) \
        #    - la.inv(G1[np.ix_(inds, inds)]) * G1Y[np.ix_(inds, inds)] \
        #    + la.inv(G1[np.ix_(inds, inds)]) * G1YG2[np.ix_(inds, inds)] * la.inv(G2[np.ix_(inds, inds)])

        invGii = la.inv(G[np.ix_(inds, inds)])
        GYii = GY[np.ix_(inds, inds)]
        invGiiGYii = invGii * GYii
        HO_rowr = self.Y[np.ix_(inds, inds)] \
            - invGiiGYii.T \
            - invGiiGYii \
            + invGii * GYG[np.ix_(inds, inds)] * invGii

        #II1 = np.mat(np.identity(self.Y.shape[0]))[inds]
        #II2 = np.mat(np.identity(self.Y.shape[1]))[:, inds]
        #HO_rowr = (II1 - la.inv(G1[np.ix_(inds, inds)]) * G1[inds]) * self.Y * (II2 - G2[:, inds] * la.inv(G2[np.ix_(inds, inds)]))

        #print HO_row.shape
        results = np.zeros((self.Y.shape[0], self.Y.shape[1]))
        cython_two_step_rls_cv.compute_symmetric_double_loo(
            G, self.Y, GY, GYG, results, self.Y.shape[0], self.Y.shape[1])
        return results
Пример #47
0
        dict = {'Size': i}

        start = time.clock()
        result = m[:, indexSet][indexSet, :]
        result @ result.T
        end = time.clock()

        mstart = time.clock()
        result @ result.T
        mend = time.clock()
        mtime = mend - mstart

        dict['FancyIndexing'] = end - start - mtime

        start = time.clock()
        result3 = m[np.ix_(indexSet, indexSet)]
        result3 @ result3.T
        end = time.clock()

        mstart = time.clock()
        result3 @ result3.T
        mend = time.clock()
        mtime = mend - mstart

        dict['ix'] = end - start - mtime

        frame = frame.append(dict, ignore_index=True)

    frame.to_csv('intel/timeNPTakeSparse' + str(x) + '.csv', index=False)

names = glob.glob('intel/timeNPTakeSparse[0-9].csv')
Пример #48
0
    def generateSpectralPoints(self, outfile, union=False, ndiimgfile=None):
        """
        Main function to generate spectral points
        """
        if union and (ndiimgfile is None):
            raise RuntimeError(
                "You must provdie a gap-filled NDI image to generation spectral points from the union of two point clouds"
            )
        self.union = union
        self.ndiimgfile = ndiimgfile

        # get inputs from command line or defaults
        nirfile = self.nirfile
        swirfile = self.swirfile
        rdiff_thresh = float(self.rdiff_thresh)
        nrows = int(self.nrows)
        ncols = int(self.ncols)

        print "Input NIR point cloud: \n\t" + nirfile
        print "Input SWIR point cloud: \n\t" + swirfile
        print "Output merged point cloud: \n\t" + outfile
        print "Range difference threshold: {0:.3f}".format(rdiff_thresh)
        print "Info for converting sample and line to shot number: \n" \
            + "\tnumber of rows: {0:d}, number of columns: {1:d}".format(nrows, ncols)

        # read points from text file
        print "Loading points"
        nirpoints = np.genfromtxt(nirfile, dtype=np.float32, usecols=self.ind, \
                                      delimiter=',', skip_header=self.headerlines, \
                                      filling_values=np.nan, usemask=False, \
                                      comments=None)

        swirpoints = np.genfromtxt(swirfile, dtype=np.float32, usecols=self.ind, \
                                      delimiter=',', skip_header=self.headerlines, \
                                      filling_values=np.nan, usemask=False, \
                                       comments=None)

        # update column index in loaded array
        cind = {il: i for i, il in enumerate(self.ind_label)}

        # index the columns that are used in searching common points
        ind = [cind['range'], cind['sample'], cind['line']]

        print "Generating spectral points ..."
        intersectout = self.intersectPointClouds(nirpoints[:, ind],
                                                 swirpoints[:, ind])
        nir_ind = intersectout[0]
        swir_ind = intersectout[1]
        return_type = intersectout[2]
        nu_shotnum = intersectout[3]
        num_of_returns = intersectout[4]
        return_num = intersectout[5]

        # return_type now gives QA flag QA flag is a three-bit integer, from MSB
        # to LSB, each bit tells if NDI, NIR or SWIR is measured value (0) or
        # synthetic value (1)
        return_type[:] = int('000', 2)
        # calculate the mean point location
        ind = [cind['x'], cind['y'], cind['z'], cind['shot_number'], cind['range'], cind['theta'], \
                   cind['phi'], cind['sample'], cind['line']]
        nreturn = len(return_type)
        meanpoints = (nirpoints[np.ix_(nir_ind, ind)] +
                      swirpoints[np.ix_(swir_ind, ind)]) / 2.0
        if self.man_col < self.tot_col:
            outpoints = np.hstack((meanpoints[:, 0:3], \
                                   nirpoints[nir_ind, cind['d_I']:cind['d_I']+1], \
                                   swirpoints[swir_ind, cind['d_I']:cind['d_I']+1], \
                                   return_num.reshape((nreturn, 1)), \
                                   num_of_returns.reshape((nreturn, 1)), \
                                   nu_shotnum.reshape((nreturn, 1)), \
                                   meanpoints[:, 4:9], \
                                   nirpoints[nir_ind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
                                   swirpoints[swir_ind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
                                   return_type.reshape((nreturn, 1)) \
            ))
        else:
            outpoints = np.hstack((meanpoints[:, 0:3], \
                                   nirpoints[nir_ind, cind['d_I']:cind['d_I']+1], \
                                   swirpoints[swir_ind, cind['d_I']:cind['d_I']+1], \
                                   return_num.reshape((nreturn, 1)), \
                                   num_of_returns.reshape((nreturn, 1)), \
                                   nu_shotnum.reshape((nreturn, 1)), \
                                   meanpoints[:, 4:9], \
                                   return_type.reshape((nreturn, 1)) \
            ))

        prefixstr = "[DWEL Dual-wavelength Point Cloud Data by {0:s}]\n".format("Union" if self.union else "Intersect") \
            +"Run made at: "+time.strftime("%c")+"\n"
        headerstr = prefixstr + "x,y,z,d_I_nir,d_I_swir,return_number,number_of_returns,shot_number,range,theta,phi,sample,line," \
                    + ",".join([ il+"_nir" for il in self.ind_label[self.man_col:self.tot_col] ]) + "," \
                    + ",".join([ il+"_swir" for il in self.ind_label[self.man_col:self.tot_col] ]) + "," \
                    + "qa,r,g,b"
        # format string
        if self.man_col < self.tot_col:
            col_dtype = self.inferColDataType()
            fmtstr = "%.3f "*5 + "%d "*3 + "%.3f "*3 + "%d "*2 \
                     + " ".join([ col_dtype[ci] for ci in self.ind[self.man_col:self.tot_col] ]) + " " \
                     + " ".join([ col_dtype[ci] for ci in self.ind[self.man_col:self.tot_col] ]) + " " \
                     + "%d "*4
        else:
            col_dtype = self.inferColDataType()
            fmtstr = "%.3f "*5 + "%d "*3 + "%.3f "*3 + "%d "*2 \
                     + "%d "*4
        fmtstr = fmtstr.strip().split(" ")

        if not (self.union):
            # spectral points by intersect
            # exclude zero-hit points from rgb generation
            rgbpoints = np.zeros((outpoints.shape[0], 3))
            hitmask = outpoints[:, 6].astype(int) > 0
            # fix extremely large reflectance values
            bound = np.percentile(outpoints[:, 3:5][hitmask, :], 98, axis=0)
            bound[bound < self.i_scale] = self.i_scale
            outpoints[outpoints[:, 3] > bound[0], 3] = bound[0]
            outpoints[outpoints[:, 4] > bound[1], 4] = bound[1]
            # generate pseudo-color composite
            rgbpoints[hitmask, :] = self.colorComposite(np.hstack((outpoints[:, 4:5][hitmask, :], \
                                                           outpoints[:, 3:4][hitmask, :], \
                                                           np.zeros_like(outpoints[:, 3:4][hitmask, :]) \
                                                           )))
            outpoints = np.hstack((outpoints, rgbpoints))

            print "Saving dual-wavelength points: " + str(nir_ind.size)

            np.savetxt(outfile, outpoints, delimiter=',', fmt=fmtstr, \
                header=headerstr.rstrip(), comments='')

        else:
            # spectral points by union
            nir_unpind = intersectout[6]
            swir_unpind = intersectout[7]
            nir_unp_nu_shotnum = intersectout[8]
            swir_unp_nu_shotnum = intersectout[9]
            ind = [cind['d_I'], cind['sample'], cind['line']]
            print "\tMatching more point pairs between NIR and SWIR for union approach"
            unionout = self.unionPointClouds(nirpoints[np.ix_(nir_unpind, ind)], \
                                                 swirpoints[np.ix_(swir_unpind, ind)], \
                                                 ndiimgfile)
            nir2swir_amp = unionout[0]
            nir2swir_qa = unionout[1]
            swir2nir_amp = unionout[2]
            swir2nir_qa = unionout[3]

            tmpflag = np.equal(nir2swir_qa, int('111', 2))
            if np.greater(nirpoints[nir_unpind[tmpflag], cind['d_I']],
                          1e-10).any():
                warnings.warn(
                    "Some no-return shots give non-zero NIR reflectance",
                    RuntimeWarning)
                nirpoints[nir_unpind[tmpflag], cind['d_I']] = 0.0
            tmpflag = np.equal(swir2nir_qa, int('111', 2))
            if np.greater(swirpoints[swir_unpind[tmpflag], cind['d_I']],
                          1e-10).any():
                warnings.warn(
                    "Some no-return shots give non-zero SWIR reflectance",
                    RuntimeWarning)
                swirpoints[swir_unpind[tmpflag], cind['d_I']] = 0.0

            if self.man_col < self.tot_col:
                nir2swir_points = np.hstack(( nirpoints[nir_unpind, 0:3], \
                                              nirpoints[nir_unpind, cind['d_I']:cind['d_I']+1], \
                                              nir2swir_amp.reshape((len(nir2swir_amp), 1)), \
                                              np.zeros((len(nir2swir_amp), 3)), \
                                              nirpoints[nir_unpind, :][:, 5:10], \
                                              nirpoints[nir_unpind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
                                              np.zeros((len(nir2swir_amp), self.tot_col-self.man_col)), \
                                              nir2swir_qa.reshape((len(nir2swir_qa), 1)) \
                ))
                swir2nir_points = np.hstack(( swirpoints[swir_unpind, 0:3], \
                                              swir2nir_amp.reshape((len(swir2nir_amp), 1)), \
                                              swirpoints[swir_unpind, cind['d_I']:cind['d_I']+1], \
                                              np.zeros((len(swir2nir_amp), 3)), \
                                              swirpoints[swir_unpind, :][:, 5:10], \
                                              np.zeros((len(swir2nir_amp), self.tot_col-self.man_col)), \
                                              swirpoints[swir_unpind, cind[self.ind_label[self.man_col]]:cind[self.ind_label[self.tot_col-1]]+1], \
                                              swir2nir_qa.reshape((len(swir2nir_qa), 1)) \
                ))
            else:
                nir2swir_points = np.hstack(( nirpoints[nir_unpind, 0:3], \
                                              nirpoints[nir_unpind, cind['d_I']:cind['d_I']+1], \
                                              nir2swir_amp.reshape((len(nir2swir_amp), 1)), \
                                              np.zeros((len(nir2swir_amp), 3)), \
                                              nirpoints[nir_unpind, :][:, 5:10], \
                                              nir2swir_qa.reshape((len(nir2swir_qa), 1)) \
                ))
                swir2nir_points = np.hstack(( swirpoints[swir_unpind, 0:3], \
                                              swir2nir_amp.reshape((len(swir2nir_amp), 1)), \
                                              swirpoints[swir_unpind, cind['d_I']:cind['d_I']+1], \
                                              np.zeros((len(swir2nir_amp), 3)), \
                                              swirpoints[swir_unpind, :][:, 5:10], \
                                              swir2nir_qa.reshape((len(swir2nir_qa), 1)) \
                ))

            unionoutpoints = np.vstack(
                (outpoints, nir2swir_points, swir2nir_points))
            sortind, num_of_returns, return_num, shotind = \
                self.updateReturnNum(unionoutpoints[:, [8, 11, 12]].copy())
            unionoutpoints = unionoutpoints[sortind, :]
            unionoutpoints[:, 5] = return_num
            unionoutpoints[:, 6] = num_of_returns
            unionoutpoints[:, 7] = shotind

            # exclude zero-hit points from rgb generation
            rgbpoints = np.zeros((unionoutpoints.shape[0], 3))
            hitmask = unionoutpoints[:, 6].astype(int) > 0
            # fix extremely large reflectance values
            bound = np.percentile(unionoutpoints[:, 3:5][hitmask, :],
                                  98,
                                  axis=0)
            bound[bound < self.i_scale] = self.i_scale
            unionoutpoints[unionoutpoints[:, 3] > bound[0], 3] = bound[0]
            unionoutpoints[unionoutpoints[:, 4] > bound[1], 4] = bound[1]
            # generate pseudo-color composite
            rgbpoints[hitmask, :] = self.colorComposite(np.hstack((unionoutpoints[:, 4:5][hitmask, :], \
                                                           unionoutpoints[:, 3:4][hitmask, :], \
                                                           np.zeros_like(unionoutpoints[:, 3:4][hitmask, :]) \
                                                           )))
            unionoutpoints = np.hstack((unionoutpoints, rgbpoints))

            print "Saving dual-wavelength points: " + str(len(unionoutpoints))

            np.savetxt(outfile, unionoutpoints, delimiter=',', fmt=fmtstr, \
                header=headerstr.rstrip(), comments='')
Пример #49
0
import scipy.misc
import matplotlib.pyplot as plt
import numpy as np

# Load the Lena array
lena = scipy.misc.lena()
xmax = lena.shape[0]
ymax = lena.shape[1]


def shuffle_indices(size):
    '''
   Shuffles an array with values 0 - size
   '''
    arr = np.arange(size)
    np.random.shuffle(arr)

    return arr


xindices = shuffle_indices(xmax)
np.testing.assert_equal(len(xindices), xmax)
yindices = shuffle_indices(ymax)
np.testing.assert_equal(len(yindices), ymax)

# Plot Lena
plt.imshow(lena[np.ix_(xindices, yindices)])
plt.show()
Пример #50
0
#----------------------------------------------------------------------
#  Calculate K
#----------------------------------------------------------------------

totDof = 2 * len(coords)

K = zeros(shape=(totDof, totDof))

for elem in elems:
    elemDofs = getDofs(elem)

    sData = getElemShapeData(coords[elem, :])

    for iData in sData:
        b = getBmatrix(iData.dhdx)
        K[ix_(elemDofs,
              elemDofs)] += dot(b.transpose(), dot(D, b)) * iData.weight

#----------------------------------------------------------------------
#  Solve Ka=f
#----------------------------------------------------------------------

consDof = len(presInds)

C = zeros(shape=(totDof, totDof - consDof))

j = 0

for i in range(totDof):
    if i in presInds:
        continue
    C[i, j] = 1.
    def regression(self, i: int, c: list=None, lam=0):
        S, P = self.S, self.P
        d = [j for j in range(self.p) if j not in c and j != i] + [i]

        if c is None or len(c) == 0:
            coefs = []
            var = S[i, i]
            S_inv = None

        # use Schur complement when conditioning to keep inverted submatrix small
        elif len(c) < self.p / 2 or P is None:
            if lam == 0 and np.isclose(np.diag(S[ix_(c, c)]), 0).any():
                coefs, var, _, _ = lstsq(S[ix_(c, c)], S[c, i])
                var = S[i, i] - S[i, c] @ coefs
                S_inv = pinv(S[ix_(c, c)])
            else:
                try:
                    S_inv = inv(S[ix_(c, c)] + lam*np.eye(len(c)))
                    coefs = S_inv @ S[c, i]
                    var = S[i, i] - S[i, c] @ S_inv @ S[c, i]
                except LinAlgError:
                    coefs, var, _, _ = lstsq(S[ix_(c, c)], S[c, i])
                    var = S[i, i] - S[i, c] @ coefs
                    S_inv = pinv(S[ix_(c, c)])

        # use Schur complement when marginalizing to keep inverted submatrix small
        else:
            P_inv = inv(P[ix_(d, d)])
            S_inv = P[ix_(c, c)] - P[ix_(c, d)] @ P_inv @ P[ix_(d, c)]
            coefs = S_inv @ S[c, i]
            var = inv(P[ix_(d, d)])[-1, -1]

        # correct the variance to account for the number of degrees of freedom
        var = var * self.n / (self.n - len(c) + 1)

        return coefs, var, S_inv
Пример #52
0
def run_dsistudio(subj,
                  subjdir,
                  masks_dir,
                  preproc_dir,
                  dset_dir,
                  modality,
                  shell='10k',
                  nr_fibers=20000,
                  trk_output='tracts20k',
                  suppress_output=False,
                  preproc_stem=None):
    import subprocess

    if modality in ('qball', 'qbi'):
        stem = 'diff_disco_eddy'
        methodnr = '4'
        methodname = 'gqi'
        param0 = '1.25'
        param1 = None
        btable_string = 'btable.txt'
    elif modality in ('singleshell', 'sgsh'):
        stem = 'diff_singleshell_%s' % shell
        methodnr = '3'
        methodname = 'qbi'
        param0 = '0.006'
        param1 = '8'
        btable_string = 'btable_singleshell_%s.txt' % shell
        trk_output = '%s_sgsh_%s' % (trk_output, shell)
    elif modality in ('dsi', ):
        stem = 'data_raw_disco_eddy-vol_moco_clean_ordered'
        methodnr = '0'
        methodname = 'dsi'
        param0 = '17'
        param1 = None
        btable_string = 'dsi515_b_table.txt'
    elif modality in ('dti', ):
        #stem = 'dti_unprocessed'
        stem = 'dti'
        methodnr = 1
        methodname = 'dti'
        param0 = None
        param1 = None
        btable_string = 'btable.txt'
    else:
        sys.stderr.write('Unrecognized modality %s\n' % modality)

    if preproc_stem is not None:
        stem = preproc_stem

    try:
        os.mkdir(preproc_dir)
    except OSError:
        pass

    if os.path.exists(os.path.join(preproc_dir, '%s.nii' % stem)):
        gzip_cmd = ('mri_convert %s %s' %
                    (os.path.join(preproc_dir, '%s.nii' % stem),
                     os.path.join(preproc_dir, '%s.nii.gz' % stem)))
        print gzip_cmd
        m = subprocess.call(gzip_cmd, shell=True)

    if modality in ('qball', 'qbi', 'singleshell', 'sgsh'):
        src_cmd = (
            'dsi_studio --action=src --source=%s --b_table=%s --output=%s' %
            (os.path.join(preproc_dir, '%s.nii.gz' % stem),
             os.path.join(preproc_dir, btable_string),
             os.path.join(preproc_dir, '%s.src.gz' % stem)))
    elif modality in ('dsi', ):
        src_cmd = (
            'dsi_studio --action=src --source=%s --b_table=%s --output=%s' %
            (os.path.join(preproc_dir, '%s.nii.gz' % stem),
             os.path.join(subjdir, btable_string),
             os.path.join(preproc_dir, '%s.src.gz' % stem)))
    elif modality in ('dti', ):
        src_cmd = (
            'dsi_studio --action=src --source=%s --output=%s --b_table=%s' %
            (os.path.join(preproc_dir, 'dti.nii.gz'),
             os.path.join(preproc_dir, '%s.src.gz' % stem),
             os.path.join(preproc_dir, btable_string)))
    print src_cmd
    p = subprocess.call(src_cmd, shell=True)

    rec_cmd = ('dsi_studio --action=rec --source=%s --thread=10 --method=%s '
               '--num_fiber=8 --odf_order=8 --record_odf=1 --mask=%s' %
               (os.path.join(preproc_dir, '%s.src.gz' % stem), methodnr,
                os.path.join(dset_dir, 'wm_b0.nii.gz')))
    if param0 is not None:
        rec_cmd += ' --param0=%s' % param0
    if param1 is not None:
        rec_cmd += ' --param1=%s' % param1
    print rec_cmd
    q = subprocess.call(rec_cmd, shell=True)

    trk_cmd = (
        'dsi_studio --action=trk --source=%s --fiber_count=%i '
        '--output=%s' % (
            os.path.join(
                preproc_dir, '%s.src.gz.%s%s.%sfib.gz' %
                (stem, '' if methodname == 'dti' else 'odf8.f8rec.',
                 methodname,
                 reduce(lambda x, y: x + y, [
                     'sh%s.' % p if i == 0 else '%s.' % p
                     for i, p in enumerate(
                         (param0, param1)[::-1]) if p is not None
                 ], ''))),
            nr_fibers,
            #os.path.join(subjdir,subj,'%s.trk'%trk_output),
            os.path.join(dset_dir, 'tracks', '%s.%s' % (trk_output, '%s'))))
    print trk_cmd
    r1 = subprocess.call(trk_cmd % 'txt', shell=True)
    r2 = subprocess.call(trk_cmd % 'trk', shell=True)

    for parc in ('laus250', ):

        ana_cmd = (
            'dsi_studio --action=ana --source=%s --tract=%s '
            '--export=connectivity --roi=%s' % (
                os.path.join(
                    preproc_dir, '%s.src.gz.%s%s.%sfib.gz' %
                    (stem, '' if methodname == 'dti' else 'odf8.f8rec.',
                     methodname,
                     reduce(lambda x, y: x + y, [
                         'sh%s.' % p if i == 0 else '%s.' % p
                         for i, p in enumerate(
                             (param0, param1)[::-1]) if p is not None
                     ], ''))),
                #os.path.join(subjdir,subj,'%s.trk'%trk_output),
                os.path.join(dset_dir, 'tracks', '%s.txt' % trk_output),
                os.path.join(masks_dir, 'bigmask_%s_wmreg.nii.gz' % parc)))
        print ana_cmd
        s = subprocess.call(ana_cmd, shell=True)

        #==================================================================
        #put the resulting matrix back in alphabetical order

        from scipy import io as sio
        import numpy as np

        #fname=os.path.join(subjdir,subj, '%s.trk.connectivity.mat'%trk_output)
        fname = os.path.join(dset_dir, 'tracks',
                             '%s.txt.connectivity.mat' % trk_output)

        adj = sio.loadmat(fname)
        labnam = adj['name']
        med_length = adj['tract_median_length']

        adj = adj['connectivity']

        adj = adj / med_length
        adj[np.isnan(adj)] = 0

        labnam = ''.join(map(chr, labnam.flat)).split('\n')[:-1]

        stem = len(os.path.commonprefix(labnam))
        ord = np.array(map(lambda x: int(x[stem:]) - 1, labnam))
        #dsistudio indexing to python indexing

        keys = {}
        for i, k in enumerate(ord):
            keys.update({k: i})
        ord = map(keys.get, xrange(len(ord)))

        try:
            adj = adj[np.ix_(ord, ord)]
        except IndexError:
            print ord
            raise IndexError('The mask has too few ROIs')
        np.save(
            os.path.join(dset_dir, 'tracks',
                         '%s_alph_normed_%s.npy' % (trk_output, parc)), adj)
Пример #53
0
def Entropy_Thompson(data, hyperparameters, cz, cy, B=50, N=10, ploton=False):

    x, z, y = data
    az, bz, lz, ay, by, theta, lyx, lyz = hyperparameters
    top = np.argsort(y)[-N:]
    n = x.shape[0]
    Dx = euclidean_distances(x.reshape(-1, 1), x.reshape(-1, 1), squared=True)
    SIG_z = az**2 * np.exp(-Dx / (2 * lz**2)) + bz**2 * np.identity(n)
    Dz = euclidean_distances(z.reshape(-1, 1), z.reshape(-1, 1), squared=True)
    SIG_y = ay**2 * np.exp(-Dx * lyx**2 / 2 -
                           Dz * lyz**2 / 2) + by**2 * np.identity(n)
    """ samples from posterior """
    def samples(tt, tu, uu, nz, nzy, ztttu, ytt):
        mu_cz = np.matmul(
            SIG_z[np.ix_(uu, tt + tu)],
            np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], ztttu))
        SIG_cz = SIG_z[np.ix_(uu, uu)] - np.matmul(
            SIG_z[np.ix_(uu, tt + tu)],
            np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[np.ix_(
                tt + tu, uu)]))
        ZS = np.zeros((n, nz))
        ZS[tt + tu, :] = np.repeat(ztttu.reshape(-1, 1), nz, 1)
        ZS[uu, :] = np.random.multivariate_normal(mu_cz, SIG_cz, nz).T
        YS = np.zeros((n, nz * nzy))
        YS[tt, :] = np.repeat(ytt.reshape(-1, 1), nz * nzy, 1)
        for i in range(nz):
            sampled_SIG_y = ay**2 * np.exp(
                -Dx * lyx**2 / 2 - euclidean_distances(ZS[:, i].reshape(-1, 1),
                                                       ZS[:, i].reshape(-1, 1),
                                                       squared=True) * lyz**2 /
                2) + by**2 * np.identity(n)
            sampled_mu_cy = np.matmul(
                sampled_SIG_y[np.ix_(tu + uu, tt)],
                np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)], ytt))
            sampled_SIG_cy = sampled_SIG_y[np.ix_(
                tu + uu, tu + uu)] - np.matmul(
                    sampled_SIG_y[np.ix_(tu + uu, tt)],
                    np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)],
                                    sampled_SIG_y[np.ix_(tt, tu + uu)]))
            YS[tu + uu, i * nzy:(i + 1) * nzy] = np.random.multivariate_normal(
                sampled_mu_cy, sampled_SIG_cy, nzy).T
        return ZS, YS

    def entropy(x):
        I = [i for i in range(len(x)) if x[i] > 0 and x[i] < 1]
        H = -np.multiply(x[I], np.log(x[I]))
        h = sum(H)
        return h

    r = 1
    tt = list(range(r))
    tu = []
    uu = list(range(r, n))

    b = B
    History = []
    res = 5
    ZR = np.linspace(-2, 2, res)
    PR = norm.pdf(ZR)
    PR = PR / np.sum(PR)
    while b > cy:
        """ sample from posterior to estimate greedy-N acquisituion function """
        ZS, alpha = samples(tt, tu, uu, 2, 1, z[tt + tu], y[tt])
        if len(tu) > 0:
            """ select best candidates from TU and UU """
            itu = tu[np.argmax(alpha[tu, 0])]
            iuu = uu[np.argmax(alpha[uu, 1])]
            """ integrate over zuu to estimate profit of action z """
            mu_uuz = np.matmul(
                SIG_z[iuu, tt + tu],
                np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], z[tt + tu]))
            SIG_uuz = SIG_z[iuu, iuu] - np.matmul(
                SIG_z[iuu, tt + tu],
                np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[tt + tu,
                                                                       iuu]))
            mu_tuy = np.matmul(SIG_y[itu, tt],
                               np.linalg.solve(SIG_y[np.ix_(tt, tt)], y[tt]))
            SIG_tuy = SIG_y[itu, itu] - np.matmul(
                SIG_y[itu, tt],
                np.linalg.solve(SIG_y[np.ix_(tt, tt)], SIG_y[tt, iuu]))
            ZuuR = mu_uuz + ZR * SIG_uuz**0.5
            YtuR = mu_tuy + ZR * SIG_tuy**0.5
            uuc = uu.copy()
            uuc.remove(iuu)
            tuc = tu.copy()
            tuc.remove(itu)
            tuc.append(iuu)
            ttc = tt.copy()
            ttc.append(itu)
            mz = 5
            my = 100
            P = np.zeros((n, res, res))
            for i in range(res):
                for j in range(res):
                    ZS, YS = samples(
                        ttc, tuc, uuc, mz, 100,
                        np.concatenate((z[tt + tu], ZuuR[i].reshape(1))),
                        np.concatenate((y[tt], YtuR[j].reshape(1))))
                    for k in range(mz * my):
                        I = [np.argpartition(YS[:, k], -N)[-N:]]
                        P[I, i, j] += 1
            P = P / (mz * my)
            p = np.zeros(n)
            for i in range(res):
                for j in range(res):
                    p = p + PR[j] * PR[i] * P[:, i, j]
            H = entropy(p)
            Hz = np.zeros(res)
            for i in range(res):
                p = np.zeros(n)
                for j in range(res):
                    p = p + PR[j] * P[:, i, j]
                Hz[i] = entropy(p)
            Hy = np.zeros(res)
            for j in range(res):
                p = np.zeros(n)
                for i in range(res):
                    p = p + PR[i] * P[:, i, j]
                Hy[j] = entropy(p)

            DEz = np.dot(H - Hz, PR)
            DEy = np.dot(H - Hy, PR)

            if ploton:
                plt.figure(figsize=(15, 5))
                plt.subplot(121)
                plt.plot(x[top], z[top], '.', color='red')
                plt.plot(x[tu], z[tu], 's', color='black')
                plt.plot(x[tt], z[tt], 'd', color='red')
                plt.plot([x[iuu], x[iuu]], [np.min(z), np.max(z)], color='red')
                plt.plot(x[iuu], z[iuu], 'x', color='red')
                plt.plot(x[itu], z[itu], 'x', markersize=25, color='red')
                plt.scatter(x, z, 10, y)
                plt.xlabel('x')
                plt.ylabel('z')
                plt.subplot(122)
                plt.plot(ZuuR, Hz, label='z')
                plt.plot([ZuuR[0], ZuuR[-1]], [H, H])
                plt.plot(YtuR, Hy, label='y')
                plt.plot([YtuR[0], YtuR[-1]], [H, H])
                plt.legend()
                plt.xlabel('z/y')
                plt.ylabel('expected entropy')
                plt.show()
            """ reward/cost ratio of different actions """
            alphay = DEy / cy
            alphaz = DEz / cz
            if alphay > alphaz:
                tu.remove(itu)
                tt.append(itu)
                b = b - cy
                History.append([itu, 'y'])
            else:
                uu.remove(iuu)
                tu.append(iuu)
                b = b - cz
                History.append([iuu, 'z'])
        else:
            iuu = uu[np.argmax(alpha[uu, 0])]
            uu.remove(iuu)
            tu.append(iuu)
            b = b - cz
            History.append([iuu, 'z'])
        if ploton:
            plt.figure(figsize=(15, 5))
            plt.subplot(121)
            plt.plot(x[top], z[top], '.', color='red')
            plt.plot(x[tu], z[tu], 's', color='black')
            plt.plot(x[tt], z[tt], 'd', color='red')
            if History[-1][1] == 'y':
                plt.plot(x[tt[-1]], z[tt[-1]], 'd', color='red', markersize=15)
            else:
                plt.plot(x[tu[-1]],
                         z[tu[-1]],
                         's',
                         color='black',
                         linewidth=3,
                         markersize=15)
            plt.scatter(x, z, 10, y)
            plt.xlabel('x')
            plt.ylabel('z')
            plt.subplot(122)
            if History[-1][1] == 'y':
                plt.plot(x[tt[-1]], z[tt[-1]], 'd', color='red', markersize=15)
            else:
                plt.plot(x[tu[-1]],
                         z[tu[-1]],
                         's',
                         color='black',
                         markersize=15)
            plt.scatter(x[tt], z[tt], 50, y[tt])
            plt.plot(x[tu], z[tu], 's', color='black')
            plt.plot(x[uu], np.ones(len(uu)) * np.min(z), 'x')
            plt.xlabel('x')
            plt.ylabel('z')
            plt.show()
            print(b)

    ZS, YS = samples(tt, tu, uu, 10, 100, z[tt + tu], y[tt])
    P = np.zeros(n)
    for i in range(1000):
        I = [np.argpartition(YS[:, i], -N)[-N:]]
        P[I] += 1
    P = P / 1000
    H = entropy(P)
    History.append([H, 'final_entropy'])

    return History
Пример #54
0
 def F(self, x: np.ndarray, Ts: float) -> np.ndarray:
     F = self._F_mat
     F[np.ix_(self._all_idx, self._all_idx)] = F_CT(x[self._all_idx], Ts)
     return F.copy()
Пример #55
0
"""

import os
import pickle
import pygsp
import numpy as np
from scipy import sparse as sp

os.chdir('..')

# One month graph construction

data = pickle.load(open('temporal_citation/1992-02', 'rb'))
auth_idx = data['active_authors'].nonzero()[0]
W = data['W'][np.ix_(auth_idx, auth_idx)]
G = pygsp.graphs.Graph(W)
G.set_coords(iterations=10)
G.plot(show_edges=True)

# Aggregation of several months

N = 366572
W = sp.csr_matrix((N, N))
# active_authors = sp.csr_matrix((N, 1), dtype='bool')

from_year = 2002
to_year = 2004
from_month = 01
to_month = 12
Пример #56
0
 def samples(tt, tu, uu, nz, nzy, ztttu, ytt):
     mu_cz = np.matmul(
         SIG_z[np.ix_(uu, tt + tu)],
         np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], ztttu))
     SIG_cz = SIG_z[np.ix_(uu, uu)] - np.matmul(
         SIG_z[np.ix_(uu, tt + tu)],
         np.linalg.solve(SIG_z[np.ix_(tt + tu, tt + tu)], SIG_z[np.ix_(
             tt + tu, uu)]))
     ZS = np.zeros((n, nz))
     ZS[tt + tu, :] = np.repeat(ztttu.reshape(-1, 1), nz, 1)
     ZS[uu, :] = np.random.multivariate_normal(mu_cz, SIG_cz, nz).T
     YS = np.zeros((n, nz * nzy))
     YS[tt, :] = np.repeat(ytt.reshape(-1, 1), nz * nzy, 1)
     for i in range(nz):
         sampled_SIG_y = ay**2 * np.exp(
             -Dx * lyx**2 / 2 - euclidean_distances(ZS[:, i].reshape(-1, 1),
                                                    ZS[:, i].reshape(-1, 1),
                                                    squared=True) * lyz**2 /
             2) + by**2 * np.identity(n)
         sampled_mu_cy = np.matmul(
             sampled_SIG_y[np.ix_(tu + uu, tt)],
             np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)], ytt))
         sampled_SIG_cy = sampled_SIG_y[np.ix_(
             tu + uu, tu + uu)] - np.matmul(
                 sampled_SIG_y[np.ix_(tu + uu, tt)],
                 np.linalg.solve(sampled_SIG_y[np.ix_(tt, tt)],
                                 sampled_SIG_y[np.ix_(tt, tu + uu)]))
         YS[tu + uu, i * nzy:(i + 1) * nzy] = np.random.multivariate_normal(
             sampled_mu_cy, sampled_SIG_cy, nzy).T
     return ZS, YS
Пример #57
0
def fit_elasticnet(model,
                   method="coord_descent",
                   maxiter=100,
                   alpha=0.,
                   L1_wt=1.,
                   start_params=None,
                   cnvrg_tol=1e-7,
                   zero_tol=1e-8,
                   refit=False,
                   check_step=True,
                   loglike_kwds=None,
                   score_kwds=None,
                   hess_kwds=None):
    """
    Return an elastic net regularized fit to a regression model.

    Parameters
    ----------
    model : model object
        A statsmodels object implementing ``loglike``, ``score``, and
        ``hessian``.
    method :
        Only the coordinate descent algorithm is implemented.
    maxiter : int
        The maximum number of iteration cycles (an iteration cycle
        involves running coordinate descent on all variables).
    alpha : scalar or array_like
        The penalty weight.  If a scalar, the same penalty weight
        applies to all variables in the model.  If a vector, it
        must have the same length as `params`, and contains a
        penalty weight for each coefficient.
    L1_wt : scalar
        The fraction of the penalty given to the L1 penalty term.
        Must be between 0 and 1 (inclusive).  If 0, the fit is
        a ridge fit, if 1 it is a lasso fit.
    start_params : array_like
        Starting values for `params`.
    cnvrg_tol : scalar
        If `params` changes by less than this amount (in sup-norm)
        in one iteration cycle, the algorithm terminates with
        convergence.
    zero_tol : scalar
        Any estimated coefficient smaller than this value is
        replaced with zero.
    refit : bool
        If True, the model is refit using only the variables that have
        non-zero coefficients in the regularized fit.  The refitted
        model is not regularized.
    check_step : bool
        If True, confirm that the first step is an improvement and search
        further if it is not.
    loglike_kwds : dict-like or None
        Keyword arguments for the log-likelihood function.
    score_kwds : dict-like or None
        Keyword arguments for the score function.
    hess_kwds : dict-like or None
        Keyword arguments for the Hessian function.

    Returns
    -------
    A results object.

    Notes
    -----
    The ``elastic net`` penalty is a combination of L1 and L2
    penalties.

    The function that is minimized is:

    -loglike/n + alpha*((1-L1_wt)*|params|_2^2/2 + L1_wt*|params|_1)

    where |*|_1 and |*|_2 are the L1 and L2 norms.

    The computational approach used here is to obtain a quadratic
    approximation to the smooth part of the target function:

    -loglike/n + alpha*(1-L1_wt)*|params|_2^2/2

    then repeatedly optimize the L1 penalized version of this function
    along coordinate axes.
    """

    k_exog = model.exog.shape[1]

    loglike_kwds = {} if loglike_kwds is None else loglike_kwds
    score_kwds = {} if score_kwds is None else score_kwds
    hess_kwds = {} if hess_kwds is None else hess_kwds

    if np.isscalar(alpha):
        alpha = alpha * np.ones(k_exog)

    # Define starting params
    if start_params is None:
        params = np.zeros(k_exog)
    else:
        params = start_params.copy()

    btol = 1e-4
    params_zero = np.zeros(len(params), dtype=bool)

    init_args = model._get_init_kwds()
    # we do not need a copy of init_args b/c get_init_kwds provides new dict
    init_args['hasconst'] = False
    model_offset = init_args.pop('offset', None)
    if 'exposure' in init_args and init_args['exposure'] is not None:
        if model_offset is None:
            model_offset = np.log(init_args.pop('exposure'))
        else:
            model_offset += np.log(init_args.pop('exposure'))

    fgh_list = [
        _gen_npfuncs(k, L1_wt, alpha, loglike_kwds, score_kwds, hess_kwds)
        for k in range(k_exog)
    ]

    for itr in range(maxiter):

        # Sweep through the parameters
        params_save = params.copy()
        for k in range(k_exog):

            # Under the active set method, if a parameter becomes
            # zero we do not try to change it again.
            # TODO : give the user the option to switch this off
            if params_zero[k]:
                continue

            # Set the offset to account for the variables that are
            # being held fixed in the current coordinate
            # optimization.
            params0 = params.copy()
            params0[k] = 0
            offset = np.dot(model.exog, params0)
            if model_offset is not None:
                offset += model_offset

            # Create a one-variable model for optimization.
            model_1var = model.__class__(model.endog,
                                         model.exog[:, k],
                                         offset=offset,
                                         **init_args)

            # Do the one-dimensional optimization.
            func, grad, hess = fgh_list[k]
            params[k] = _opt_1d(func,
                                grad,
                                hess,
                                model_1var,
                                params[k],
                                alpha[k] * L1_wt,
                                tol=btol,
                                check_step=check_step)

            # Update the active set
            if itr > 0 and np.abs(params[k]) < zero_tol:
                params_zero[k] = True
                params[k] = 0.

        # Check for convergence
        pchange = np.max(np.abs(params - params_save))
        if pchange < cnvrg_tol:
            break

    # Set approximate zero coefficients to be exactly zero
    params[np.abs(params) < zero_tol] = 0

    if not refit:
        results = RegularizedResults(model, params)
        return RegularizedResultsWrapper(results)

    # Fit the reduced model to get standard errors and other
    # post-estimation results.
    ii = np.flatnonzero(params)
    cov = np.zeros((k_exog, k_exog))
    init_args = dict([(k, getattr(model, k, None)) for k in model._init_keys])
    if len(ii) > 0:
        model1 = model.__class__(model.endog, model.exog[:, ii], **init_args)
        rslt = model1.fit()
        params[ii] = rslt.params
        cov[np.ix_(ii, ii)] = rslt.normalized_cov_params
    else:
        # Hack: no variables were selected but we need to run fit in
        # order to get the correct results class.  So just fit a model
        # with one variable.
        model1 = model.__class__(model.endog, model.exog[:, 0], **init_args)
        rslt = model1.fit(maxiter=0)

    # fit may return a results or a results wrapper
    if issubclass(rslt.__class__, wrap.ResultsWrapper):
        klass = rslt._results.__class__
    else:
        klass = rslt.__class__

    # Not all models have a scale
    if hasattr(rslt, 'scale'):
        scale = rslt.scale
    else:
        scale = 1.

    # The degrees of freedom should reflect the number of parameters
    # in the refit model, not including the zeros that are displayed
    # to indicate which variables were dropped.  See issue #1723 for
    # discussion about setting df parameters in model and results
    # classes.
    p, q = model.df_model, model.df_resid
    model.df_model = len(ii)
    model.df_resid = model.nobs - model.df_model

    # Assuming a standard signature for creating results classes.
    refit = klass(model, params, cov, scale=scale)
    refit.regularized = True
    refit.method = method
    refit.fit_history = {'iteration': itr + 1}

    # Restore df in model class, see issue #1723 for discussion.
    model.df_model, model.df_resid = p, q

    return refit
Пример #58
0
    def create_new_grid(self):
        # GRID DEFINITION IN DEGREES 
        self.lon0_deg = lon0_deg
        self.lon1_deg = lon1_deg
        self.lat0_deg = lat0_deg
        self.lat1_deg = lat1_deg
        self.dlon_deg = dlon_deg
        self.dlat_deg = dlat_deg

        # GRID DEFINITION IN RADIANS
        self.lon0_rad    = self.lon0_deg/180*np.pi
        self.lon1_rad    = self.lon1_deg/180*np.pi
        self.lat0_rad    = self.lat0_deg/180*np.pi
        self.lat1_rad    = self.lat1_deg/180*np.pi
        self.dlon_rad_1D = self.dlon_deg/180*np.pi
        self.dlat_rad_1D = self.dlat_deg/180*np.pi

        # NUMBER OF GRID POINTS IN EACH DIMENSION
        self.nz = nz
        self.nzs = nzs
        self.nx = nx
        self.nxs = nxs
        self.ny = ny
        self.nys = nys
        self.nb = nb

        # INDEX ARRAYS
        self.i   = np.arange((self.nb),(self.nx +self.nb)) 
        self.i_s = np.arange((self.nb),(self.nxs+self.nb)) 
        self.j   = np.arange((self.nb),(self.ny +self.nb)) 
        self.js  = np.arange((self.nb),(self.nys+self.nb)) 
        self.k   = np.arange(self.nz) 
        self.ii,  self.jj  = np.ix_(self.i    ,self.j)
        self.iis, self.jjs = np.ix_(self.i_s  ,self.js)

        # 2D MATRIX OF LONGITUDES AND LATITUDES IN DEGREES
        self.lon_deg   = np.full(
                (self.nx +2*self.nb,self.ny+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.lat_deg   = np.full(
                (self.nx +2*self.nb,self.ny+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.lon_is_deg = np.full(
                (self.nxs+2*self.nb,self.ny+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.lat_is_deg = np.full(
                (self.nxs+2*self.nb,self.ny+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.lon_js_deg = np.full(
                (self.nx+2*self.nb,self.nys+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.lat_js_deg = np.full(
                (self.nx+2*self.nb,self.nys+2*self.nb,1),
                                    np.nan, dtype=wp)
        self.dlon_rad = np.full(
                (self.nx +2*self.nb,self.nys+2*self.nb,1),
                                    self.dlon_rad_1D, dtype=wp)
        self.dlat_rad = np.full(
                (self.nxs+2*self.nb,self.ny +2*self.nb,1),
                                    self.dlat_rad_1D, dtype=wp)

        for j in range(self.nb, self.ny+self.nb):
            self.lon_deg[self.ii,j,0] = (self.lon0_deg +
                                (self.ii-self.nb+0.5)*self.dlon_deg)
            self.lon_is_deg[self.iis,j,0] = (self.lon0_deg +
                                (self.iis-self.nb)*self.dlon_deg)
        for j_s in range(self.nb, self.nys+self.nb):
            self.lon_js_deg[self.ii,j_s,0] = (self.lon0_deg +
                                (self.ii-self.nb+0.5)*self.dlon_deg)
        for i in range(self.nb, self.nx+self.nb):
            self.lat_deg[i,self.jj,0] = (self.lat0_deg +
                                (self.jj-self.nb+0.5)*self.dlat_deg)
            self.lat_js_deg[i,self.jjs,0] = (self.lat0_deg +
                                (self.jjs-self.nb)*self.dlat_deg)
        for i_s in range(self.nb, self.nxs+self.nb):
            self.lat_is_deg[i_s,self.jj,0] = (self.lat0_deg +
                                (self.jj-self.nb+0.5)*self.dlat_deg)

        # 2D MATRIX OF LONGITUDES AND LATITUDES IN RADIANS
        self.lon_rad = self.lon_deg/180*np.pi
        self.lat_rad = self.lat_deg/180*np.pi
        self.lon_is_rad = self.lon_is_deg/180*np.pi
        self.lat_is_rad = self.lat_is_deg/180*np.pi
        self.lon_js_rad = self.lon_js_deg/180*np.pi
        self.lat_js_rad = self.lat_js_deg/180*np.pi

        # 2D MATRIX OF GRID SPACING IN METERS
        self.dx   = np.full(
                    (self.nx +2*self.nb,self.ny +2*self.nb,1),
                            np.nan, dtype=wp)
        self.dxjs = np.full(
                    (self.nx +2*self.nb,self.nys+2*self.nb,1),
                            np.nan, dtype=wp)
        self.dyis = np.full(
                    (self.nxs+2*self.nb,self.ny +2*self.nb,1),
                            np.nan, dtype=wp)

        self.dx[self.ii,self.jj,0] = (
                    np.cos( self.lat_rad[self.ii,self.jj,0] ) *
                                self.dlon_rad_1D*con_rE )
        self.dxjs[self.ii,self.jjs,0] = ( 
                    np.cos( self.lat_js_rad[self.ii,self.jjs,0] ) *
                                self.dlon_rad_1D*con_rE )
        self.dyis[self.iis,self.jj,0] = self.dlat_rad_1D*con_rE 
        self.dx   = self.exchange_BC(self.dx)
        self.dxjs = self.exchange_BC(self.dxjs)
        self.dyis = self.exchange_BC(self.dyis)
        self.dy = self.dlat_rad*con_rE

        self.A = np.full( (self.nx+2*self.nb,self.ny+2*self.nb, 1),
                            np.nan, dtype=wp)
        for i in self.i:
            for j in self.j:
                self.A[i,j,0] = lat_lon_recangle_area(self.lat_rad[i,j,0],
                        self.dlon_rad_1D, self.dlat_rad_1D)
        self.A = self.exchange_BC(self.A)
        print('fraction of earth covered: ' +
                str(np.round(np.sum(
                self.A[self.ii,self.jj,0])/(4*np.pi*con_rE**2),2)))

        # CORIOLIS FORCE
        self.corf    = np.full(
                        (self.nx +2*self.nb, self.ny +2*self.nb,1), 
                                np.nan, dtype=wp)
        self.corf_is = np.full(
                        (self.nxs+2*self.nb, self.ny +2*self.nb,1),
                                np.nan, dtype=wp)
        self.corf[self.ii,self.jj,0] = 2*con_omega*np.sin(
                                    self.lat_rad[self.ii,self.jj,0])
        self.corf_is[self.iis,self.jj,0] = 2*con_omega*np.sin(
                                    self.lat_is_rad[self.iis,self.jj,0])

        # SIGMA LEVELS
        self.level  = np.arange(0,self.nz )
        self.levels = np.arange(0,self.nzs)
        # will be set in load_profile of IO
        self.sigma_vb = np.full( self.nzs, np.nan, dtype=wp)
        self.dsigma   = np.full( self.nz , np.nan, dtype=wp)

        set_up_sigma_levels(self)
        self.dsigma       = np.expand_dims(
                            np.expand_dims(self.dsigma   , 0),0)
        self.sigma_vb     = np.expand_dims(
                            np.expand_dims(self.sigma_vb , 0),0)

        # TIME STEP
        mindx = np.nanmin(self.dx)
        self.CFL = CFL
        self.i_out_nth_hour = i_out_nth_hour
        self.nc_output_count = 0
        self.i_sim_n_days = i_sim_n_days
        self.dt = int(self.CFL*mindx/400)
        while i_out_nth_hour*3600 % self.dt > 0:
            self.dt -= 1
        self.nts = i_sim_n_days*3600*24/self.dt
        self.ts = 0
        self.i_out_nth_ts = int(self.i_out_nth_hour*3600 / self.dt)
        self.i_restart_nth_day = i_restart_nth_day
        self.i_restart_nth_ts = int(self.i_restart_nth_day*24/ \
                self.i_out_nth_hour*self.i_out_nth_ts)
        self.sim_time_sec = 0
        self.GMT = GMT_initialization

        # TIMER
        self.timer = Timer()

        # NUMERICAL DIFUSION
        self.UVFLX_dif_coef = np.zeros((1,1,nz), dtype=wp)
        self.POTT_dif_coef  = np.zeros((1,1,nz), dtype=wp)
        self.moist_dif_coef = np.zeros((1,1,nz), dtype=wp)
        # vert_reduce for 32 vertical levels:
        # 1.0 : uppermost level has 40% of lowest
        # 1.5 : uppermost level has 23% of lowest
        # 2.0 : uppermost level has 10% of lowest
        # 3.0 : uppermost level has 5% of lowest
        vert_reduce = .0
        self.UVFLX_dif_coef[0,0,self.k] = (UVFLX_dif_coef * 
                                np.exp(-vert_reduce*(nz-self.k-1)/nz))
        vert_reduce = 1.5
        self.POTT_dif_coef[0,0,self.k] = (POTT_dif_coef * 
                                np.exp(-vert_reduce*(nz-self.k-1)/nz))
        self.moist_dif_coef[0,0,self.k] = (moist_dif_coef * 
                                np.exp(-vert_reduce*(nz-self.k-1)/nz))

        self.copy_to_gpu()
Пример #59
0
    def load_factor(self,
                    file_name,
                    file_dir=None,
                    factor_names=None,
                    dates=None,
                    ids=None,
                    idx=None,
                    df=True,
                    h5_style=True):
        """" 读取单因子数据

        paramters:
        ==========
        idx: DataFrame
            返回data.align(idx, how='right')[0]
        dates: list or array-like
            dates终将把每个元素转换成int格式
        ids: list or array-like
            ids终将把每个元素转换成string格式
        df: bool
            是否已DataFrame格式返回. True by default.
        """

        file_shape = self.list_file_shape(file_name, file_dir)
        all_factors = self.list_file_factors(file_name, file_dir)
        all_ids = self.list_file_ids(file_name, file_dir)
        all_dates = self.list_file_dates(file_name, file_dir)
        if factor_names is None:
            factor_idx = (0, file_shape[2])
            factor_names = all_factors
        else:
            temp_factors = np.sort(factor_names)
            factor_idx = (all_factors.index(temp_factors[0]),
                          all_factors.index(temp_factors[-1]))
        if dates is None:
            date_idx = (0, file_shape[0])
            dates = all_dates
        else:
            dates = np.sort(
                Datetime2MatlabDatetime(pd.DatetimeIndex(dates).values))
            date_idx = (np.searchsorted(all_dates, dates[0]),
                        np.searchsorted(all_dates, dates[-1], side='right'))
        if ids is None:
            ids_idx = (0, file_shape[1])
            ids = all_ids
        else:
            ids = np.sort(ids).astype('int')
            ids_idx = (np.searchsorted(all_ids, ids[0]),
                       np.searchsorted(all_ids, ids[-1], side='right'))
        factor_data = self._read_raw(self.abs_factor_path(file_dir, file_name),
                                     date_idx, ids_idx, factor_idx)
        date_idx2 = np.in1d(all_dates[date_idx[0]:date_idx[-1]], dates)
        ids_idx2 = np.in1d(all_ids[ids_idx[0]:ids_idx[-1]], ids)
        factor_idx2 = np.in1d(all_factors[factor_idx[0]:factor_idx[-1]],
                              factor_names)
        factor_data = factor_data[np.ix_(date_idx2, ids_idx2, factor_idx2)]
        if df:
            if h5_style:
                ids_str_func = np.frompyfunc(intcode_to_tradecode, 1, 1)
                ids_str = ids_str_func(ids)
                datetimes = MatlabDatetime2Datetime(dates)
                df = self.arr3d2df(factor_data, datetimes, ids_str,
                                   factor_names)
            else:
                df = self.arr3d2df(factor_data, dates, ids, factor_names)
            if idx is not None:
                df = df.reindex(idx.index)
            return df
        return factor_data
Пример #60
0
def uspatial2spin(cc, moidx, mo_coeff):
    '''Convert the results of an unrestricted mean-field calculation to spin-orbital form.

    Spin-orbital ordering is determined by orbital energy without regard for spin.

    Returns:
        fock : (nso,nso) ndarray
            The Fock matrix in the basis of spin-orbitals
        so_coeff : (nao, nso) ndarray
            The matrix of spin-orbital coefficients in the AO basis
        spin : (nso,) ndarary
            The spin (0 or 1) of each spin-orbital
    '''

    dm = cc._scf.make_rdm1(cc.mo_coeff, cc.mo_occ)
    fockao = cc._scf.get_hcore() + cc._scf.get_veff(cc.mol, dm)
    fockab = list()
    for a in range(2):
        fockab.append(
            reduce(numpy.dot, (mo_coeff[a].T, fockao[a], mo_coeff[a])))

    nocc = cc.nocc
    nao = cc.mo_coeff[0].shape[0]
    nmo = cc.nmo
    nvir = nmo - nocc
    so_coeff = np.zeros((nao, nmo), dtype=mo_coeff[0].dtype)
    nocc_a = int(sum(cc.mo_occ[0] * moidx[0]))
    nocc_b = int(sum(cc.mo_occ[1] * moidx[1]))
    nmo_a = fockab[0].shape[0]
    nmo_b = fockab[1].shape[0]
    nvir_a = nmo_a - nocc_a
    nvir_b = nmo_b - nocc_b
    oa = range(0, nocc_a)
    ob = range(nocc_a, nocc)
    va = range(nocc, nocc + nvir_a)
    vb = range(nocc + nvir_a, nmo)
    spin = np.zeros(nmo, dtype=int)
    spin[oa] = 0
    spin[ob] = 1
    spin[va] = 0
    spin[vb] = 1
    so_coeff[:, oa] = mo_coeff[0][:, :nocc_a]
    so_coeff[:, ob] = mo_coeff[1][:, :nocc_b]
    so_coeff[:, va] = mo_coeff[0][:, nocc_a:nmo_a]
    so_coeff[:, vb] = mo_coeff[1][:, nocc_b:nmo_b]

    fock = np.zeros((nmo, nmo), dtype=fockab[0].dtype)
    fock[np.ix_(oa, oa)] = fockab[0][:nocc_a, :nocc_a]
    fock[np.ix_(oa, va)] = fockab[0][:nocc_a, nocc_a:]
    fock[np.ix_(va, oa)] = fockab[0][nocc_a:, :nocc_a]
    fock[np.ix_(va, va)] = fockab[0][nocc_a:, nocc_a:]
    fock[np.ix_(ob, ob)] = fockab[1][:nocc_b, :nocc_b]
    fock[np.ix_(ob, vb)] = fockab[1][:nocc_b, nocc_b:]
    fock[np.ix_(vb, ob)] = fockab[1][nocc_b:, :nocc_b]
    fock[np.ix_(vb, vb)] = fockab[1][nocc_b:, nocc_b:]

    # Do not sort because it's different to the orbital ordering generated by
    # get_frozen_mask function in AO-direct vvvv contraction
    #    idxo = np.diagonal(fock[:nocc,:nocc]).argsort()
    #    idxv = nocc + np.diagonal(fock[nocc:,nocc:]).argsort()
    #    idx = np.concatenate((idxo,idxv))
    #    spin = spin[idx]
    #    so_coeff = so_coeff[:,idx]
    #    fock = fock[:, idx][idx]

    return fock, so_coeff, spin