def sparseScalarProductOfDot (A, B, C, out=None): ''' Returns A * np.dot(B, C), however it does so keeping in mind the sparsity of A, calculating values only where required. Params A - a sparse CSR matrix B - a dense matrix C - a dense matrix out - if specified, must be a sparse CSR matrix with identical non-zero pattern to A (i.e. same indices and indptr) Returns out_data, though note that this is the same parameter passed in and overwitten. ''' assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix" assert not np.isfortran(B), "B matrix is not stored in row-major form" assert not np.isfortran(C), "C matrix is not stored in row-major form" if out is None: out = A.copy() if A.dtype == np.float64: compiled.sparseScalarProductOfDot_f8(A.data, A.indices, A.indptr, B, C, out.data) elif A.dtype == np.float32: compiled.sparseScalarProductOfDot_f4(A.data, A.indices, A.indptr, B, C, out.data) else: _sparseScalarProductOfDot_py(A,B,C, out) return out
def generate_itp_pn(lut): ndim=lut.ndim-1 if ndim == 1: interpolator = interpolators.interpol_1pn elif ndim == 2: interpolator = interpolators.interpol_2pn elif ndim == 3: interpolator = interpolators.interpol_3pn elif ndim == 4: interpolator = interpolators.interpol_4pn elif ndim == 5: interpolator = interpolators.interpol_5pn elif ndim == 6: interpolator = interpolators.interpol_6pn elif ndim >= 7: interpolator = interpolators.interpol_npn else: print 'Not implemented' return if ndim <= 6: if np.isfortran(lut): my_lut=lut else: my_lut=np.asfortranarray(lut) def function(wo): return interpolator(wo,my_lut) else: if np.isfortran(lut): flat_lut=np.asfortranarray(lut.reshape((-1,lut.shape[-1]) ,order='C')) else: flat_lut=np.asfortranarray(lut.reshape((-1,lut.shape[-1]), order='C')) shape=np.array(lut.shape) size=shape[:-1].prod() def function(wo): return interpolator(wo,flat_lut,shape[0:-1],shape[-1]) return function
def sparseScalarQuotientOfNormedDot (A, B, C, d, out=None): ''' Returns A / np.dot(B, C/D), however it does so keeping in mind the sparsity of A, calculating values only where required. Params A - a sparse CSR matrix B - a dense matrix C - a dense matrix d - a dense vector whose dimensionality matches the column-count of C out - if specified, must be a sparse CSR matrix with identical non-zero pattern to A (i.e. same indices and indptr) Returns out_data, though note that this is the same parameter passed in and overwitten. ''' assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix" assert not np.isfortran(B), "B matrix is not stored in row-major form" assert not np.isfortran(C), "C matrix is not stored in row-major form" if out is None: out = A.copy() if A.dtype == np.float64: compiled.sparseScalarQuotientOfNormedDot_f8(A.data, A.indices, A.indptr, B, C, d, out.data) elif A.dtype == np.float32: compiled.sparseScalarQuotientOfNormedDot_f4(A.data, A.indices, A.indptr, B, C, d, out.data) else: raise ValueError ("No implementation for the datatype " + str(A.dtype)) return out
def remove_adjacencies(labels, bwconn): test = np.zeros((2,2), dtype = np.uint32) if type(labels) != type(test): raise Exception('In remove_adjacencies, labels is not a *NumPy* array') if len(labels.shape) != 3: raise Exception('In remove_adjacencies, labels is not 3 dimensional') if not labels.flags.contiguous or np.isfortran(labels): raise Exception('In remove_adjacencies, labels not contiguous or not C-order') if labels.dtype != test.dtype: raise Exception('In remove_adjacencies, labels not uint32') testB=np.zeros((2,2),dtype=np.bool) if type(bwconn) != type(testB): raise Exception( 'In remove_adjacencies, bwconn is not *NumPy* array') if len(bwconn.shape) != 3: raise Exception( 'In remove_adjacencies, bwconn is not 3 dimensional') if not bwconn.flags.contiguous or np.isfortran(bwconn): raise Exception( 'In remove_adjacencies, bwconn not C-order contiguous') if bwconn.dtype != testB.dtype: raise Exception( 'In remove_adjacencies, bwconn not correct data type') sz = [x+2 for x in labels.shape] # need border for neighborhoods around the edge voxels lbls = np.zeros(sz, dtype=labels.dtype); lbls[1:-1,1:-1,1:-1] = labels _pyCext.remove_adjacencies(lbls, bwconn) return lbls[1:-1,1:-1,1:-1]
def label_affinities(affinities, labels, nextlabel, threshold): test=np.zeros((2,2)); testI=np.zeros((2,2),dtype=np.uint32) if type(affinities) != type(test): raise Exception( 'In label_affinities, affinities is not *NumPy* array') if len(affinities.shape) != 4: raise Exception( 'In label_affinities, affinities shape not 4 dimensional') if affinities.shape[3] != 3: raise Exception( 'In label_affinities, affinities not 3 dimensional') if not affinities.flags.contiguous or np.isfortran(affinities): raise Exception( 'In label_affinities, affinities not C-order contiguous') if affinities.dtype != np.single: raise Exception( 'In label_affinities, affinities not single floats') if type(labels) != type(testI): raise Exception( 'In label_affinities, labels is not *NumPy* array') if len(labels.shape) != 3: raise Exception( 'In label_affinities, labels is not 3 dimensional') if not labels.flags.contiguous or np.isfortran(labels): raise Exception( 'In label_affinities, labels not C-order contiguous') if labels.dtype != np.uint32: raise Exception( 'In label_affinities, labels not uint32') if type(threshold) != type(1.0): raise Exception( 'In label_affinities, threshold argument is not a float') if type(nextlabel) != type(1): raise Exception( 'In label_affinities, nextlabel argument is not a integer') if nextlabel < 1: raise Exception( 'In label_components, nextlabel argument is less than one') return _pyCext.label_affinities(affinities,labels,nextlabel,threshold)
def __init__(self, data, block_length=1, use_blocks=None, offsets=None): """ data can be a numpy array (in C order), a tuple of such arrays, or a list of such tuples or arrays """ self.files = list() if isinstance(data, list): #Several files for file in data: if isinstance(file, tuple): for d in file: assert(isinstance(d, np.ndarray) and not np.isfortran(d)) self.files.append(file) elif isinstance(data, tuple): #Just one file for d in data: assert(isinstance(d, np.ndarray) and d.ndim == 2 and not np.isfortran(d)) self.files.append(data) elif isinstance(data, np.ndarray): #One file with one kind of element only (not input-output) assert(isinstance(data, np.ndarray) and not np.isfortran(data)) self.files.append(tuple([data])) # Support for block datapoints self.block_length = block_length if block_length == 1: self.block_lengths = [np.int(1)] * self.get_arity() self.offsets = [np.int(0)] * self.get_arity() elif block_length > 1: self.block_lengths = [np.int(block_length) if ub else np.int(1) for ub in use_blocks] # np.asarray(dtype=np.int) and [np.int(x)] have elements with diff type. Careful! self.offsets = [np.int(off) for off in offsets] for ub, off in zip(use_blocks, offsets): if off != 0 and ub: raise Exception("Can't have both a block size greater than 1 and an offset.") else: raise Exception("Block size must be positive")
def generate_nan_itp(lut): ndim=lut.ndim if ndim == 1: interpolator = interpolators.interpol_nan_1 elif ndim == 2: interpolator = interpolators.interpol_nan_2 elif ndim == 3: interpolator = interpolators.interpol_nan_3 elif ndim == 4: interpolator = interpolators.interpol_nan_4 elif ndim == 5: interpolator = interpolators.interpol_5 elif ndim == 6: interpolator = interpolators.interpol_6 elif ndim == 7: interpolator = interpolators.interpol_7 elif ndim >= 8: interpolator = interpolators.interpol_n else: print 'Not implemented' return if ndim <= 7: if np.isfortran(lut): my_lut=lut.astype(float) else: my_lut=np.asfortranarray(lut.astype(float)) def function(wo): return interpolator(wo,my_lut) else: if np.isfortran(lut): flat_lut=lut.astype(float).ravel('C') else: flat_lut=lut.astype(float).T.ravel('F') shape=np.array(lut.shape) size=lut.size def function(wo): return interpolator(wo,flat_lut,shape) return function
def isfortran(self, array): """ Same as np.isfortran, but works with lists and tuples. """ if isinstance(array,(list,tuple)): return all([np.isfortran(val) for val in array]) else: return np.isfortran(array)
def add_dot(c, a, b): if use_blas and isinstance(c, numpy.ndarray): if numpy.isfortran(c.T): scipy.linalg.blas.dgemm(1., b.T, a.T, 1., c.T, overwrite_c=True) return c elif numpy.isfortran(c): scipy.linalg.blas.dgemm(1., a, b, 1., c, overwrite_c=True) return c c += numpy.dot(a, b) return c
def niiToArray(niiImg, c_contiguos=True, canonical=False): if canonical: rough_data = nib.as_closest_canonical(niiImg).get_data() else: rough_data = niiImg.get_data() print np.isfortran(rough_data) #if c_contiguos and np.isfortran(rough_data): # rough_data = rough_data.T if c_contiguos and not rough_data.flags['C_CONTIGUOUS']: rough_data = rough_data.T return rough_data
def test_ndim_indexing(aggregate_all, ndim, order, outsize=10): nindices = int(outsize ** ndim) outshape = tuple([outsize] * ndim) group_idx = np.random.randint(0, outsize, size=(ndim, nindices)) a = np.random.random(group_idx.shape[1]) res = aggregate_all(group_idx, a, size=outshape, order=order) if ndim > 1 and order == 'F': # 1d arrays always return False here assert np.isfortran(res) else: assert not np.isfortran(res) assert res.shape == outshape
def can_add(self, data): """Returns True iff data can be stored. This usually means it is of the same kind as previously stored arrays. """ # Obviously we could just store arbitrary arrays in some implementations (e.g. NPY) # But lets keep jagged contracts... template = self.template() if template is None: return True return (template.dtype >= data.dtype and data.shape[-1] == template.shape[-1] and np.isfortran(data) == np.isfortran(data))
def add_AB_to_C(A, B, C): """ Compute C += AB in-place. This uses gemm from whatever BLAS is available. MKL requires Fortran ordered arrays to avoid copies. Hence we work with transpositions of default c-style arrays. This function throws error if computation is not in-place. """ gemm = sl.get_blas_funcs("gemm", (A, B, C)) assert np.isfortran(C.T) and np.isfortran(A.T) and np.isfortran(B.T) D = gemm(1.0, B.T, A.T, beta=1, c=C.T, overwrite_c=1) assert D.base is C or D.base is C.base
def __init__(self, cl_ctx, h0, eta0, u0, v0): #Make sure that the data is single precision floating point if (not np.issubdtype(h0.dtype, np.float32) or np.isfortran(h0)): print "Converting H0" h0 = h0.astype(np.float32, order='C') if (not np.issubdtype(eta0.dtype, np.float32) or np.isfortran(eta0)): print "Converting Eta0" eta0 = eta0.astype(np.float32, order='C') if (not np.issubdtype(u0.dtype, np.float32) or np.isfortran(u0)): print "Converting U0" u0 = u0.astype(np.float32, order='C') if (not np.issubdtype(v0.dtype, np.float32) or np.isfortran(v0)): print "Converting V0" v0 = v0.astype(np.float32, order='C') self.ny, self.nx = h0.shape self.nx = self.nx - 2 # Ghost cells self.ny = self.ny - 2 assert(h0.shape == (self.ny+2, self.nx+2)) assert(eta0.shape == (self.ny+2, self.nx+2)) assert(u0.shape == (self.ny+2, self.nx+1)) assert(v0.shape == (self.ny+1, self.nx+2)) #Upload data to the device mf = cl.mem_flags self.h0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=h0) self.eta0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=eta0) self.eta1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=eta0) self.u0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=u0) self.u1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=u0) self.v0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=v0) self.v1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=v0) self.h0_pitch = np.int32(h0.shape[1]*4) self.eta0_pitch = np.int32(eta0.shape[1]*4) self.eta1_pitch = np.int32(eta0.shape[1]*4) self.u0_pitch = np.int32(u0.shape[1]*4) self.u1_pitch = np.int32(u0.shape[1]*4) self.v0_pitch = np.int32(v0.shape[1]*4) self.v1_pitch = np.int32(v0.shape[1]*4)
def test_fstecr_fstluk_order(self): rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) fname = '__rpnstd__testfile2__.fst' try: os.unlink(fname) except: pass funit = rmn.fstopenall(fname,rmn.FST_RW) (ig1,ig2,ig3,ig4) = rmn.cxgaig(self.grtyp,self.xg14[0],self.xg14[1],self.xg14[2],self.xg14[3]) (ni,nj) = (90,45) la = rmn.FST_RDE_META_DEFAULT.copy() la.update( {'nomvar' : 'LA', 'typvar' : 'C', 'ni' : ni, 'nj' : nj, 'nk' : 1, 'grtyp' : self.grtyp, 'ig1' : ig1, 'ig2' : ig2, 'ig3' : ig3, 'ig4' : ig4 } ) lo = la.copy() lo['nomvar'] = 'LO' #Note: For the order to be ok in the FSTD file, order='FORTRAN' is mandatory la['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN') lo['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN') for j in xrange(nj): for i in xrange(ni): lo['d'][i,j] = 100.+float(i) la['d'][i,j] = float(j) rmn.fstecr(funit,la['d'],la) rmn.fstecr(funit,lo) rmn.fstcloseall(funit) funit = rmn.fstopenall(fname,rmn.FST_RW) kla = rmn.fstinf(funit,nomvar='LA')['key'] la2 = rmn.fstluk(kla)#,rank=2) klo = rmn.fstinf(funit,nomvar='LO')['key'] lo2 = rmn.fstluk(klo)#,rank=2) rmn.fstcloseall(funit) try: os.unlink(fname) except: pass self.assertTrue(np.isfortran(la2['d'])) self.assertTrue(np.isfortran(lo2['d'])) self.assertFalse(np.any(np.fabs(la2['d'] - la['d']) > self.epsilon)) self.assertFalse(np.any(np.fabs(lo2['d'] - lo['d']) > self.epsilon))
def __init__(self, ndarray): self.metadata = _ndarray_meta(ndarray) self.size = ndarray.size * ndarray.itemsize # The following is guesswork # non contiguous fortran array (if ever such a thing exists) if numpy.isfortran(ndarray) and not ndarray.flags['F_CONTIGUOUS']: self.ndarray = numpy.asfortranarray(ndarray) # non contiguous C array elif not numpy.isfortran(ndarray) and not ndarray.flags['C_CONTIGUOUS']: self.ndarray = numpy.ascontiguousarray(ndarray) # contiguous fortran or C array, do nothing else: self.ndarray = ndarray self.ptr = self.ndarray.__array_interface__['data'][0]
def test_cmp_ndim(aggregate_cmp, ndim, order, outsize=100, decimal=14): nindices = int(outsize ** ndim) outshape = tuple([outsize] * ndim) group_idx = np.random.randint(0, outsize, size=(ndim, nindices)) a = np.random.random(group_idx.shape[1]) res = aggregate_cmp.func(group_idx, a, size=outshape, order=order) ref = aggregate_cmp.func_ref(group_idx, a, size=outshape, order=order) if ndim > 1 and order == 'F': # 1d arrays always return False here assert np.isfortran(res) else: assert not np.isfortran(res) assert res.shape == outshape np.testing.assert_array_almost_equal(res, ref, decimal=decimal)
def add_outer(a, x, y): """Add the outer product of x and y to a, possibly overwriting a. a = add_outer(a, x, y) is equivalent to a += numpy.outer(x, y).""" if use_blas and isinstance(a, numpy.ndarray): if numpy.isfortran(a.T): scipy.linalg.blas.dger(1., y, x, a=a.T, overwrite_a=1) return a elif numpy.isfortran(a): scipy.linalg.blas.dger(1., x, y, a=a, overwrite_a=1) return a # einsum is written in C and is faster than outer a += numpy.einsum('i,j->ij', x, y) return a
def verify_is_c_contiguous_and_is_not_fortran(x): if isinstance(x,CpuGpuArray): raise TypeError(type(x)) if np.isfortran(x): raise ValueError("Must be 'C' order") if not x.flags.c_contiguous: raise ValueError("Must be 'C'-contiguous")
def tolocal(self, comm=MPI.COMM_WORLD): """Scatter a global image into local ones.""" if hasattr(self, 'shape_global') and self.shape != self.shape_global or\ hasattr(self, 'comm') and self.comm.Get_size() > 1: raise ValueError('This array is not a global image.') order = 'f' if np.isfortran(self) else 'c' if hasattr(self, 'empty'): output = self.empty(self.shape, dtype=self.dtype, order=order, comm=comm) else: shape = split_shape(self.shape, comm) output = np.empty(shape, dtype=self.dtype, order=order) output = DistributedArray.__new__(DistributedArray, output, self.shape, comm) if hasattr(self, '__dict__'): for k,v in self.__dict__.items(): setattr(output, k, v) output.comm = comm s = split_work(self.shape[0], comm=comm) n = s.stop - s.start output[0:n] = self[s.start:s.stop] if n < output.shape[0]: output[n:] = 0 return output
def __check_image_c_order(img): """Raise if the image is not in FORTRAN memory order""" # Check array memory order if np.isfortran(img): # i.e. not C-ordered raise ValueError("cv_algorithms works only on C-ordered arrays") if not img.flags['C_CONTIGUOUS']: raise ValueError("cv_algorithms works only on contiguous arrays")
def set_array(dataobject, newarray): # Ensure we have Fortran ordered flat array to assign to image data. This # is ideally done without additional copies, but if C order we must copy. if np.isfortran(newarray): arr = newarray.reshape(-1, order='F') else: print 'Warning, array does not have Fortran order, making deep copy and fixing...' tmp = np.asfortranarray(newarray) arr = tmp.reshape(-1, order='F') print '...done.' # Set the extents (they may have changed). dataobject.SetExtent(0, newarray.shape[0] - 1, 0, newarray.shape[1] - 1, 0, newarray.shape[2] - 1) # Now replace the scalars array with the new array. vtkarray = np_s.numpy_to_vtk(arr) vtkarray.Association = dsa.ArrayAssociation.POINT do = dsa.WrapDataObject(dataobject) oldscalars = do.PointData.GetScalars() name = oldscalars.GetName() del oldscalars do.PointData.append(arr, name) do.PointData.SetActiveScalars(name)
def test_as_float_array(): # Test function for as_float_array X = np.ones((3, 10), dtype=np.int32) X = X + np.arange(10, dtype=np.int32) # Checks that the return type is ok X2 = as_float_array(X, copy=False) np.testing.assert_equal(X2.dtype, np.float32) # Another test X = X.astype(np.int64) X2 = as_float_array(X, copy=True) # Checking that the array wasn't overwritten assert_true(as_float_array(X, False) is not X) # Checking that the new type is ok np.testing.assert_equal(X2.dtype, np.float64) # Here, X is of the right type, it shouldn't be modified X = np.ones((3, 2), dtype=np.float32) assert_true(as_float_array(X, copy=False) is X) # Test that if X is fortran ordered it stays X = np.asfortranarray(X) assert_true(np.isfortran(as_float_array(X, copy=True))) # Test the copy parameter with some matrices matrices = [ np.matrix(np.arange(5)), sp.csc_matrix(np.arange(5)).toarray(), sparse_random_matrix(10, 10, density=0.10).toarray() ] for M in matrices: N = as_float_array(M, copy=True) N[0, 0] = np.nan assert_false(np.isnan(M).any())
def standard_traj(traj,particles=None,dimensions=None): if type(traj) not in [numpy.ndarray]: traj=numpy.array(traj,dtype=float,order='F') if len(traj.shape)==1: traj.resize(traj.shape[0],1,1) elif len(traj.shape)==2: if particles==None and dimensions==None: nn=str(traj.shape[-1]) print '# Error! The trajectory format should be [frames,particles,dimensions].' print '# Your input has the shape ['+str(traj.shape[0])+','+str(traj.shape[1])+']:' print '# '+nn+' particles with a reaction coordinate? or just a '+nn+'-D reaction coordinate?' print '# ' print '# Please, make use of the variables "particles" or/and "dimensions":' print '# traj:=[100 frames, 3 dimensions] --> "particles=1" or/and "dimensions=3"' print '# traj:=[100 frames, 8 particles] --> "particles=8" or/and "dimensions=1"' print '# ' return 0 elif particles==1 or dimensions>1: traj.resize(traj.shape[0],1,traj.shape[1]) elif particles>1 or dimensions==1: traj.resize(traj.shape[0],traj.shape[1],1) elif particles==1 and dimensions==1: traj.resize(traj.shape[0],traj.shape[1],1) if not numpy.isfortran(traj): traj=numpy.array(traj,dtype=float,order='F') return traj
def binary_neighbours(img): """ Takes a binary image and, for each pixel, computes which surrounding pixels are non-zero. Depdending on those pixels, bits in the uint8 output array are set or unset Parameters ========== img : numpy array-like A grayscale image that is assumed to be binary (every non-zero value is interpreted as 0). Usually this is a pre-thinned image. Returns ======= A uint8-type output array the same shape of img, where the following bits are set or unset, if the respective neighbouring pixel is set or unset. Bit index by position: 0 1 2 3 4 5 6 7 Note that for Numpy due to the coordinate system, the respective pixels can be accessed like this: [y-1,x-1] [y-1,x] [y-1,x+1] [y,x-1] [y,x] [y,x+1] [y+1,x-1] [y+1,x] [y+1,x+1] This is equivalent to the coordinate system when displaying the image using matplotlib imshow The positions in this matrix correspond to the bit number shown above, e.g. bit #4 is (1 << 4) ORed to the result. """ # Check if image has the correct type __check_image_grayscale_2d(img) img = force_c_order_contiguous(img) __check_array_uint8(img) height, width = img.shape # Allocate output array # uint32 is used so there is no overflow for large inputs out = np.zeros(img.shape, dtype=np.uint8, order="C") assert not np.isfortran(out) # Extract pointer to binary data srcptr = _ffi.cast("uint8_t*", img.ctypes.data) dstptr = _ffi.cast("uint8_t*", out.ctypes.data) rc = _libcv_algorithms.binary_neighbours(dstptr, srcptr, width, height) if rc != 0: raise ValueError("Internal error (return code {0}) in algorithm C code".format(rc)) return out
def convert_to_float32(data): """ Converts to C-style float 32 array suitable for the GPU/CUDA """ if (not np.issubdtype(data.dtype, np.float32) or np.isfortran(data)): return data.astype(np.float32, order='C') else: return data
def sparseScalarProductOfSafeLnDot (A, B, C, out=None, start=None, end=None): ''' Returns A * np.log(np.dot(B, C)), however it does so keeping in mind the sparsity of A, calculating values only where required. Moreover if any product of the dot is zero, it's replaced with the minimum non-zero value allowed by the datatype, to avoid NaNs Params A - a sparse CSR matrix B - a dense matrix C - a dense matrix out - if specified, must be a sparse CSR matrix with identical non-zero pattern to A (i.e. same indices and indptr) start - where to start indexing A end - where to stop indexing A Returns out_data, though note that this is the same parameter passed in and overwitten. ''' assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix" assert not np.isfortran(B), "B matrix is not stored in row-major form" assert not np.isfortran(C), "C matrix is not stored in row-major form" if start is None: start = 0 if end is None: end = A.shape[0] if out is None: out = A[start:end,:].copy() if (start, end) == (0, A.shape[0]) and A.dtype == np.float64: out = A.copy() oldWay = np.sum(compiled.sparseScalarProductOfSafeLnDot_f8_full( A.data, A.indices, A.indptr, B, C, out.data)) out = A.copy() newWay = np.sum(compiled.sparseScalarProductOfSafeLnDot_f8( A.data, A.indices, A.indptr, B, C, out.data, 0, A.shape[0])) #print ("The old way was " + str(oldWay) + " and the new way was " + str(newWay)) elif A.dtype == np.float64: compiled.sparseScalarProductOfSafeLnDot_f8(A.data, A.indices, A.indptr, B, C, out.data, start, end) elif A.dtype == np.float32: compiled.sparseScalarProductOfSafeLnDot_f4(A.data, A.indices, A.indptr, B, C, out.data, start, end) else: _sparseScalarProductOfSafeLnDot_py(A,B,C, out) return out
def info(Z): import sys import numpy as np endianness = {'=': 'native (%s)' % sys.byteorder, '<': 'little', '>': 'big', '|': 'not applicable'} print("------------------------------") print("Interface (item)") print(" shape: ", Z.shape) print(" dtype: ", Z.dtype) print(" length: ", len(Z)) print(" size: ", Z.size) print(" endianness: ", endianness[Z.dtype.byteorder]) if np.isfortran(Z): print(" order: ☐ C ☑ Fortran") else: print(" order: ☑ C ☐ Fortran") print("") print("Memory (byte)") print(" item size: ", Z.itemsize) print(" array size: ", Z.size*Z.itemsize) print(" strides: ", Z.strides) print("") print("Properties") if Z.flags["OWNDATA"]: print(" own data: ☑ Yes ☐ No") else: print(" own data: ☐ Yes ☑ No") if Z.flags["WRITEABLE"]: print(" writeable: ☑ Yes ☐ No") else: print(" writeable: ☐ Yes ☑ No") if np.isfortran(Z) and Z.flags["F_CONTIGUOUS"]: print(" contiguous: ☑ Yes ☐ No") elif not np.isfortran(Z) and Z.flags["C_CONTIGUOUS"]: print(" contiguous: ☑ Yes ☐ No") else: print(" contiguous: ☐ Yes ☑ No") if Z.flags["ALIGNED"]: print(" aligned: ☑ Yes ☐ No") else: print(" aligned: ☐ Yes ☑ No") print("------------------------------") print()
def type_components(labels, voxel_type, supervoxel_type, voxel_out_type, num_types=2): test=np.zeros((2,2),dtype=np.uint32) if type(labels) != type(test): raise Exception( 'In type_components, labels is not *NumPy* array') if len(labels.shape) != 3: raise Exception( 'In type_components, labels is not 3 dimensional') if not labels.flags.contiguous or np.isfortran(labels): raise Exception( 'In type_components, labels not C-order contiguous') if labels.dtype != np.uint32: raise Exception( 'In type_components, labels not uint32') if type(voxel_type) != type(test): raise Exception( 'In type_components, voxel_type is not *NumPy* array') if len(voxel_type.shape) != 3: raise Exception( 'In type_components, voxel_type is not 3 dimensional') if not voxel_type.flags.contiguous or np.isfortran(voxel_type): raise Exception( 'In type_components, voxel_type not C-order contiguous') if voxel_type.dtype != np.uint8: raise Exception( 'In type_components, voxel_type not uint8') if type(supervoxel_type) != type(test): raise Exception( 'In type_components, supervoxel_type is not *NumPy* array') if len(supervoxel_type.shape) != 1: raise Exception( 'In type_components, supervoxel_type is not array') if not supervoxel_type.flags.contiguous: raise Exception( 'In type_components, supervoxel_type is not contiguous') if supervoxel_type.dtype != np.uint8: raise Exception( 'In type_components, supervoxel_type not uint8') if not np.array_equal(labels.shape, voxel_type.shape): raise Exception( 'In type_components, labels and voxel_type not same shape') if type(voxel_out_type) != type(test): raise Exception( 'In type_components, voxel_out_type is not *NumPy* array') if len(voxel_out_type.shape) != 3: raise Exception( 'In type_components, voxel_out_type is not 3 dimensional') if not voxel_out_type.flags.contiguous or np.isfortran(voxel_out_type): raise Exception( 'In type_components, voxel_out_type not C-order contiguous') if voxel_out_type.dtype != np.uint8: raise Exception( 'In type_components, voxel_out_type not uint8') # max operation is slow, assumer caller did this correctly... segfault'able tho if wrong #if supervoxel_type.size != labels.max(): # raise Exception( 'In type_components, supervoxel_type size not equal to number of supervoxels in labels') if type(num_types) != type(1): raise Exception( 'In type_components, num_types argument is not an integer') if num_types < 2: raise Exception( 'In type_components, num_types < 2, need at least two types') return _pyCext.type_components(labels, voxel_type, supervoxel_type, voxel_out_type, num_types)
def verify_is_c_contiguous_and_is_not_fortran(cls,arr): """ arr is a numpy array. """ if np.isfortran(arr): raise ValueError("Must be 'C' order") if not arr.flags.c_contiguous: msg="Must be 'C'-contiguous. Consider passing arr.copy() instead." raise ValueError(msg)
def _initialize_with_attributes(self, coordinates=None, box=None, cell=None, timestep=None, integstep=None, step=None, time=None): self.coordinates = coordinates self.box = box self.cell = cell self.timestep = timestep self.integstep = integstep self.step = step self.time = time if box is not None: if box[0] is None: self.box = None if cell is not None: if cell[0] is None: self.cell = None if self.coordinates is not None: self.n_structures = self.coordinates.shape[0] self.n_atoms = self.coordinates.shape[1] ii = self.coordinates if ii is not None: if _np.isfortran(ii) == False: ii = _np.asfortranarray(ii) ii = self.box if ii is not None: if _np.isfortran(ii) == False: ii = _np.asfortranarray(ii) ii = self.cell if ii is not None: if _np.isfortran(ii) == False: ii = _np.asfortranarray(ii) ii = self.time if ii is not None: if _np.isfortran(ii) == False: ii = _np.asfortranarray(ii) ii = self.step if ii is not None: if _np.isfortran(ii) == False: ii = _np.asfortranarray(ii) if (self.cell is None) and (self.box is not None): self.box2cell() if (self.cell is not None) and (self.box is None): self.cell2box() if (self.box is not None): self.box2invbox() pass
def norm(a, ord=None): """ Matrix or vector norm. This function is able to return one of seven different matrix norms, or one of an infinite number of vector norms (described below), depending on the value of the ``ord`` parameter. Parameters ---------- a : (M,) or (M, N) array_like Input array. ord : {non-zero int, inf, -inf, 'fro'}, optional Order of the norm (see table under ``Notes``). inf means numpy's `inf` object. Returns ------- norm : float Norm of the matrix or vector. Notes ----- For values of ``ord <= 0``, the result is, strictly speaking, not a mathematical 'norm', but it may still be useful for various numerical purposes. The following norms can be calculated: ===== ============================ ========================== ord norm for matrices norm for vectors ===== ============================ ========================== None Frobenius norm 2-norm 'fro' Frobenius norm -- inf max(sum(abs(x), axis=1)) max(abs(x)) -inf min(sum(abs(x), axis=1)) min(abs(x)) 0 -- sum(x != 0) 1 max(sum(abs(x), axis=0)) as below -1 min(sum(abs(x), axis=0)) as below 2 2-norm (largest sing. value) as below -2 smallest singular value as below other -- sum(abs(x)**ord)**(1./ord) ===== ============================ ========================== The Frobenius norm is given by [1]_: :math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}` References ---------- .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*, Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15 Examples -------- >>> from scipy.linalg import norm >>> a = np.arange(9) - 4 >>> a array([-4, -3, -2, -1, 0, 1, 2, 3, 4]) >>> b = a.reshape((3, 3)) >>> b array([[-4, -3, -2], [-1, 0, 1], [ 2, 3, 4]]) >>> norm(a) 7.745966692414834 >>> norm(b) 7.745966692414834 >>> norm(b, 'fro') 7.745966692414834 >>> norm(a, np.inf) 4 >>> norm(b, np.inf) 9 >>> norm(a, -np.inf) 0 >>> norm(b, -np.inf) 2 >>> norm(a, 1) 20 >>> norm(b, 1) 7 >>> norm(a, -1) -4.6566128774142013e-010 >>> norm(b, -1) 6 >>> norm(a, 2) 7.745966692414834 >>> norm(b, 2) 7.3484692283495345 >>> norm(a, -2) nan >>> norm(b, -2) 1.8570331885190563e-016 >>> norm(a, 3) 5.8480354764257312 >>> norm(a, -3) nan """ # Differs from numpy only in non-finite handling and the use of blas. a = np.asarray_chkfinite(a) if a.dtype.char in 'fdFD': if ord in (None, 2) and (a.ndim == 1): # use blas for fast and stable euclidean norm nrm2 = get_blas_funcs('nrm2', dtype=a.dtype) return nrm2(a) if a.ndim == 2: # Use lapack for a couple fast matrix norms. # For some reason the *lange frobenius norm is slow. lange_args = None if ord == 1: if np.isfortran(a): lange_args = '1', a elif np.isfortran(a.T): lange_args = 'i', a.T elif ord == np.inf: if np.isfortran(a): lange_args = 'i', a elif np.isfortran(a.T): lange_args = '1', a.T if lange_args: lange = get_lapack_funcs('lange', dtype=a.dtype) return lange(*lange_args) return np.linalg.norm(a, ord=ord)
def _build_tree(X, y, is_classification, criterion, max_depth, min_split, min_density, max_features, random_state, n_classes, find_split, sample_mask=None, X_argsorted=None): """Build a tree by recursively partitioning the data.""" if max_depth <= 10: init_capacity = (2**(max_depth + 1)) - 1 else: init_capacity = 2047 # num nodes of tree with depth 10 tree = Tree(n_classes, init_capacity) # Recursively partition X def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent, is_left_child): # Count samples n_node_samples = sample_mask.sum() if n_node_samples == 0: raise ValueError("Attempting to find a split " "with an empty sample_mask") # Split samples if depth < max_depth and n_node_samples >= min_split: feature, threshold, best_error, init_error = find_split( X, y, X_argsorted, sample_mask, n_node_samples, max_features, criterion, random_state) else: feature = -1 # Value at this node current_y = y[sample_mask] if is_classification: value = np.zeros((n_classes, )) t = current_y.max() + 1 value[:t] = np.bincount(current_y.astype(np.int)) else: value = np.asarray(np.mean(current_y)) # Terminal node if feature == -1: # compute error at leaf error = _tree._error_at_leaf(y, sample_mask, criterion, n_node_samples) tree.add_leaf(parent, is_left_child, value, error, n_node_samples) # Internal node else: # Sample mask is too sparse? if n_node_samples / X.shape[0] <= min_density: X = X[sample_mask] X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) y = current_y sample_mask = np.ones((X.shape[0], ), dtype=np.bool) # Split and and recurse split = X[:, feature] <= threshold node_id = tree.add_split_node(parent, is_left_child, feature, threshold, best_error, init_error, n_node_samples, value) # left child recursion recursive_partition(X, X_argsorted, y, split & sample_mask, depth + 1, node_id, True) # right child recursion recursive_partition(X, X_argsorted, y, ~split & sample_mask, depth + 1, node_id, False) # Launch the construction if X.dtype != DTYPE or not np.isfortran(X): X = np.asanyarray(X, dtype=DTYPE, order="F") if y.dtype != DTYPE or not y.flags.contiguous: y = np.ascontiguousarray(y, dtype=DTYPE) if sample_mask is None: sample_mask = np.ones((X.shape[0], ), dtype=np.bool) if X_argsorted is None: X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False) tree.resize(tree.node_count) return tree
def _build_tree(X, y, criterion, max_depth, min_samples_split, min_samples_leaf, min_density, max_features, random_state, n_classes, find_split, sample_mask=None, X_argsorted=None, store_terminal_region=False): """Build a tree by recursively partitioning the data. """ if max_depth <= 10: init_capacity = (2**(max_depth + 1)) - 1 else: init_capacity = 2047 # num nodes of tree with depth 10 tree = Tree(n_classes, init_capacity) # Recursively partition X def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent, is_left_child, sample_indices): # Count samples n_node_samples = sample_mask.sum() if n_node_samples == 0: raise ValueError("Attempting to find a split " "with an empty sample_mask") # Split samples if depth < max_depth and n_node_samples >= min_samples_split \ and n_node_samples >= 2 * min_samples_leaf: feature, threshold, best_error, init_error = find_split( X, y, X_argsorted, sample_mask, n_node_samples, min_samples_leaf, max_features, criterion, random_state) else: feature = -1 init_error = _tree._error_at_leaf(y, sample_mask, criterion, n_node_samples) value = criterion.init_value() # Current node is leaf if feature == -1: node_id = tree.add_leaf(parent, is_left_child, value, init_error, n_node_samples) if store_terminal_region: # remember which samples ended up in this leaf tree.terminal_region[sample_indices[sample_mask]] = node_id # Current node is internal node (= split node) else: # Sample mask is too sparse? if n_node_samples / X.shape[0] <= min_density: X = X[sample_mask] sample_indices = sample_indices[sample_mask] X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) y = y[sample_mask] sample_mask = np.ones((X.shape[0], ), dtype=np.bool) # Split and and recurse split = X[:, feature] <= threshold node_id = tree.add_split_node(parent, is_left_child, feature, threshold, best_error, init_error, n_node_samples, value) # left child recursion recursive_partition(X, X_argsorted, y, np.logical_and(split, sample_mask), depth + 1, node_id, True, sample_indices) # right child recursion recursive_partition( X, X_argsorted, y, np.logical_and(np.logical_not(split), sample_mask), depth + 1, node_id, False, sample_indices) # setup auxiliary data structures and check input before # recursive partitioning if store_terminal_region: tree.terminal_region = np.empty((X.shape[0], ), dtype=np.int32) tree.terminal_region.fill(-1) if X.dtype != DTYPE or not np.isfortran(X): X = np.asanyarray(X, dtype=DTYPE, order="F") if y.dtype != DTYPE or not y.flags.contiguous: y = np.ascontiguousarray(y, dtype=DTYPE) if sample_mask is None: sample_mask = np.ones((X.shape[0], ), dtype=np.bool) if X_argsorted is None: X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) sample_indices = np.arange(X.shape[0]) # build the tree by recursive partitioning recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False, sample_indices) # compactify the tree data structure tree.resize(tree.node_count) return tree
def _fit(self, data): """Actual method calling the underlying fitting implementation.""" data_ord = ord('c' if np.isfortran(data) else 'r') data, c_data_ptr, data_ctype = self._to_cdata(data) if self.init == "random" or self.init == "k-means++": c_init = 1 else: c_init = 0 if self.init_data == "random": c_init_data = 0 elif self.init_data == "selectstrat": c_init_data = 1 elif self.init_data == "randomselect": c_init_data = 2 else: print(""" Unknown init_data "%s", should be "random", "selectstrat" or "randomselect". """ % self.init_data) sys.stdout.flush() return pred_centers = c_void_p(0) pred_labels = c_void_p(0) lib = self._load_lib() rows = np.shape(data)[0] cols = np.shape(data)[1] if self.double_precision == 0: status = lib.make_ptr_float_kmeans( 0, self.verbose, self.random_state, self._gpu_id, self.n_gpus, rows, cols, c_int(data_ord), self._n_clusters, self._max_iter, c_init, c_init_data, self.tol, c_data_ptr, None, pointer(pred_centers), pointer(pred_labels)) else: status = lib.make_ptr_double_kmeans( 0, self.verbose, self.random_state, self._gpu_id, self.n_gpus, rows, cols, c_int(data_ord), self._n_clusters, self._max_iter, c_init, c_init_data, self.tol, c_data_ptr, None, pointer(pred_centers), pointer(pred_labels)) if status: raise ValueError('KMeans failed in C++ library.') centroids = np.fromiter(cast(pred_centers, POINTER(data_ctype)), dtype=data_ctype, count=self._n_clusters * cols) centroids = np.reshape(centroids, (self._n_clusters, cols)) if np.isnan(centroids).any(): centroids = centroids[~np.isnan(centroids).any(axis=1)] self._print_verbose( 0, "Removed %d empty centroids" % (self._n_clusters - centroids.shape[0])) self._n_clusters = centroids.shape[0] self.cluster_centers_ = centroids labels = np.fromiter(cast(pred_labels, POINTER(c_int)), dtype=np.int32, count=rows) self.labels_ = np.reshape(labels, rows) return self.cluster_centers_, self.labels_
def elastic_net(predictors, target, balance, memlimit=None, largest=None, **kwargs): """ Raw-output wrapper for elastic net linear regression. """ # Mandatory parameters predictors = np.asanyarray(predictors) target = np.asanyarray(target) # Decide on largest allowable models for memory/convergence. memlimit = predictors.shape[1] if memlimit is None else memlimit # If largest isn't specified use memlimit. largest = memlimit if largest is None else largest if memlimit < largest: raise ValueError('Need largest <= memlimit') # Flags determining overwrite behavior overwrite_pred_ok = False overwrite_targ_ok = False thr = _DEFAULT_THRESH # Minimum change in largest coefficient weights = None # Relative weighting per observation case vp = None # Relative penalties per predictor (0 = no penalty) isd = True # Standardize input variables before proceeding? jd = np.zeros(1) # Predictors to exclude altogether from fitting ulam = None # User-specified lambda values flmin = _DEFAULT_FLMIN # Fraction of largest lambda at which to stop nlam = _DEFAULT_NLAM # The (maximum) number of lambdas to try. for keyword in kwargs: if keyword == 'overwrite_pred_ok': overwrite_pred_ok = kwargs[keyword] elif keyword == 'overwrite_targ_ok': overwrite_targ_ok = kwargs[keyword] elif keyword == 'threshold': thr = kwargs[keyword] elif keyword == 'weights': weights = np.asarray(kwargs[keyword]).copy() elif keyword == 'penalties': vp = kwargs[keyword].copy() elif keyword == 'standardize': isd = bool(kwargs[keyword]) elif keyword == 'exclude': # Add one since Fortran indices start at 1 exclude = (np.asarray(kwargs[keyword]) + 1).tolist() jd = np.array([len(exclude)] + exclude) elif keyword == 'lambdas': if 'flmin' in kwargs: raise ValueError("Can't specify both lambdas & flmin keywords") ulam = np.asarray(kwargs[keyword]) flmin = 2. # Pass flmin > 1.0 indicating to use the user-supplied. nlam = len(ulam) elif keyword == 'flmin': flmin = kwargs[keyword] ulam = None elif keyword == 'nlam': if 'lambdas' in kwargs: raise ValueError("Can't specify both lambdas & nlam keywords") nlam = kwargs[keyword] else: raise ValueError("Unknown keyword argument '%s'" % keyword) # If predictors is a Fortran contiguous array, it will be overwritten. # Decide whether we want this. If it's not Fortran contiguous it will # be copied into that form anyway so there's no chance of overwriting. if np.isfortran(predictors): if not overwrite_pred_ok: # Might as well make it F-ordered to avoid ANOTHER copy. predictors = predictors.copy(order='F') # target being a 1-dimensional array will usually be overwritten # with the standardized version unless we take steps to copy it. if not overwrite_targ_ok: target = target.copy() # Uniform weighting if no weights are specified. if weights is None: weights = np.ones(predictors.shape[0]) # Uniform penalties if none were specified. if vp is None: vp = np.ones(predictors.shape[1]) # Call the Fortran wrapper. lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr = \ _glmnet.elnet(balance, predictors, target, weights, jd, vp, memlimit, flmin, ulam, thr, nlam=nlam) # Check for errors, documented in glmnet.f. if jerr != 0: if jerr == 10000: raise ValueError('cannot have max(vp) < 0.0') elif jerr == 7777: raise ValueError('all used predictors have 0 variance') elif jerr < 7777: raise MemoryError('elnet() returned error code %d' % jerr) else: raise Exception('unknown error: %d' % jerr) return lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr
#a=(array[:]).reshape(224,224) # Unpacking the array from the vector shape imageTmp = np.reshape(array, (imageSize1, imageSize2), order='F') / 255.0 #imageTmp = np.reshape(array, (imageSize1, imageSize2), order='F') cv2.imshow('BOSS image', imageTmp) #cv2.waitKey(0) #imageTmp = imageTmp.astype(np.uint8) #imageTmp = skimage.exposure.rescale_intensity(imageTmp * 1.0, out_range=np.float32) im = Image.fromarray(imageTmp) #im.show() print(np.isfortran(imageTmp)) # #print(image.shape) image = np.zeros((imageSize1, imageSize2, 3), float, 'C') image = np.stack((imageTmp, ) * 3, -1) ''' image[:, :, 0] = imageTmp image[:, :, 1] = imageTmp image[:, :, 2] = imageTmp ''' ''' image[:, :, 1] = imageTmp image[:, :, 2] = imageTmp '''
def solver_group(X, y, alpha, grp_size, max_iter=10000, tol=1e-4, f_gap=10, K=5, use_acc=False, algo='bcd'): """Solve the GroupLasso with BCD/ISTA/FISTA, eventually with extrapolation. Groups are contiguous, of size grp_size. Objective: norm(y - Xw, ord=2)**2 / 2 + alpha * sum_g ||w_{[g]}||_2 Parameters: algo: string 'bcd', 'pgd', 'fista' alpha: strength of the group penalty """ is_sparse = sparse.issparse(X) n_features = X.shape[1] if n_features % grp_size != 0: raise ValueError("n_features is not a multiple of group size") n_groups = n_features // grp_size if not is_sparse and not np.isfortran(X): X = np.asfortranarray(X) last_K_w = np.zeros([K + 1, n_features]) U = np.zeros([K, n_features]) if algo in ('pgd', 'fista'): if is_sparse: L = power_method(X, max_iter=1000)**2 else: L = norm(X, ord=2)**2 lc = np.zeros(n_groups) for g in range(n_groups): X_g = X[:, g * grp_size:(g + 1) * grp_size] if is_sparse: gram = (X_g.T @ X_g).todense() lc[g] = norm(gram, ord=2) else: lc[g] = norm(X_g, ord=2)**2 w = np.zeros(n_features) if algo == 'fista': z = np.zeros(n_features) t_new = 1 R = y.copy() E = [] gaps = np.zeros(max_iter // f_gap) for it in range(max_iter): if it % f_gap == 0: if algo == 'fista': R = y - X @ w p_obj = primal_grp(R, w, alpha, grp_size) E.append(p_obj) theta = R / alpha d_norm_theta = np.max( norm((X.T @ theta).reshape(-1, grp_size), axis=1)) if d_norm_theta > 1.: theta /= d_norm_theta d_obj = dual_lasso(y, theta, alpha) gap = p_obj - d_obj print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" % (it, p_obj, d_obj, gap)) gaps[it // f_gap] = gap if gap < tol: print("Early exit") break if algo == 'bcd': if is_sparse: _bcd_sparse(X.data, X.indices, X.indptr, w, R, alpha, lc) else: _bcd(X, w, R, alpha, lc) elif algo == 'pgd': w[:] = BST_vec(w + 1. / L * X.T @ R, alpha / L, grp_size) R[:] = y - X @ w elif algo == 'fista': w_old = w.copy() w[:] = BST_vec(z - X.T @ (X @ z - y) / L, alpha / L, grp_size) t_old = t_new t_new = (1. + np.sqrt(1 + 4 * t_old**2)) / 2. z[:] = w + (t_old - 1.) / t_new * (w - w_old) else: raise ValueError("Unknown algo %s" % algo) if use_acc: if it < K + 1: last_K_w[it] = w else: for k in range(K): last_K_w[k] = last_K_w[k + 1] last_K_w[K - 1] = w for k in range(K): U[k] = last_K_w[k + 1] - last_K_w[k] C = np.dot(U, U.T) try: z = np.linalg.solve(C, np.ones(K)) c = z / z.sum() w_acc = np.sum(last_K_w[:-1] * c[:, None], axis=0) p_obj = primal_grp(R, w, alpha, grp_size) R_acc = y - X @ w_acc p_obj_acc = primal_grp(R_acc, w_acc, alpha, grp_size) if p_obj_acc < p_obj: w = w_acc R = R_acc except np.linalg.LinAlgError: print("----------Linalg error") return w, np.array(E), gaps[:it // f_gap + 1]
def sync(data, idx, aggregate=None, pad=True, axis=-1): """Synchronous aggregation of a multi-dimensional array between boundaries .. note:: In order to ensure total coverage, boundary points may be added to `idx`. If synchronizing a feature matrix against beat tracker output, ensure that frame index numbers are properly aligned and use the same hop length. Parameters ---------- data : np.ndarray multi-dimensional array of features idx : iterable of ints or slices Either an ordered array of boundary indices, or an iterable collection of slice objects. aggregate : function aggregation function (default: `np.mean`) pad : boolean If `True`, `idx` is padded to span the full range `[0, data.shape[axis]]` axis : int The axis along which to aggregate data Returns ------- data_sync : ndarray `data_sync` will have the same dimension as `data`, except that the `axis` coordinate will be reduced according to `idx`. For example, a 2-dimensional `data` with `axis=-1` should satisfy `data_sync[:, i] = aggregate(data[:, idx[i-1]:idx[i]], axis=-1)` Raises ------ ParameterError If the index set is not of consistent type (all slices or all integers) Examples -------- Beat-synchronous CQT spectra >>> y, sr = librosa.load(librosa.util.example_audio_file()) >>> tempo, beats = librosa.beat.beat_track(y=y, sr=sr, trim=False) >>> cqt = librosa.cqt(y=y, sr=sr) >>> beats = librosa.util.fix_frames(beats, x_max=cqt.shape[1]) By default, use mean aggregation >>> cqt_avg = librosa.util.sync(cqt, beats) Use median-aggregation instead of mean >>> cqt_med = librosa.util.sync(cqt, beats, ... aggregate=np.median) Or sub-beat synchronization >>> sub_beats = librosa.segment.subsegment(cqt, beats) >>> sub_beats = librosa.util.fix_frames(sub_beats, x_max=cqt.shape[1]) >>> cqt_med_sub = librosa.util.sync(cqt, sub_beats, aggregate=np.median) Plot the results >>> import matplotlib.pyplot as plt >>> beat_t = librosa.frames_to_time(beats, sr=sr) >>> subbeat_t = librosa.frames_to_time(sub_beats, sr=sr) >>> plt.figure() >>> plt.subplot(3, 1, 1) >>> librosa.display.specshow(librosa.logamplitude(cqt**2, ... ref_power=np.max), ... x_axis='time') >>> plt.colorbar(format='%+2.0f dB') >>> plt.title('CQT power, shape={}'.format(cqt.shape)) >>> plt.subplot(3, 1, 2) >>> librosa.display.specshow(librosa.logamplitude(cqt_med**2, ... ref_power=np.max), ... x_coords=beat_t, x_axis='time') >>> plt.colorbar(format='%+2.0f dB') >>> plt.title('Beat synchronous CQT power, ' ... 'shape={}'.format(cqt_med.shape)) >>> plt.subplot(3, 1, 3) >>> librosa.display.specshow(librosa.logamplitude(cqt_med_sub**2, ... ref_power=np.max), ... x_coords=subbeat_t, x_axis='time') >>> plt.colorbar(format='%+2.0f dB') >>> plt.title('Sub-beat synchronous CQT power, ' ... 'shape={}'.format(cqt_med_sub.shape)) >>> plt.tight_layout() """ if aggregate is None: aggregate = np.mean shape = list(data.shape) if np.all([isinstance(_, slice) for _ in idx]): slices = idx elif np.all([np.issubdtype(type(_), np.int) for _ in idx]): slices = index_to_slice(np.asarray(idx), 0, shape[axis], pad=pad) else: raise ParameterError('Invalid index set: {}'.format(idx)) agg_shape = list(shape) agg_shape[axis] = len(slices) data_agg = np.empty(agg_shape, order='F' if np.isfortran(data) else 'C', dtype=data.dtype) idx_in = [slice(None)] * data.ndim idx_agg = [slice(None)] * data_agg.ndim for (i, segment) in enumerate(slices): idx_in[axis] = segment idx_agg[axis] = i data_agg[idx_agg] = aggregate(data[idx_in], axis=axis) return data_agg
def apcg_enet(X, y, alpha, rho=0, max_iter=10000, tol=1e-4, f_gap=10, verbose=False): """Solve the Lasso with accelerated proximal coordinate gradient. Parameters ---------- X : {array_like, sparse matrix}, shape (n_samples, n_features) Design matrix y : ndarray, shape (n_samples,) Observation vector alpha : float Regularization strength for the l1 norm rho: float Regularization strength for the l2 (squared) norm max_iter : int, default=1000 Maximum number of iterations tol : float, default=1e-4 The algorithm early stops if the duality gap is less than tol. f_gap: int, default=10 The gap is computed every f_gap iterations. verbose : bool, default=False Verbosity. Returns ------- W : array, shape (n_features,) Estimated coefficients. E : ndarray Objectives every gap_freq iterations. gaps : ndarray Duality gaps every gap_freq iterations. """ np.random.seed(0) n_samples, n_features = X.shape is_sparse = sparse.issparse(X) if not is_sparse and not np.isfortran(X): X = np.asfortranarray(X) if is_sparse: lc = sparse.linalg.norm(X, axis=0)**2 else: lc = (X**2).sum(axis=0) # Algo 2 in Li, Lu, Xiao 2014 w = np.zeros(n_features) u = np.zeros(n_features) z = w.copy() tau = 1. / n_features Xu = np.zeros(n_samples) Xz = np.zeros(n_samples) E = [] gaps = np.zeros(max_iter // f_gap) for it in range(max_iter): if sparse.issparse(X): tau, tau_old = _apcg_sparse(X.data, X.indices, X.indptr, z, u, tau, Xu, Xz, y, alpha, rho, lc, n_features) else: tau, tau_old = _apcg(X, z, u, tau, Xu, Xz, y, alpha, rho, lc) if it % f_gap == 0: w = tau_old**2 * u + z R = y - X @ w # MM: todo this is brutal if f_gap = 1 p_obj = primal_enet(R, w, alpha, rho) E.append(p_obj) if np.abs(p_obj) > np.abs(E[0] * 1e3): break if alpha != 0: theta = R / alpha if rho == 0: d_norm_theta = np.max(np.abs(X.T @ theta)) if d_norm_theta > 1.: theta /= d_norm_theta d_obj = dual_lasso(y, theta, alpha) else: XTR = X.T @ R d_obj = dual_enet(XTR, y - R, y, alpha, rho) gap = p_obj - d_obj if verbose: print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" % (it, p_obj, d_obj, gap)) gaps[it // f_gap] = gap if gap < tol: print("Early exit") break else: if verbose: print("Iteration %d, p_obj::%.10f" % (it, p_obj)) return w, np.array(E), gaps[:it // f_gap + 1]
def convert_to_float32(data): if (not np.issubdtype(data.dtype, np.float32) or np.isfortran(data)): print "Converting H0" return data.astype(np.float32, order='C') else: return data
def set_array(self, new_array): if not np.isfortran(new_array): new_array = np.asfortranarray(new_array) self.array = new_array
def set_scalars(self, new_scalars): order = 'C' if np.isfortran(self.array): order = 'F' new_scalars = np.reshape(new_scalars, self.array.shape, order=order) self.array = np.asfortranarray(new_scalars)
i8: np.int64 A: np.ndarray B: List[int] C: SubClass reveal_type(np.count_nonzero(i8)) # E: int reveal_type(np.count_nonzero(A)) # E: int reveal_type(np.count_nonzero(B)) # E: int reveal_type(np.count_nonzero( A, keepdims=True)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] reveal_type(np.count_nonzero( A, axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray] reveal_type(np.isfortran(i8)) # E: bool reveal_type(np.isfortran(A)) # E: bool reveal_type(np.argwhere(i8)) # E: numpy.ndarray reveal_type(np.argwhere(A)) # E: numpy.ndarray reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray reveal_type(np.flatnonzero(A)) # E: numpy.ndarray reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray reveal_type(np.outer(i8, A)) # E: numpy.ndarray
def neldermead(fcn, x0, xmin, xmax, ftol=EPSILON, maxfev=None, initsimplex=0, finalsimplex=9, step=None, iquad=1, verbose=0): x, xmin, xmax = _check_args(x0, xmin, xmax) order = 'F' if numpy.isfortran(x) else 'C' if step is None or (numpy.iterable(step) and len(step) != len(x)): step = 1.2 * numpy.ones(x.shape, numpy.float_, order) elif numpy.isscalar(step): step = step * numpy.ones(x.shape, numpy.float_, order) def stat_cb0(pars): return fcn(pars)[0] # # A safeguard just in case the initial simplex is outside the bounds # orig_fcn = stat_cb0 def stat_cb0(x_new): if _my_is_nan(x_new) or _outside_limits(x_new, xmin, xmax): return FUNC_MAX return orig_fcn(x_new) debug = False # for internal use only if numpy.isscalar(finalsimplex) and 0 == numpy.iterable(finalsimplex): finalsimplex = int(finalsimplex) if 0 == finalsimplex: finalsimplex = [1] elif 1 == finalsimplex: finalsimplex = [2] elif 2 == finalsimplex: finalsimplex = [0, 0] elif 3 == finalsimplex: finalsimplex = [0, 1] elif 4 == finalsimplex: finalsimplex = [0, 1, 0] elif 5 == finalsimplex: finalsimplex = [0, 2, 0] elif 6 == finalsimplex: finalsimplex = [1, 1, 0] elif 7 == finalsimplex: finalsimplex = [2, 1, 0] elif 8 == finalsimplex: finalsimplex = [1, 2, 0] elif 9 == finalsimplex: finalsimplex = [0, 1, 1] elif 10 == finalsimplex: finalsimplex = [0, 2, 1] elif 11 == finalsimplex: finalsimplex = [1, 1, 1] elif 12 == finalsimplex: finalsimplex = [1, 2, 1] elif 13 == finalsimplex: finalsimplex = [2, 1, 1] else: finalsimplex = [2, 2, 2] elif (False == numpy.isscalar(finalsimplex) and 1 == numpy.iterable(finalsimplex)): pass else: finalsimplex = [2, 2, 2] finalsimplex = numpy.asarray(finalsimplex, numpy.int_) if maxfev is None: maxfev = 1024 * len(x) if debug: print( 'opfcts.py neldermead() finalsimplex=%s\tisscalar=%s\titerable=%d' % (finalsimplex, numpy.isscalar(finalsimplex), numpy.iterable(finalsimplex))) def simplex(verbose, maxfev, init, final, tol, step, xmin, xmax, x, myfcn, debug, ofval=FUNC_MAX): tmpfinal = final[:] if len(final) >= 3: tmpfinal = final[0:-1] # get rid of the last entry in the list xx, ff, nf, er = _saoopt.neldermead(verbose, maxfev, init, tmpfinal, tol, step, xmin, xmax, x, myfcn) if debug: print('finalsimplex=%s, nfev=%d:\tf%s=%.20e' % (tmpfinal, nf, xx, ff)) if len(final) >= 3 and ff < 0.995 * ofval and nf < maxfev: myfinal = [final[-1]] x, fval, nfev, err = simplex(verbose, maxfev - nf, init, myfinal, tol, step, xmin, xmax, x, myfcn, debug, ofval=ff) return x, fval, nfev + nf, err else: return xx, ff, nf, er x, fval, nfev, ier = simplex(verbose, maxfev, initsimplex, finalsimplex, ftol, step, xmin, xmax, x, stat_cb0, debug) if debug: print('f%s=%e in %d nfev' % (x, fval, nfev)) info = 1 covarerr = None if len(finalsimplex) >= 3 and 0 != iquad: nelmea = minim(fcn, x, xmin, xmax, ftol=10.0 * ftol, maxfev=maxfev - nfev - 12, iquad=1) nelmea_x = numpy.asarray(nelmea[1], numpy.float_) nelmea_nfev = nelmea[4].get('nfev') info = nelmea[4].get('info') covarerr = nelmea[4].get('covarerr') nfev += nelmea_nfev minim_fval = nelmea[2] if minim_fval < fval: x = nelmea_x fval = minim_fval if debug: print('minim: f%s=%e %d nfev, info=%d' % (x, fval, nelmea_nfev, info)) if nfev >= maxfev: ier = 3 key = { 0: (True, 'Optimization terminated successfully'), 1: (False, 'improper input parameters'), 2: (False, 'improper values for x, xmin or xmax'), 3: (False, 'number of function evaluations has exceeded %d' % maxfev) } status, msg = key.get(ier, (False, 'unknown status flag (%d)' % ier)) rv = (status, x, fval) print_covar_err = False if print_covar_err and None != covarerr: rv += (msg, {'covarerr': covarerr, 'info': status, 'nfev': nfev}) else: rv += (msg, {'info': status, 'nfev': nfev}) return rv
def verify_is_c_contiguous_and_is_not_fortran(x): if np.isfortran(x): raise ValueError("Must be 'C' order") if not x.flags.c_contiguous: raise ValueError("Must be 'C'-contiguous")
def norm(a, ord=None, axis=None, keepdims=False, check_finite=True): """ Matrix or vector norm. This function is able to return one of seven different matrix norms, or one of an infinite number of vector norms (described below), depending on the value of the ``ord`` parameter. Parameters ---------- a : (M,) or (M, N) array_like Input array. If `axis` is None, `a` must be 1D or 2D. ord : {non-zero int, inf, -inf, 'fro'}, optional Order of the norm (see table under ``Notes``). inf means NumPy's `inf` object axis : {int, 2-tuple of ints, None}, optional If `axis` is an integer, it specifies the axis of `a` along which to compute the vector norms. If `axis` is a 2-tuple, it specifies the axes that hold 2-D matrices, and the matrix norms of these matrices are computed. If `axis` is None then either a vector norm (when `a` is 1-D) or a matrix norm (when `a` is 2-D) is returned. keepdims : bool, optional If this is set to True, the axes which are normed over are left in the result as dimensions with size one. With this option the result will broadcast correctly against the original `a`. check_finite : bool, optional Whether to check that the input matrix contains only finite numbers. Disabling may give a performance gain, but may result in problems (crashes, non-termination) if the inputs do contain infinities or NaNs. Returns ------- n : float or ndarray Norm of the matrix or vector(s). Notes ----- For values of ``ord <= 0``, the result is, strictly speaking, not a mathematical 'norm', but it may still be useful for various numerical purposes. The following norms can be calculated: ===== ============================ ========================== ord norm for matrices norm for vectors ===== ============================ ========================== None Frobenius norm 2-norm 'fro' Frobenius norm -- inf max(sum(abs(x), axis=1)) max(abs(x)) -inf min(sum(abs(x), axis=1)) min(abs(x)) 0 -- sum(x != 0) 1 max(sum(abs(x), axis=0)) as below -1 min(sum(abs(x), axis=0)) as below 2 2-norm (largest sing. value) as below -2 smallest singular value as below other -- sum(abs(x)**ord)**(1./ord) ===== ============================ ========================== The Frobenius norm is given by [1]_: :math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}` The ``axis`` and ``keepdims`` arguments are passed directly to ``numpy.linalg.norm`` and are only usable if they are supported by the version of numpy in use. References ---------- .. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*, Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15 Examples -------- >>> from scipy.linalg import norm >>> a = np.arange(9) - 4.0 >>> a array([-4., -3., -2., -1., 0., 1., 2., 3., 4.]) >>> b = a.reshape((3, 3)) >>> b array([[-4., -3., -2.], [-1., 0., 1.], [ 2., 3., 4.]]) >>> norm(a) 7.745966692414834 >>> norm(b) 7.745966692414834 >>> norm(b, 'fro') 7.745966692414834 >>> norm(a, np.inf) 4 >>> norm(b, np.inf) 9 >>> norm(a, -np.inf) 0 >>> norm(b, -np.inf) 2 >>> norm(a, 1) 20 >>> norm(b, 1) 7 >>> norm(a, -1) -4.6566128774142013e-010 >>> norm(b, -1) 6 >>> norm(a, 2) 7.745966692414834 >>> norm(b, 2) 7.3484692283495345 >>> norm(a, -2) 0 >>> norm(b, -2) 1.8570331885190563e-016 >>> norm(a, 3) 5.8480354764257312 >>> norm(a, -3) 0 """ # Differs from numpy only in non-finite handling and the use of blas. if check_finite: a = np.asarray_chkfinite(a) else: a = np.asarray(a) # Only use optimized norms if axis and keepdims are not specified. if a.dtype.char in 'fdFD' and axis is None and not keepdims: if ord in (None, 2) and (a.ndim == 1): # use blas for fast and stable euclidean norm nrm2 = get_blas_funcs('nrm2', dtype=a.dtype) return nrm2(a) if a.ndim == 2 and axis is None and not keepdims: # Use lapack for a couple fast matrix norms. # For some reason the *lange frobenius norm is slow. lange_args = None # Make sure this works if the user uses the axis keywords # to apply the norm to the transpose. if ord == 1: if np.isfortran(a): lange_args = '1', a elif np.isfortran(a.T): lange_args = 'i', a.T elif ord == np.inf: if np.isfortran(a): lange_args = 'i', a elif np.isfortran(a.T): lange_args = '1', a.T if lange_args: lange = get_lapack_funcs('lange', dtype=a.dtype) return lange(*lange_args) # Filter out the axis and keepdims arguments if they aren't used so they # are never inadvertently passed to a version of numpy that doesn't # support them. if axis is not None: if keepdims: return np.linalg.norm(a, ord=ord, axis=axis, keepdims=keepdims) return np.linalg.norm(a, ord=ord, axis=axis) return np.linalg.norm(a, ord=ord)
def from_array(cls, arr: np.ndarray): return cls(arr.shape, arr.dtype, strides=arr.strides, order='CF'[np.isfortran(arr)])
def GBDT_test(data, fold_n, num_rounds=100000, bf=1, ff=1): model_type = "mort" if isMORT else "lgb" nFeatures = data.X_train.shape[1] early_stop = 100 verbose_eval = 20 #lr = 0.01; bf = bf ff = ff if data.problem() == "classification": metric = 'auc' #"rmse" params = { "objective": "binary", "metric": metric, 'n_estimators': num_rounds, "bagging_fraction": bf, "feature_fraction": ff, 'verbose_eval': verbose_eval, "early_stopping_rounds": early_stop, 'n_jobs': -1, } else: metric = 'l2' #"rmse" params = { "objective": "regression", "metric": metric, 'n_estimators': num_rounds, "bagging_fraction": bf, "feature_fraction": ff, 'verbose_eval': verbose_eval, "early_stopping_rounds": early_stop, 'n_jobs': -1, } print(f"====== GBDT_test\tparams={params}") X_train, y_train = data.X_train, data.y_train X_valid, y_valid = data.X_valid, data.y_valid X_test, y_test = data.X_test, data.y_test if not np.isfortran( X_train): #Very important!!! mort need COLUMN-MAJOR format X_train = np.asfortranarray(X_train) X_valid = np.asfortranarray(X_valid) #X_train, X_valid = pd.DataFrame(X_train), pd.DataFrame(X_valid) print(f"GBDT_test\ttrain={X_train.shape} valid={X_valid.shape}") #print(f"X_train=\n{X_train.head()}\n{X_train.tail()}") if model_type == 'mort': params['verbose'] = 667 model = LiteMORT(params).fit(X_train, y_train, eval_set=[(X_valid, y_valid)]) #y_pred_valid = model.predict(X_valid) #y_pred = model.predict(X_test) if model_type == 'lgb': if data.problem() == "classification": model = lgb.LGBMClassifier(**params) else: model = lgb.LGBMRegressor(**params) model.fit(X_train, y_train, eval_set=[(X_train, y_train), (X_valid, y_valid)], verbose=min(num_rounds // 10, 1000)) pred_val = model.predict(data.X_test) #plot_importance(model) lgb.plot_importance(model, max_num_features=32) plt.title("Featurertances") plt.savefig(f"./results/{dataset}_feat_importance_.jpg") #plt.show(block=False) plt.close() fold_importance = pd.DataFrame() fold_importance["importance"] = model.feature_importances_ fold_importance["feature"] = [i for i in range(nFeatures)] fold_importance["fold"] = fold_n #fold_importance.to_pickle(f"./results/{dataset}_feat_{fold_n}.pickle") print('best_score', model.best_score_) acc_train, acc_ = model.best_score_['training'][ metric], model.best_score_['valid_1'][metric] if data.X_test is not None: pred_val = model.predict(data.X_test) acc_ = ((data.y_test - pred_val)**2).mean() print( f'====== Best step: test={data.X_test.shape} ACCU@Test={acc_:.5f}') return acc_, fold_importance
def write_edf(edf_file, signals, signal_headers, header=None, digital=False, file_type=-1, block_size=1): """ Write signals to an edf_file. Header can be generated on the fly with generic values. EDF+/BDF+ is selected based on the filename extension, but can be overwritten by setting filetype to pyedflib.FILETYPE_XXX Parameters ---------- edf_file : np.ndarray or list where to save the EDF file signals : list The signals as a list of arrays or a ndarray. signal_headers : list of dict a list with one signal header(dict) for each signal. See pyedflib.EdfWriter.setSignalHeader.. header : dict a main header (dict) for the EDF file, see pyedflib.EdfWriter.setHeader for details. If no header present, will create an empty header digital : bool, optional whether the signals are in digital format (ADC). The default is False. filetype: int, optional choose filetype for saving. EDF = 0, EDF+ = 1, BDF = 2, BDF+ = 3, automatic from extension = -1 block_size : int set the block size for writing. Should be divisor of signal length in seconds. Higher values mean faster writing speed, but if it is not a divisor of the signal duration, it will append zeros. Can be any value between 1=><=60, -1 will auto-infer the fastest value. Returns ------- bool True if successful, False if failed. """ assert header is None or isinstance(header, dict), \ 'header must be dictioniary or None' assert isinstance(signal_headers, list), \ 'signal headers must be list' assert len(signal_headers)==len(signals), \ 'signals and signal_headers must be same length' assert file_type in [-1, 0, 1, 2, 3], \ 'filetype must be in range -1, 3' assert block_size<=60 and block_size>=-1 and block_size!=0, \ 'blocksize must be smaller or equal to 60' # copy objects to prevent accidential changes to mutable objects header = deepcopy(header) signal_headers = deepcopy(signal_headers) if file_type == -1: ext = os.path.splitext(edf_file)[-1] if ext.lower() == '.edf': file_type = pyedflib.FILETYPE_EDFPLUS elif ext.lower() == '.bdf': file_type = pyedflib.FILETYPE_BDFPLUS else: raise ValueError('Unknown extension {}'.format(ext)) n_channels = len(signals) # if there is no header, we create one with dummy values if header is None: header = {} default_header = make_header() default_header.update(header) header = default_header # block_size sets the size of each writing block and should be a divisor # of the length of the signal. If it is not, the remainder of the file # will be filled with zeros. signal_duration = len(signals[0]) / signal_headers[0]['sample_rate'] if block_size == -1: samplefrequencies = np.array( [hdr["sample_rate"] for hdr in signal_headers]) block_size = min([ d for d in range(1, 61) if ((signal_duration % d == 0) and np.all( ((samplefrequencies * d) % 1) == 0)) ]) elif signal_duration % block_size != 0: warnings.warn('Signal length is not dividable by block_size. ' + 'The file will have a zeros appended.') # check dmin, dmax and pmin, pmax dont exceed signal min/max for sig, shead in zip(signals, signal_headers): dmin, dmax = shead['digital_min'], shead['digital_max'] pmin, pmax = shead['physical_min'], shead['physical_max'] label = shead['label'] if digital: # exception as it will lead to clipping assert dmin<=sig.min(), \ 'digital_min is {}, but signal_min is {}' \ 'for channel {}'.format(dmin, sig.min(), label) assert dmax>=sig.max(), \ 'digital_min is {}, but signal_min is {}' \ 'for channel {}'.format(dmax, sig.max(), label) assert pmin != pmax, \ 'physical_min {} should be different from physical_max {}'.format(pmin,pmax) else: # only warning, as this will not lead to clipping assert pmin<=sig.min(), \ 'phys_min is {}, but signal_min is {} ' \ 'for channel {}'.format(pmin, sig.min(), label) assert pmax>=sig.max(), \ 'phys_max is {}, but signal_max is {} ' \ 'for channel {}'.format(pmax, sig.max(), label) shead['sample_rate'] *= block_size # get annotations, in format [[timepoint, duration, description], [...]] annotations = header.get('annotations', []) if any([np.isfortran(s) for s in signals]) or \ (isinstance(signals, np.ndarray) and np.isfortran(signals)): warnings.warn('signals are in Fortran order. Will automatically ' \ 'transfer to C order for compatibility with edflib.') if isinstance(signals, list): signals = [ s.copy(order='C') if np.isfortran(s) else s for s in signals ] elif isinstance(signals, np.ndarray) and np.isfortran(signals): signals = signals.copy(order='C') with pyedflib.EdfWriter(edf_file, n_channels=n_channels, file_type=file_type) as f: f.setDatarecordDuration(int(100000 * block_size)) f.setSignalHeaders(signal_headers) f.setHeader(header) f.writeSamples(signals, digital=digital) for annotation in annotations: f.writeAnnotation(*annotation) del f return os.path.isfile(edf_file)
def build(self, X, y, criterion, max_depth, min_samples_split, min_samples_leaf, min_density, max_features, random_state, find_split, sample_mask=None, X_argsorted=None): # Recursive algorithm def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent, is_left_child): # Count samples n_node_samples = sample_mask.sum() if n_node_samples == 0: raise ValueError("Attempting to find a split " "with an empty sample_mask") # Split samples if depth < max_depth and n_node_samples >= min_samples_split \ and n_node_samples >= 2 * min_samples_leaf: feature, threshold, best_error, init_error = find_split( X, y, X_argsorted, sample_mask, n_node_samples, min_samples_leaf, max_features, criterion, random_state) else: feature = -1 init_error = _tree._error_at_leaf(y, sample_mask, criterion, n_node_samples) value = criterion.init_value() # Current node is leaf if feature == -1: self._add_leaf(parent, is_left_child, value, init_error, n_node_samples) # Current node is internal node (= split node) else: # Sample mask is too sparse? if n_node_samples / X.shape[0] <= min_density: X = X[sample_mask] X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) y = y[sample_mask] sample_mask = np.ones((X.shape[0], ), dtype=np.bool) # Split and and recurse split = X[:, feature] <= threshold node_id = self._add_split_node(parent, is_left_child, feature, threshold, best_error, init_error, n_node_samples, value) # left child recursion recursive_partition(X, X_argsorted, y, np.logical_and(split, sample_mask), depth + 1, node_id, True) # right child recursion recursive_partition( X, X_argsorted, y, np.logical_and(np.logical_not(split), sample_mask), depth + 1, node_id, False) # Setup auxiliary data structures and check input before # recursive partitioning if X.dtype != DTYPE or not np.isfortran(X): X = np.asanyarray(X, dtype=DTYPE, order="F") if y.dtype != DTYPE or not y.flags.contiguous: y = np.ascontiguousarray(y, dtype=DTYPE) if sample_mask is None: sample_mask = np.ones((X.shape[0], ), dtype=np.bool) if X_argsorted is None: X_argsorted = np.asfortranarray( np.argsort(X.T, axis=1).astype(np.int32).T) # Pre-allocate some space if max_depth <= 10: # allocate space for complete binary tree init_capacity = (2**(max_depth + 1)) - 1 else: # allocate fixed size and dynamically resize later init_capacity = 2047 self._resize(init_capacity) # Build the tree by recursive partitioning recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False) # Compactify the tree data structure self._resize(self.node_count) return self
def fit_file(self, motion_correct=False, indices=[slice(None)] * 2): """ This method packages the analysis pipeline (motion correction, memory mapping, patch based CNMF processing) in a single method that can be called on a specific (sequence of) file(s). It is assumed that the CNMF object already contains a params object where the location of the files and all the relevant parameters have been specified. The method does not perform the quality evaluation step. Consult demo_pipeline for an example. Args: motion_correct (bool) flag for performing motion correction indices (list of slice objects) perform analysis only on a part of the FOV Returns: cnmf object with the current estimates """ fnames = self.params.get('data', 'fnames') if os.path.exists(fnames[0]): _, extension = os.path.splitext(fnames[0])[:2] extension = extension.lower() else: logging.warning("Error: File not found, with file list:\n" + fnames[0]) raise Exception('File not found!') if extension == '.mmap': fname_new = fnames[0] Yr, dims, T = mmapping.load_memmap(fnames[0]) if np.isfortran(Yr): raise Exception( 'The file should be in C order (see save_memmap function)') else: if motion_correct: mc = MotionCorrect(fnames, dview=self.dview, **self.params.motion) mc.motion_correct(save_movie=True) fname_mc = mc.fname_tot_els if self.params.motion[ 'pw_rigid'] else mc.fname_tot_rig if self.params.get('motion', 'pw_rigid'): b0 = np.ceil( np.maximum(np.max(np.abs(mc.x_shifts_els)), np.max(np.abs(mc.y_shifts_els)))).astype( np.int) self.estimates.shifts = [mc.x_shifts_els, mc.y_shifts_els] else: b0 = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int) self.estimates.shifts = mc.shifts_rig b0 = 0 if self.params.get('motion', 'border_nan') is 'copy' else 0 fname_new = mmapping.save_memmap(fname_mc, base_name='memmap_', order='C', border_to_0=b0) else: fname_new = mmapping.save_memmap(fnames, base_name='memmap_', order='C') # now load the file Yr, dims, T = mmapping.load_memmap(fname_new) images = np.reshape(Yr.T, [T] + list(dims), order='F') self.mmap_file = fname_new return self.fit(images, indices=indices)
def bin_array(arraytotcorr, indZ, threeD_array): """ Function to reduce the size of large datasets. Data placed into equidistant bins for each x,y coordinate and new vector created from the mean of each bin. Size of equidistant bins determined by 0.01 nm increments of Zsensor data. :param arraytotcorr: Zsensor data corrected for sample tilt using correct_slope function. Important to use this and not raw Zsensor data so as to get an accurate Zmax value. :param indZ: Index of Zmax for each x,y coordinate to cut data set into approach and retract. :param rawarray: 3D numpy array the user wishes to reduce in size (e.g. phase, amp) :return: 3D numpy array of binned approach values, 3D numpy array of binned retract values. """ assert np.isfortran( threeD_array ) == True, "Input array not passed through generate_array fucntion. \ Needs to be column-major indexing." # Generate empty numpy array to populate. global reduced_array_approach global reduced_array_retract global linearized arraymean = np.zeros(len(arraytotcorr[:, 1, 1])) digitized = np.empty_like(arraymean) # Create list of the mean Zsensor value for each horizontal slice of Zsensor array. for z in range(len(arraymean)): arraymean[z] = np.mean(arraytotcorr[z, :, :]) # Turn mean Zsensor data into a linear vector with a step size of 0.02 nm. linearized = np.arange(-0.2, arraymean.max(), 0.02) # Generate empty array to populate reduced_array_approach = np.zeros( (len(linearized), len(arraytotcorr[1, :, 1]), len(arraytotcorr[1, 1, :]))) reduced_array_retract = np.zeros( (len(linearized), len(arraytotcorr[1, :, 1]), len(arraytotcorr[1, 1, :]))) # Cut raw phase/amp datasets into approach and retract, then bin data according to the linearized Zsensor data. # Generate new arrays from the means of each bin. Perform on both approach and retract data. for j in range(len(arraytotcorr[1, :, 1])): for i in range(len(arraytotcorr[1, 1, :])): z = arraytotcorr[:(int(indZ[i, j])), i, j] # Create dataset with just retract data digitized = np.digitize( z, linearized ) # Bin Z data based on standardized linearized vector. for n in range(len(linearized)): ind = list(np.where(digitized == n) [0]) # Find which indices belong to which bins reduced_array_approach[n, i, j] = np.mean( threeD_array[ind, i, j]) # Find the mean of the bins and # populate new array. for j in range(len(arraytotcorr[1, :, 1])): for i in range(len(arraytotcorr[1, 1, :])): z = arraytotcorr[-(int(indZ[i, j])):, i, j] # Create dataset with just approach data. z = np.flipud( z) # Flip array so surface is at the bottom on the plot. digitized = np.digitize( z, linearized ) # Bin Z data based on standardized linearized vector. for n in range(len(linearized)): ind = list(np.where(digitized == n) [0]) # Find which indices belong to which bins reduced_array_retract[n, i, j] = np.mean( threeD_array[ind, i, j]) # Find the mean of the bins and # populate new array. return linearized, reduced_array_approach, reduced_array_retract
#b = numpy.ones(4) #err = libzquatev.test(a,b) #exit(1) n = 2 #1000 print("n=",n) fock = libzquatev.gen_array(n) # return a Hermitian matrix print(numpy.linalg.norm(fock - fock.conj().T)) print("F=\n",fock) print("\nzquatev") e,v = zquatev.eigh(fock,iop=1) print("e=",e) numpy.set_printoptions(4, linewidth=120) print("v=",v) print("Fc0-c0*E0 Should be clouse to zero", e[0]) #print(numpy.dot(fock, v[:,0]) - e[0]*v[:,0]) print(abs(numpy.dot(fock, v) - e*v).max()) print("v0=",v[0,:]) print(numpy.isfortran(fock)) print(fock.flags) print("\nnumpy") e,v = numpy.linalg.eigh(fock) print("e=",e.shape,e) print("v0=",v[:,0]) print("v=\n",v) print("Should be clouse to zero", e[0]) #print(abs(numpy.dot(fock, v[:,0]) - e[0]*v[:,0]).max()) print(abs(numpy.dot(fock, v) - e*v).max())
def apcg_logreg(X, y, alpha, max_iter=10000, tol=1e-4, f_gap=10, seed=0, verbose=False): """Solve the l1 regularized logistic regression with with accelerated proximal coordinate gradient. Parameters ---------- X : {array_like, sparse matrix}, shape (n_samples, n_features) Design matrix y : ndarray, shape (n_samples,) Observation vector alpha : float Regularization strength max_iter : int, default=1000 Maximum number of iterations tol : float, default=1e-4 The algorithm early stops if the duality gap is less than tol. f_gap: int, default=10 The gap is computed every f_gap iterations. seed : int (default=0) Seed for randomness. verbose : bool, default=False Verbosity. Returns ------- w : array, shape (n_features,) Estimated coefficients. E : ndarray Objectives every gap_freq iterations. gaps : ndarray Duality gaps every gap_freq iterations. """ np.random.seed(seed) n_samples, n_features = X.shape is_sparse = sparse.issparse(X) if not is_sparse and not np.isfortran(X): X = np.asfortranarray(X) if is_sparse: lc = sparse.linalg.norm(X, axis=0) ** 2 / 4. else: lc = (X ** 2).sum(axis=0) / 4. w = np.zeros(n_features) u = np.zeros(n_features) z = w.copy() tau = 1. / n_features Xu = np.zeros(n_samples) Xz = np.zeros(n_samples) E = [] gaps = np.zeros(max_iter // f_gap) for it in range(max_iter): if is_sparse: tau, tau_old = _apcg_sparse( X.data, X.indices, X.indptr, z, u, tau, Xu, Xz, y, alpha, lc, n_features) else: tau, tau_old = _apcg(X, z, u, tau, Xu, Xz, y, alpha, lc) if it % f_gap == 0: w = tau_old ** 2 * u + z Xw = X @ w p_obj = primal_logreg(Xw, y, w, alpha) E.append(p_obj) if np.abs(p_obj) > np.abs(E[0] * 1e3): break if alpha != 0: theta = y * sigmoid(-y * Xw) / alpha d_norm_theta = np.max(np.abs(X.T @ theta)) if d_norm_theta > 1.: theta /= d_norm_theta d_obj = dual_logreg(y, theta, alpha) gap = p_obj - d_obj if verbose: print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" % (it, p_obj, d_obj, gap)) gaps[it // f_gap] = gap if gap < tol: print("Early exit") break else: if verbose: print("Iteration %d, p_obj::%.10f" % (it, p_obj)) return w, np.array(E), gaps[:it // f_gap + 1]
def elastic_net(predictors, target, balance, memlimit=None, largest=None, thr=_DEFAULT_THRESH, weights=None, penalties=None, standardize=True, exclude=None, lambdas=None, flmin=None, nlam=None, overwrite_pred_ok=False, overwrite_targ_ok=True): """ Raw-output wrapper for elastic net linear regression. Args: predictors (np.ndarray): X design matrix target (np.ndarray): y dependent variables balance (float): Family member index (0 is ridge, 1 is Lasso) memlimit (int): Maximum number of variables allowed to enter all models along path largest (int): Maximum number of variables allowed to enter largest model thr (float): Minimum change in largest coefficient weights (np.ndarray): Relative weighting per observation case penalties (np.ndarray): Relative penalties per predictor (0 = no penalty)(vp in Fortran code) standardize (bool): Standardize input variables before proceeding? (isd in Fortran code) exclude (np.ndarray): Predictors to exclude altogether from fitting (jd in Fortran code) lambdas (np.ndarray): User specified lambda values (ulam in Fortran code). Do not specify ``lambdas`` if ``flmin`` or ``nlam`` are also provided. flmin (float): Fraction of largest lambda at which to stop (default: 0.001) nlam (int): The (maximum) number of lambdas to try (default: 100) overwrite_pred_ok (bool): Allow overwriting of X overwrite_targ_ok (bool): Allow overwriting of y """ # Decide on largest allowable models for memory/convergence. memlimit = predictors.shape[1] if memlimit is None else memlimit # If largest isn't specified use memlimit. largest = memlimit if largest is None else largest if memlimit < largest: raise ValueError('Need largest <= memlimit') if exclude is not None: # Add one since Fortran indices start at 1 exclude += 1 jd = np.array([len(exclude)] + exclude) else: jd = np.zeros(1) if lambdas is not None and flmin is not None: raise ValueError("Can't specify both lambdas & flmin keywords") elif lambdas is not None: lambdas = np.atleast_1d(lambdas) flmin = 2. # Pass flmin > 1.0 indicating to use the user-supplied. nlam = len(lambdas) else: lambdas = None flmin = _DEFAULT_FLMIN nlam = _DEFAULT_NLAM # If predictors is a Fortran contiguous array, it will be overwritten. # Decide whether we want this. If it's not Fortran contiguous it will # be copied into that form anyway so there's no chance of overwriting. if np.isfortran(predictors): if not overwrite_pred_ok: # Might as well make it F-ordered to avoid ANOTHER copy. predictors = predictors.copy(order='F') # Target being a 1-dimensional array will usually be overwritten # with the standardized version unless we take steps to copy it. if not overwrite_targ_ok: target = target.copy() # Uniform weighting if no weights are specified. if weights is None: weights = np.ones(predictors.shape[0]) # Uniform penalties if none were specified. if penalties is None: penalties = np.ones(predictors.shape[1]) # Call the Fortran wrapper. lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr = \ _glmnet.elnet(balance, predictors, target, weights, jd, penalties, memlimit, flmin, lambdas, thr, nlam=nlam, isd=standardize) # Check for errors, documented in glmnet.f. if jerr != 0: if jerr == 10000: raise ValueError('cannot have max(vp) < 0.0') elif jerr == 7777: raise ValueError('all used predictors have 0 variance') elif jerr < 7777: raise MemoryError('elnet() returned error code %d' % jerr) else: raise Exception('unknown error: %d' % jerr) return lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr
def solver_logreg( X, y, alpha, rho=0, max_iter=10000, tol=1e-4, f_gap=10, K=5, use_acc=False, algo='cd', reg_amount=None, seed=0, verbose=False): """Solve l1+l2 regularized logistic regression with CD/ISTA/FISTA, eventually using Anderson extrapolation. The minimized objective is: np.sum(np.log(1 + np.exp(- y * Xw))) / 2 + alpha * norm(x, ord=1) + rho/2 * norm(w) ** 2 Parameters ---------- X : {array_like, sparse matrix}, shape (n_samples, n_features) Design matrix y : ndarray, shape (n_samples,) Observation vector alpha : float L1 regularization strength rho : float (optional, default=0) L2 regularization strength. max_iter : int, default=1000 Maximum number of iterations tol : float, default=1e-4 The algorithm early stops if the duality gap is less than tol. f_gap: int, default=10 The gap is computed every f_gap iterations. K : int, default=5 Number of points used for Anderson extrapolation. use_acc : bool, default=False TODO change to True? Whether or not to use Anderson acceleration. algo : {'cd', 'pgd', 'fista'} Algorithm used to solve the Elastic net problem. reg_amount : float or None (default=None) Amount of regularization used when solving for the extrapolation coefficients. None means 0 and should be preferred. seed : int (default=0) Seed for randomness. verbose : bool, default=False Verbosity. Returns ------- W : array, shape (n_features,) Estimated coefficients. E : ndarray Objectives every gap_freq iterations. gaps : ndarray Duality gaps every gap_freq iterations. """ np.random.seed(seed) is_sparse = sparse.issparse(X) n_features = X.shape[1] _range = np.arange(n_features) feats = dict(cd=lambda: _range, cd2=lambda: np.hstack((_range, _range)), cdsym=lambda: np.hstack((_range, _range[::-1])), cdshuf=lambda: np.random.choice(n_features, n_features, replace=False)) if not is_sparse and not np.isfortran(X): X = np.asfortranarray(X) if use_acc: last_K_w = np.zeros([K + 1, n_features]) U = np.zeros([K, n_features]) if algo == 'pgd' or algo == 'fista': if is_sparse: L = power_method(X, max_iter=1000) ** 2 / 4 else: L = norm(X, ord=2) ** 2 / 4 if is_sparse: lc = sparse.linalg.norm(X, axis=0) ** 2 / 4 else: lc = (X ** 2).sum(axis=0) / 4 w = np.zeros(n_features) if algo == 'fista': z = np.zeros(n_features) t_new = 1 Xw = np.zeros(len(y)) E = [] gaps = np.zeros(max_iter // f_gap) for it in range(max_iter): if it % f_gap == 0: if algo == 'fista': Xw = X @ w p_obj = primal_logreg(Xw, y, w, alpha, rho) E.append(p_obj) if alpha != 0: theta = y * sigmoid(-y * Xw) / alpha d_norm_theta = np.max(np.abs(X.T @ theta)) if d_norm_theta > 1.: theta /= d_norm_theta d_obj = dual_logreg(y, theta, alpha) gap = p_obj - d_obj if verbose: print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" % (it, p_obj, d_obj, gap)) gaps[it // f_gap] = gap if gap < tol: print("Early exit") break else: if verbose: print("Iteration %d, p_obj::%.10f" % (it, p_obj)) if algo.startswith('cd'): if is_sparse: _cd_logreg_sparse(X.data, X.indices, X.indptr, w, Xw, y, alpha, rho, lc, feats[algo]()) else: _cd_logreg(X, w, Xw, y, alpha, rho, lc, feats[algo]()) elif algo == 'pgd': w[:] = ST_vec( w + 1. / L * X.T @ (y / (1. + np.exp(y * Xw))), alpha / L) if rho != 0: w /= 1. + rho / L Xw[:] = X @ w elif algo == 'fista': w_old = w.copy() w[:] = ST_vec( z + X.T @ (y / (1. + np.exp(y * (X @ z)))) / L, alpha / L) if rho != 0: w /= 1. + rho / L t_old = t_new t_new = (1. + np.sqrt(1 + 4 * t_old ** 2)) / 2. z[:] = w + (t_old - 1.) / t_new * (w - w_old) else: raise ValueError("Unknown algo %s" % algo) if use_acc: if it < K + 1: last_K_w[it] = w else: for k in range(K): last_K_w[k] = last_K_w[k + 1] last_K_w[K] = w for k in range(K): U[k] = last_K_w[k + 1] - last_K_w[k] C = np.dot(U, U.T) if reg_amount is not None: C += reg_amount * norm(C, ord=2) * np.eye(C.shape[0]) try: z = np.linalg.solve(C, np.ones(K)) c = z / z.sum() w_acc = np.sum(last_K_w[:-1] * c[:, None], axis=0) p_obj = primal_logreg(Xw, y, w, alpha, rho) Xw_acc = X @ w_acc p_obj_acc = primal_logreg(Xw_acc, y, w_acc, alpha, rho) if p_obj_acc < p_obj: w = w_acc Xw = Xw_acc except np.linalg.LinAlgError: if verbose: print("----------Linalg error") return w, np.array(E), gaps[:it // f_gap + 1]
def fit(self, images, indices=[slice(None), slice(None)]): """ This method uses the cnmf algorithm to find sources in data. it is calling every function from the cnmf folder you can find out more at how the functions are called and how they are laid out at the ipython notebook Args: images : mapped np.ndarray of shape (t,x,y[,z]) containing the images that vary over time. indices: list of slice objects along dimensions (x,y[,z]) for processing only part of the FOV Returns: self: updated using the cnmf algorithm with C,A,S,b,f computed according to the given initial values Raises: Exception 'You need to provide a memory mapped file as input if you use patches!!' See Also: ..image::docs/img/quickintro.png http://www.cell.com/neuron/fulltext/S0896-6273(15)01084-3 """ # Todo : to compartment if isinstance(indices, slice): indices = [indices] indices = [slice(None)] + indices if len(indices) < len(images.shape): indices = indices + [slice(None) ] * (len(images.shape) - len(indices)) dims_orig = images.shape[1:] dims_sliced = images[tuple(indices)].shape[1:] is_sliced = (dims_orig != dims_sliced) if self.params.get('patch', 'rf') is None and (is_sliced or 'ndarray' in str(type(images))): images = images[tuple(indices)] self.dview = None logging.info("Parallel processing in a single patch " "is not available for loaded in memory or sliced" + " data.") T = images.shape[0] self.params.set('online', {'init_batch': T}) self.dims = images.shape[1:] #self.params.data['dims'] = images.shape[1:] Y = np.transpose(images, list(range(1, len(self.dims) + 1)) + [0]) Yr = np.transpose(np.reshape(images, (T, -1), order='F')) if np.isfortran(Yr): raise Exception( 'The file is in F order, it should be in C order (see save_memmap function)' ) logging.info((T, ) + self.dims) # Make sure filename is pointed correctly (numpy sets it to None sometimes) try: Y.filename = images.filename Yr.filename = images.filename self.mmap_file = images.filename except AttributeError: # if no memmapping cause working with small data pass # update/set all options that depend on data dimensions # number of rows, columns [and depths] self.params.set( 'spatial', { 'medw': (3, ) * len(self.dims), 'se': np.ones((3, ) * len(self.dims), dtype=np.uint8), 'ss': np.ones((3, ) * len(self.dims), dtype=np.uint8) }) logging.info(('Using ' + str(self.params.get('patch', 'n_processes')) + ' processes')) if self.params.get('preprocess', 'n_pixels_per_process') is None: avail_memory_per_process = psutil.virtual_memory( )[1] / 2.**30 / self.params.get('patch', 'n_processes') mem_per_pix = 3.6977678498329843e-09 npx_per_proc = np.int(avail_memory_per_process / 8. / mem_per_pix / T) npx_per_proc = np.int( np.minimum( npx_per_proc, np.prod(self.dims) // self.params.get('patch', 'n_processes'))) self.params.set('preprocess', {'n_pixels_per_process': npx_per_proc}) self.params.set( 'spatial', { 'n_pixels_per_process': self.params.get('preprocess', 'n_pixels_per_process') }) logging.info( 'using ' + str(self.params.get('preprocess', 'n_pixels_per_process')) + ' pixels per process') logging.info('using ' + str(self.params.get('spatial', 'block_size_spat')) + ' block_size_spat') logging.info('using ' + str(self.params.get('temporal', 'block_size_temp')) + ' block_size_temp') if self.params.get('patch', 'rf') is None: # no patches logging.info('preprocessing ...') Yr = self.preprocess(Yr) if self.estimates.A is None: logging.info('initializing ...') self.initialize(Y) if self.params.get( 'patch', 'only_init'): # only return values after initialization if not (self.params.get('init', 'method_init') == 'corr_pnr' and self.params.get('init', 'ring_size_factor') is not None): self.compute_residuals(Yr) self.estimates.bl = None self.estimates.c1 = None self.estimates.neurons_sn = None if self.remove_very_bad_comps: logging.info('removing bad components : ') final_frate = 10 r_values_min = 0.5 # threshold on space consistency fitness_min = -15 # threshold on time variability fitness_delta_min = -15 Npeaks = 10 traces = np.array(self.C) logging.info('estimating the quality...') idx_components, idx_components_bad, fitness_raw,\ fitness_delta, r_values = estimate_components_quality( traces, Y, self.estimates.A, self.estimates.C, self.estimates.b, self.estimates.f, final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min, fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True, N=5) logging.info( ('Keeping ' + str(len(idx_components)) + ' and discarding ' + str(len(idx_components_bad)))) self.estimates.C = self.estimates.C[idx_components] self.estimates.A = self.estimates.A[:, idx_components] # type: ignore # not provable that self.initialise provides a value self.estimates.YrA = self.estimates.YrA[idx_components] self.estimates.normalize_components() return self logging.info('update spatial ...') self.update_spatial(Yr, use_init=True) logging.info('update temporal ...') if not self.skip_refinement: # set this to zero for fast updating without deconvolution self.params.set('temporal', {'p': 0}) else: self.params.set('temporal', {'p': self.params.get('preprocess', 'p')}) logging.info('deconvolution ...') self.update_temporal(Yr) if not self.skip_refinement: logging.info('refinement...') if self.params.get('merging', 'do_merge'): logging.info('merging components ...') self.merge_comps(Yr, mx=50, fast_merge=True, max_merge_area=self.params.get( 'merging', 'max_merge_area')) logging.info('Updating spatial ...') self.update_spatial(Yr, use_init=False) # set it back to original value to perform full deconvolution self.params.set('temporal', {'p': self.params.get('preprocess', 'p')}) logging.info('update temporal ...') self.update_temporal(Yr, use_init=False) # else: # todo : ask for those.. # C, f, S, bl, c1, neurons_sn, g1, YrA, lam = self.estimates.C, self.estimates.f, self.estimates.S, self.estimates.bl, self.estimates.c1, self.estimates.neurons_sn, self.estimates.g, self.estimates.YrA, self.estimates.lam # embed in the whole FOV if is_sliced: FOV = np.zeros(dims_orig, order='C') FOV[indices[1:]] = 1 FOV = FOV.flatten(order='F') ind_nz = np.where(FOV > 0)[0].tolist() self.estimates.A = self.estimates.A.tocsc() A_data = self.estimates.A.data A_ind = np.array(ind_nz)[self.estimates.A.indices] A_ptr = self.estimates.A.indptr A_FOV = scipy.sparse.csc_matrix( (A_data, A_ind, A_ptr), shape=(FOV.shape[0], self.estimates.A.shape[-1])) b_FOV = np.zeros((FOV.shape[0], self.estimates.b.shape[-1])) b_FOV[ind_nz] = self.estimates.b self.estimates.A = A_FOV self.estimates.b = b_FOV else: # use patches if self.params.get('patch', 'stride') is None: self.params.set('patch', { 'stride': np.int(self.params.get('patch', 'rf') * 2 * .1) }) logging.info( ('Setting the stride to 10% of 2*rf automatically:' + str(self.params.get('patch', 'stride')))) if type(images) is np.ndarray: raise Exception( 'You need to provide a memory mapped file as input if you use patches!!' ) self.estimates.A, self.estimates.C, self.estimates.YrA, self.estimates.b, self.estimates.f, \ self.estimates.sn, self.estimates.optional_outputs = run_CNMF_patches( images.filename, self.dims + (T,), self.params, dview=self.dview, memory_fact=self.params.get('patch', 'memory_fact'), gnb=self.params.get('init', 'nb'), border_pix=self.params.get('patch', 'border_pix'), low_rank_background=self.params.get('patch', 'low_rank_background'), del_duplicates=self.params.get('patch', 'del_duplicates'), indices=indices) self.estimates.bl, self.estimates.c1, self.estimates.g, self.estimates.neurons_sn = None, None, None, None logging.info("merging") self.estimates.merged_ROIs = [0] if self.params.get('init', 'center_psf'): # merge taking best neuron if self.params.get('patch', 'nb_patch') > 0: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf, fast_merge=True) logging.info("update temporal") self.update_temporal(Yr, use_init=False) self.params.set( 'spatial', { 'se': np.ones( (1, ) * len(self.dims), dtype=np.uint8) }) logging.info('update spatial ...') self.update_spatial(Yr, use_init=False) logging.info("update temporal") self.update_temporal(Yr, use_init=False) else: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf, fast_merge=True) if len(self.estimates.merged_ROIs) > 0: not_merged = np.setdiff1d( list(range(len(self.estimates.YrA))), np.unique( np.concatenate( self.estimates.merged_ROIs))) self.estimates.YrA = np.concatenate([ self.estimates.YrA[not_merged], np.array([ self.estimates.YrA[m].mean(0) for ind, m in enumerate(self.estimates.merged_ROIs) if not self.empty_merged[ind] ]) ]) if self.params.get('init', 'nb') == 0: self.estimates.W, self.estimates.b0 = compute_W( Yr, self.estimates.A.toarray(), self.estimates.C, self.dims, self.params.get('init', 'ring_size_factor') * self.params.get('init', 'gSiz')[0], ssub=self.params.get('init', 'ssub_B')) if len(self.estimates.C): self.deconvolve() else: self.estimates.S = self.estimates.C else: while len(self.estimates.merged_ROIs) > 0: self.merge_comps(Yr, mx=np.Inf) logging.info("update temporal") self.update_temporal(Yr, use_init=False) self.estimates.normalize_components() return self
def is_c_contiguous(a): return np.isfortran(a.T)
def nnls_regularized(A, b, l=0, maxiter=None): """ Solve math:`argmin_x || Ax - b ||_2^2 + \lambda^2 ||x||_2^2` for ``x>=0``. This is a wrapper for a FORTRAN non-negative least squares solver, with regularization (added by stacking $A$ on top an identity matrix times $\lambda$ and $b$ on top of a matching array of zero. Parameters ---------- A : ndarray Matrix ``A`` as shown above. b : ndarray Right-hand side vector. l : double (default 0) :math:`lambda` -- if this is set to 0, the algorithm reverts to standard nnls (rather than stacking on top of two zero matrices for no reason) maxiter: int, optional Maximum number of iterations, optional. Default is ``3 * A.shape[1]``. Returns ------- x : ndarray Solution vector. rnorm : float The residual, ``|| Ax-b ||_2``. Notes ----- The FORTRAN code was published in the book below. The algorithm is an active set method. It solves the KKT (Karush-Kuhn-Tucker) conditions for the non-negative least squares problem. This was adapted from the source distributed with scipy -- see scipy for relevant licensing. References ---------- Lawson C., Hanson R.J., (1987) Solving Least Squares Problems, SIAM """ logger.debug(strm("isfortran result",isfortran(A),isfortran(b))) A, b = list(map(asarray_chkfinite, (A, b))) if len(A.shape) != 2: raise ValueError("expected matrix") if len(b.shape) > 2: raise ValueError("expected vector") m, n = A.shape if m != b.shape[-1]: raise ValueError(strm("incompatible dimensions (rows of A",m," do not match size of data",b.shape,")")) maxiter = -1 if maxiter is None else int(maxiter) if isscalar(l): if l == 0.0: w = zeros((n,), dtype=double) zz = zeros((m,), dtype=double) index = zeros((n,), dtype=int) x, rnorm, mode = _nnls.nnls(A, b, w, zz, index, maxiter) else: w = zeros((n,), dtype=double) zz = zeros((m+n,), dtype=double) index = zeros((n,), dtype=int) # choose the correct subroutine based on the dimension if len(b.shape) == 1: x, rnorm, mode = _nnls.nnls_regularized(A, b, w, zz, index, maxiter, l) if len(b.shape) == 2: x, rnorm, mode = _nnls.nnls_regularized_loop(A, redim_C_to_F(b), w, zz, index, maxiter, l) x = redim_F_to_C(x) else: nCPU = cpu_count() #print("I found",nCPU,"CPU's") p = mpd.Pool(nCPU) if len(b.shape) == 1: def nnls_func(l): w = zeros((n,), dtype=double) zz = zeros((m+n,), dtype=double) index = zeros((n,), dtype=int) return _nnls.nnls_regularized(A, b, w, zz, index, maxiter, l) if len(b.shape) == 2: def nnls_func(l): w = zeros((n,), dtype=double) zz = zeros((m+n,), dtype=double) index = zeros((n,), dtype=int) x, rnorm, mode = _nnls.nnls_regularized_loop(A, redim_C_to_F(b), w, zz, index, maxiter, l) return redim_F_to_C(x), rnorm, mode retval = p.map(nnls_func,l) x,rnorm,mode = list(map(np_array,list(zip(*retval)))) if (isscalar(mode) and mode != 1): # need something for the multiple lambda raise RuntimeError("too many iterations") return x, rnorm