def sparseScalarProductOfDot (A, B, C, out=None):
'''
Returns A * np.dot(B, C), however it does so keeping in mind
the sparsity of A, calculating values only where required.
Params
A - a sparse CSR matrix
B - a dense matrix
C - a dense matrix
out - if specified, must be a sparse CSR matrix with identical
non-zero pattern to A (i.e. same indices and indptr)
Returns
out_data, though note that this is the same parameter passed in and overwitten.
'''
assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix"
assert not np.isfortran(B), "B matrix is not stored in row-major form"
assert not np.isfortran(C), "C matrix is not stored in row-major form"
if out is None:
out = A.copy()
if A.dtype == np.float64:
compiled.sparseScalarProductOfDot_f8(A.data, A.indices, A.indptr, B, C, out.data)
elif A.dtype == np.float32:
compiled.sparseScalarProductOfDot_f4(A.data, A.indices, A.indptr, B, C, out.data)
else:
_sparseScalarProductOfDot_py(A,B,C, out)
return out
def generate_itp_pn(lut):
ndim=lut.ndim-1
if ndim == 1: interpolator = interpolators.interpol_1pn
elif ndim == 2: interpolator = interpolators.interpol_2pn
elif ndim == 3: interpolator = interpolators.interpol_3pn
elif ndim == 4: interpolator = interpolators.interpol_4pn
elif ndim == 5: interpolator = interpolators.interpol_5pn
elif ndim == 6: interpolator = interpolators.interpol_6pn
elif ndim >= 7: interpolator = interpolators.interpol_npn
else:
print 'Not implemented'
return
if ndim <= 6:
if np.isfortran(lut): my_lut=lut
else: my_lut=np.asfortranarray(lut)
def function(wo):
return interpolator(wo,my_lut)
else:
if np.isfortran(lut):
flat_lut=np.asfortranarray(lut.reshape((-1,lut.shape[-1]) ,order='C'))
else:
flat_lut=np.asfortranarray(lut.reshape((-1,lut.shape[-1]), order='C'))
shape=np.array(lut.shape)
size=shape[:-1].prod()
def function(wo):
return interpolator(wo,flat_lut,shape[0:-1],shape[-1])
return function
def sparseScalarQuotientOfNormedDot (A, B, C, d, out=None):
'''
Returns A / np.dot(B, C/D), however it does so keeping in mind
the sparsity of A, calculating values only where required.
Params
A - a sparse CSR matrix
B - a dense matrix
C - a dense matrix
d - a dense vector whose dimensionality matches the column-count of C
out - if specified, must be a sparse CSR matrix with identical
non-zero pattern to A (i.e. same indices and indptr)
Returns
out_data, though note that this is the same parameter passed in and overwitten.
'''
assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix"
assert not np.isfortran(B), "B matrix is not stored in row-major form"
assert not np.isfortran(C), "C matrix is not stored in row-major form"
if out is None:
out = A.copy()
if A.dtype == np.float64:
compiled.sparseScalarQuotientOfNormedDot_f8(A.data, A.indices, A.indptr, B, C, d, out.data)
elif A.dtype == np.float32:
compiled.sparseScalarQuotientOfNormedDot_f4(A.data, A.indices, A.indptr, B, C, d, out.data)
else:
raise ValueError ("No implementation for the datatype " + str(A.dtype))
return out
def remove_adjacencies(labels, bwconn):
test = np.zeros((2,2), dtype = np.uint32)
if type(labels) != type(test):
raise Exception('In remove_adjacencies, labels is not a *NumPy* array')
if len(labels.shape) != 3:
raise Exception('In remove_adjacencies, labels is not 3 dimensional')
if not labels.flags.contiguous or np.isfortran(labels):
raise Exception('In remove_adjacencies, labels not contiguous or not C-order')
if labels.dtype != test.dtype:
raise Exception('In remove_adjacencies, labels not uint32')
testB=np.zeros((2,2),dtype=np.bool)
if type(bwconn) != type(testB):
raise Exception( 'In remove_adjacencies, bwconn is not *NumPy* array')
if len(bwconn.shape) != 3:
raise Exception( 'In remove_adjacencies, bwconn is not 3 dimensional')
if not bwconn.flags.contiguous or np.isfortran(bwconn):
raise Exception( 'In remove_adjacencies, bwconn not C-order contiguous')
if bwconn.dtype != testB.dtype:
raise Exception( 'In remove_adjacencies, bwconn not correct data type')
sz = [x+2 for x in labels.shape] # need border for neighborhoods around the edge voxels
lbls = np.zeros(sz, dtype=labels.dtype); lbls[1:-1,1:-1,1:-1] = labels
_pyCext.remove_adjacencies(lbls, bwconn)
return lbls[1:-1,1:-1,1:-1]
def label_affinities(affinities, labels, nextlabel, threshold):
test=np.zeros((2,2)); testI=np.zeros((2,2),dtype=np.uint32)
if type(affinities) != type(test):
raise Exception( 'In label_affinities, affinities is not *NumPy* array')
if len(affinities.shape) != 4:
raise Exception( 'In label_affinities, affinities shape not 4 dimensional')
if affinities.shape[3] != 3:
raise Exception( 'In label_affinities, affinities not 3 dimensional')
if not affinities.flags.contiguous or np.isfortran(affinities):
raise Exception( 'In label_affinities, affinities not C-order contiguous')
if affinities.dtype != np.single:
raise Exception( 'In label_affinities, affinities not single floats')
if type(labels) != type(testI):
raise Exception( 'In label_affinities, labels is not *NumPy* array')
if len(labels.shape) != 3:
raise Exception( 'In label_affinities, labels is not 3 dimensional')
if not labels.flags.contiguous or np.isfortran(labels):
raise Exception( 'In label_affinities, labels not C-order contiguous')
if labels.dtype != np.uint32:
raise Exception( 'In label_affinities, labels not uint32')
if type(threshold) != type(1.0):
raise Exception( 'In label_affinities, threshold argument is not a float')
if type(nextlabel) != type(1):
raise Exception( 'In label_affinities, nextlabel argument is not a integer')
if nextlabel < 1:
raise Exception( 'In label_components, nextlabel argument is less than one')
return _pyCext.label_affinities(affinities,labels,nextlabel,threshold)
def __init__(self, data, block_length=1, use_blocks=None, offsets=None):
"""
data can be a numpy array (in C order), a tuple of such arrays, or a list of such tuples or arrays
"""
self.files = list()
if isinstance(data, list): #Several files
for file in data:
if isinstance(file, tuple):
for d in file:
assert(isinstance(d, np.ndarray) and not np.isfortran(d))
self.files.append(file)
elif isinstance(data, tuple): #Just one file
for d in data:
assert(isinstance(d, np.ndarray) and d.ndim == 2 and not np.isfortran(d))
self.files.append(data)
elif isinstance(data, np.ndarray): #One file with one kind of element only (not input-output)
assert(isinstance(data, np.ndarray) and not np.isfortran(data))
self.files.append(tuple([data]))
# Support for block datapoints
self.block_length = block_length
if block_length == 1:
self.block_lengths = [np.int(1)] * self.get_arity()
self.offsets = [np.int(0)] * self.get_arity()
elif block_length > 1:
self.block_lengths = [np.int(block_length) if ub else np.int(1) for ub in use_blocks] # np.asarray(dtype=np.int) and [np.int(x)] have elements with diff type. Careful!
self.offsets = [np.int(off) for off in offsets]
for ub, off in zip(use_blocks, offsets):
if off != 0 and ub:
raise Exception("Can't have both a block size greater than 1 and an offset.")
else:
raise Exception("Block size must be positive")
def generate_nan_itp(lut):
ndim=lut.ndim
if ndim == 1: interpolator = interpolators.interpol_nan_1
elif ndim == 2: interpolator = interpolators.interpol_nan_2
elif ndim == 3: interpolator = interpolators.interpol_nan_3
elif ndim == 4: interpolator = interpolators.interpol_nan_4
elif ndim == 5: interpolator = interpolators.interpol_5
elif ndim == 6: interpolator = interpolators.interpol_6
elif ndim == 7: interpolator = interpolators.interpol_7
elif ndim >= 8: interpolator = interpolators.interpol_n
else:
print 'Not implemented'
return
if ndim <= 7:
if np.isfortran(lut): my_lut=lut.astype(float)
else: my_lut=np.asfortranarray(lut.astype(float))
def function(wo):
return interpolator(wo,my_lut)
else:
if np.isfortran(lut): flat_lut=lut.astype(float).ravel('C')
else: flat_lut=lut.astype(float).T.ravel('F')
shape=np.array(lut.shape)
size=lut.size
def function(wo):
return interpolator(wo,flat_lut,shape)
return function
def isfortran(self, array):
"""
Same as np.isfortran, but works with lists and tuples.
"""
if isinstance(array,(list,tuple)):
return all([np.isfortran(val) for val in array])
else:
return np.isfortran(array)
def add_dot(c, a, b):
if use_blas and isinstance(c, numpy.ndarray):
if numpy.isfortran(c.T):
scipy.linalg.blas.dgemm(1., b.T, a.T, 1., c.T, overwrite_c=True)
return c
elif numpy.isfortran(c):
scipy.linalg.blas.dgemm(1., a, b, 1., c, overwrite_c=True)
return c
c += numpy.dot(a, b)
return c
def niiToArray(niiImg, c_contiguos=True, canonical=False):
if canonical:
rough_data = nib.as_closest_canonical(niiImg).get_data()
else:
rough_data = niiImg.get_data()
print np.isfortran(rough_data)
#if c_contiguos and np.isfortran(rough_data):
# rough_data = rough_data.T
if c_contiguos and not rough_data.flags['C_CONTIGUOUS']:
rough_data = rough_data.T
return rough_data
def test_ndim_indexing(aggregate_all, ndim, order, outsize=10):
nindices = int(outsize ** ndim)
outshape = tuple([outsize] * ndim)
group_idx = np.random.randint(0, outsize, size=(ndim, nindices))
a = np.random.random(group_idx.shape[1])
res = aggregate_all(group_idx, a, size=outshape, order=order)
if ndim > 1 and order == 'F':
# 1d arrays always return False here
assert np.isfortran(res)
else:
assert not np.isfortran(res)
assert res.shape == outshape
def can_add(self, data):
"""Returns True iff data can be stored.
This usually means it is of the same kind as previously stored arrays.
"""
# Obviously we could just store arbitrary arrays in some implementations (e.g. NPY)
# But lets keep jagged contracts...
template = self.template()
if template is None:
return True
return (template.dtype >= data.dtype and
data.shape[-1] == template.shape[-1] and
np.isfortran(data) == np.isfortran(data))
def add_AB_to_C(A, B, C):
"""
Compute C += AB in-place.
This uses gemm from whatever BLAS is available.
MKL requires Fortran ordered arrays to avoid copies.
Hence we work with transpositions of default c-style arrays.
This function throws error if computation is not in-place.
"""
gemm = sl.get_blas_funcs("gemm", (A, B, C))
assert np.isfortran(C.T) and np.isfortran(A.T) and np.isfortran(B.T)
D = gemm(1.0, B.T, A.T, beta=1, c=C.T, overwrite_c=1)
assert D.base is C or D.base is C.base
def __init__(self, cl_ctx, h0, eta0, u0, v0):
#Make sure that the data is single precision floating point
if (not np.issubdtype(h0.dtype, np.float32) or np.isfortran(h0)):
print "Converting H0"
h0 = h0.astype(np.float32, order='C')
if (not np.issubdtype(eta0.dtype, np.float32) or np.isfortran(eta0)):
print "Converting Eta0"
eta0 = eta0.astype(np.float32, order='C')
if (not np.issubdtype(u0.dtype, np.float32) or np.isfortran(u0)):
print "Converting U0"
u0 = u0.astype(np.float32, order='C')
if (not np.issubdtype(v0.dtype, np.float32) or np.isfortran(v0)):
print "Converting V0"
v0 = v0.astype(np.float32, order='C')
self.ny, self.nx = h0.shape
self.nx = self.nx - 2 # Ghost cells
self.ny = self.ny - 2
assert(h0.shape == (self.ny+2, self.nx+2))
assert(eta0.shape == (self.ny+2, self.nx+2))
assert(u0.shape == (self.ny+2, self.nx+1))
assert(v0.shape == (self.ny+1, self.nx+2))
#Upload data to the device
mf = cl.mem_flags
self.h0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=h0)
self.eta0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=eta0)
self.eta1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=eta0)
self.u0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=u0)
self.u1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=u0)
self.v0 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=v0)
self.v1 = cl.Buffer(cl_ctx, mf.READ_WRITE | mf.COPY_HOST_PTR, hostbuf=v0)
self.h0_pitch = np.int32(h0.shape[1]*4)
self.eta0_pitch = np.int32(eta0.shape[1]*4)
self.eta1_pitch = np.int32(eta0.shape[1]*4)
self.u0_pitch = np.int32(u0.shape[1]*4)
self.u1_pitch = np.int32(u0.shape[1]*4)
self.v0_pitch = np.int32(v0.shape[1]*4)
self.v1_pitch = np.int32(v0.shape[1]*4)
def test_fstecr_fstluk_order(self):
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
fname = '__rpnstd__testfile2__.fst'
try:
os.unlink(fname)
except:
pass
funit = rmn.fstopenall(fname,rmn.FST_RW)
(ig1,ig2,ig3,ig4) = rmn.cxgaig(self.grtyp,self.xg14[0],self.xg14[1],self.xg14[2],self.xg14[3])
(ni,nj) = (90,45)
la = rmn.FST_RDE_META_DEFAULT.copy()
la.update(
{'nomvar' : 'LA',
'typvar' : 'C',
'ni' : ni,
'nj' : nj,
'nk' : 1,
'grtyp' : self.grtyp,
'ig1' : ig1,
'ig2' : ig2,
'ig3' : ig3,
'ig4' : ig4
}
)
lo = la.copy()
lo['nomvar'] = 'LO'
#Note: For the order to be ok in the FSTD file, order='FORTRAN' is mandatory
la['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN')
lo['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN')
for j in xrange(nj):
for i in xrange(ni):
lo['d'][i,j] = 100.+float(i)
la['d'][i,j] = float(j)
rmn.fstecr(funit,la['d'],la)
rmn.fstecr(funit,lo)
rmn.fstcloseall(funit)
funit = rmn.fstopenall(fname,rmn.FST_RW)
kla = rmn.fstinf(funit,nomvar='LA')['key']
la2 = rmn.fstluk(kla)#,rank=2)
klo = rmn.fstinf(funit,nomvar='LO')['key']
lo2 = rmn.fstluk(klo)#,rank=2)
rmn.fstcloseall(funit)
try:
os.unlink(fname)
except:
pass
self.assertTrue(np.isfortran(la2['d']))
self.assertTrue(np.isfortran(lo2['d']))
self.assertFalse(np.any(np.fabs(la2['d'] - la['d']) > self.epsilon))
self.assertFalse(np.any(np.fabs(lo2['d'] - lo['d']) > self.epsilon))
def __init__(self, ndarray):
self.metadata = _ndarray_meta(ndarray)
self.size = ndarray.size * ndarray.itemsize
# The following is guesswork
# non contiguous fortran array (if ever such a thing exists)
if numpy.isfortran(ndarray) and not ndarray.flags['F_CONTIGUOUS']:
self.ndarray = numpy.asfortranarray(ndarray)
# non contiguous C array
elif not numpy.isfortran(ndarray) and not ndarray.flags['C_CONTIGUOUS']:
self.ndarray = numpy.ascontiguousarray(ndarray)
# contiguous fortran or C array, do nothing
else:
self.ndarray = ndarray
self.ptr = self.ndarray.__array_interface__['data'][0]
def test_cmp_ndim(aggregate_cmp, ndim, order, outsize=100, decimal=14):
nindices = int(outsize ** ndim)
outshape = tuple([outsize] * ndim)
group_idx = np.random.randint(0, outsize, size=(ndim, nindices))
a = np.random.random(group_idx.shape[1])
res = aggregate_cmp.func(group_idx, a, size=outshape, order=order)
ref = aggregate_cmp.func_ref(group_idx, a, size=outshape, order=order)
if ndim > 1 and order == 'F':
# 1d arrays always return False here
assert np.isfortran(res)
else:
assert not np.isfortran(res)
assert res.shape == outshape
np.testing.assert_array_almost_equal(res, ref, decimal=decimal)
def add_outer(a, x, y):
"""Add the outer product of x and y to a, possibly overwriting a.
a = add_outer(a, x, y) is equivalent to a += numpy.outer(x, y)."""
if use_blas and isinstance(a, numpy.ndarray):
if numpy.isfortran(a.T):
scipy.linalg.blas.dger(1., y, x, a=a.T, overwrite_a=1)
return a
elif numpy.isfortran(a):
scipy.linalg.blas.dger(1., x, y, a=a, overwrite_a=1)
return a
# einsum is written in C and is faster than outer
a += numpy.einsum('i,j->ij', x, y)
return a
def verify_is_c_contiguous_and_is_not_fortran(x):
if isinstance(x,CpuGpuArray):
raise TypeError(type(x))
if np.isfortran(x):
raise ValueError("Must be 'C' order")
if not x.flags.c_contiguous:
raise ValueError("Must be 'C'-contiguous")
def tolocal(self, comm=MPI.COMM_WORLD):
"""Scatter a global image into local ones."""
if hasattr(self, 'shape_global') and self.shape != self.shape_global or\
hasattr(self, 'comm') and self.comm.Get_size() > 1:
raise ValueError('This array is not a global image.')
order = 'f' if np.isfortran(self) else 'c'
if hasattr(self, 'empty'):
output = self.empty(self.shape, dtype=self.dtype, order=order,
comm=comm)
else:
shape = split_shape(self.shape, comm)
output = np.empty(shape, dtype=self.dtype, order=order)
output = DistributedArray.__new__(DistributedArray, output,
self.shape, comm)
if hasattr(self, '__dict__'):
for k,v in self.__dict__.items():
setattr(output, k, v)
output.comm = comm
s = split_work(self.shape[0], comm=comm)
n = s.stop - s.start
output[0:n] = self[s.start:s.stop]
if n < output.shape[0]:
output[n:] = 0
return output
def __check_image_c_order(img):
"""Raise if the image is not in FORTRAN memory order"""
# Check array memory order
if np.isfortran(img): # i.e. not C-ordered
raise ValueError("cv_algorithms works only on C-ordered arrays")
if not img.flags['C_CONTIGUOUS']:
raise ValueError("cv_algorithms works only on contiguous arrays")
def set_array(dataobject, newarray):
# Ensure we have Fortran ordered flat array to assign to image data. This
# is ideally done without additional copies, but if C order we must copy.
if np.isfortran(newarray):
arr = newarray.reshape(-1, order='F')
else:
print 'Warning, array does not have Fortran order, making deep copy and fixing...'
tmp = np.asfortranarray(newarray)
arr = tmp.reshape(-1, order='F')
print '...done.'
# Set the extents (they may have changed).
dataobject.SetExtent(0, newarray.shape[0] - 1,
0, newarray.shape[1] - 1,
0, newarray.shape[2] - 1)
# Now replace the scalars array with the new array.
vtkarray = np_s.numpy_to_vtk(arr)
vtkarray.Association = dsa.ArrayAssociation.POINT
do = dsa.WrapDataObject(dataobject)
oldscalars = do.PointData.GetScalars()
name = oldscalars.GetName()
del oldscalars
do.PointData.append(arr, name)
do.PointData.SetActiveScalars(name)
def test_as_float_array():
# Test function for as_float_array
X = np.ones((3, 10), dtype=np.int32)
X = X + np.arange(10, dtype=np.int32)
# Checks that the return type is ok
X2 = as_float_array(X, copy=False)
np.testing.assert_equal(X2.dtype, np.float32)
# Another test
X = X.astype(np.int64)
X2 = as_float_array(X, copy=True)
# Checking that the array wasn't overwritten
assert_true(as_float_array(X, False) is not X)
# Checking that the new type is ok
np.testing.assert_equal(X2.dtype, np.float64)
# Here, X is of the right type, it shouldn't be modified
X = np.ones((3, 2), dtype=np.float32)
assert_true(as_float_array(X, copy=False) is X)
# Test that if X is fortran ordered it stays
X = np.asfortranarray(X)
assert_true(np.isfortran(as_float_array(X, copy=True)))
# Test the copy parameter with some matrices
matrices = [
np.matrix(np.arange(5)),
sp.csc_matrix(np.arange(5)).toarray(),
sparse_random_matrix(10, 10, density=0.10).toarray()
]
for M in matrices:
N = as_float_array(M, copy=True)
N[0, 0] = np.nan
assert_false(np.isnan(M).any())
def standard_traj(traj,particles=None,dimensions=None):
if type(traj) not in [numpy.ndarray]:
traj=numpy.array(traj,dtype=float,order='F')
if len(traj.shape)==1:
traj.resize(traj.shape[0],1,1)
elif len(traj.shape)==2:
if particles==None and dimensions==None:
nn=str(traj.shape[-1])
print '# Error! The trajectory format should be [frames,particles,dimensions].'
print '# Your input has the shape ['+str(traj.shape[0])+','+str(traj.shape[1])+']:'
print '# '+nn+' particles with a reaction coordinate? or just a '+nn+'-D reaction coordinate?'
print '# '
print '# Please, make use of the variables "particles" or/and "dimensions":'
print '# traj:=[100 frames, 3 dimensions] --> "particles=1" or/and "dimensions=3"'
print '# traj:=[100 frames, 8 particles] --> "particles=8" or/and "dimensions=1"'
print '# '
return 0
elif particles==1 or dimensions>1:
traj.resize(traj.shape[0],1,traj.shape[1])
elif particles>1 or dimensions==1:
traj.resize(traj.shape[0],traj.shape[1],1)
elif particles==1 and dimensions==1:
traj.resize(traj.shape[0],traj.shape[1],1)
if not numpy.isfortran(traj):
traj=numpy.array(traj,dtype=float,order='F')
return traj
def binary_neighbours(img):
"""
Takes a binary image and, for each pixel, computes
which surrounding pixels are non-zero.
Depdending on those pixels, bits in the uint8 output
array are set or unset
Parameters
==========
img : numpy array-like
A grayscale image that is assumed to be binary
(every non-zero value is interpreted as 0).
Usually this is a pre-thinned image.
Returns
=======
A uint8-type output array the same shape of img,
where the following bits are set or unset,
if the respective neighbouring pixel is set or unset.
Bit index by position:
0 1 2
3 4
5 6 7
Note that for Numpy due to the coordinate system,
the respective pixels can be accessed like this:
[y-1,x-1] [y-1,x] [y-1,x+1]
[y,x-1] [y,x] [y,x+1]
[y+1,x-1] [y+1,x] [y+1,x+1]
This is equivalent to the coordinate system when displaying
the image using matplotlib imshow
The positions in this matrix correspond to the bit number
shown above, e.g. bit #4 is (1 << 4) ORed to the result.
"""
# Check if image has the correct type
__check_image_grayscale_2d(img)
img = force_c_order_contiguous(img)
__check_array_uint8(img)
height, width = img.shape
# Allocate output array
# uint32 is used so there is no overflow for large inputs
out = np.zeros(img.shape, dtype=np.uint8, order="C")
assert not np.isfortran(out)
# Extract pointer to binary data
srcptr = _ffi.cast("uint8_t*", img.ctypes.data)
dstptr = _ffi.cast("uint8_t*", out.ctypes.data)
rc = _libcv_algorithms.binary_neighbours(dstptr, srcptr, width, height)
if rc != 0:
raise ValueError("Internal error (return code {0}) in algorithm C code".format(rc))
return out
def convert_to_float32(data):
"""
Converts to C-style float 32 array suitable for the GPU/CUDA
"""
if (not np.issubdtype(data.dtype, np.float32) or np.isfortran(data)):
return data.astype(np.float32, order='C')
else:
return data
def sparseScalarProductOfSafeLnDot (A, B, C, out=None, start=None, end=None):
'''
Returns A * np.log(np.dot(B, C)), however it does so keeping in
mind the sparsity of A, calculating values only where required.
Moreover if any product of the dot is zero, it's replaced with
the minimum non-zero value allowed by the datatype, to avoid NaNs
Params
A - a sparse CSR matrix
B - a dense matrix
C - a dense matrix
out - if specified, must be a sparse CSR matrix with identical
non-zero pattern to A (i.e. same indices and indptr)
start - where to start indexing A
end - where to stop indexing A
Returns
out_data, though note that this is the same parameter passed in and overwitten.
'''
assert ssp.isspmatrix_csr(A), "A matrix is not a CSR matrix"
assert not np.isfortran(B), "B matrix is not stored in row-major form"
assert not np.isfortran(C), "C matrix is not stored in row-major form"
if start is None:
start = 0
if end is None:
end = A.shape[0]
if out is None:
out = A[start:end,:].copy()
if (start, end) == (0, A.shape[0]) and A.dtype == np.float64:
out = A.copy()
oldWay = np.sum(compiled.sparseScalarProductOfSafeLnDot_f8_full(
A.data, A.indices, A.indptr, B, C, out.data))
out = A.copy()
newWay = np.sum(compiled.sparseScalarProductOfSafeLnDot_f8(
A.data, A.indices, A.indptr, B, C, out.data, 0, A.shape[0]))
#print ("The old way was " + str(oldWay) + " and the new way was " + str(newWay))
elif A.dtype == np.float64:
compiled.sparseScalarProductOfSafeLnDot_f8(A.data, A.indices, A.indptr, B, C, out.data, start, end)
elif A.dtype == np.float32:
compiled.sparseScalarProductOfSafeLnDot_f4(A.data, A.indices, A.indptr, B, C, out.data, start, end)
else:
_sparseScalarProductOfSafeLnDot_py(A,B,C, out)
return out
def info(Z):
import sys
import numpy as np
endianness = {'=': 'native (%s)' % sys.byteorder,
'<': 'little',
'>': 'big',
'|': 'not applicable'}
print("------------------------------")
print("Interface (item)")
print(" shape: ", Z.shape)
print(" dtype: ", Z.dtype)
print(" length: ", len(Z))
print(" size: ", Z.size)
print(" endianness: ", endianness[Z.dtype.byteorder])
if np.isfortran(Z):
print(" order: ☐ C ☑ Fortran")
else:
print(" order: ☑ C ☐ Fortran")
print("")
print("Memory (byte)")
print(" item size: ", Z.itemsize)
print(" array size: ", Z.size*Z.itemsize)
print(" strides: ", Z.strides)
print("")
print("Properties")
if Z.flags["OWNDATA"]:
print(" own data: ☑ Yes ☐ No")
else:
print(" own data: ☐ Yes ☑ No")
if Z.flags["WRITEABLE"]:
print(" writeable: ☑ Yes ☐ No")
else:
print(" writeable: ☐ Yes ☑ No")
if np.isfortran(Z) and Z.flags["F_CONTIGUOUS"]:
print(" contiguous: ☑ Yes ☐ No")
elif not np.isfortran(Z) and Z.flags["C_CONTIGUOUS"]:
print(" contiguous: ☑ Yes ☐ No")
else:
print(" contiguous: ☐ Yes ☑ No")
if Z.flags["ALIGNED"]:
print(" aligned: ☑ Yes ☐ No")
else:
print(" aligned: ☐ Yes ☑ No")
print("------------------------------")
print()
def type_components(labels, voxel_type, supervoxel_type, voxel_out_type, num_types=2):
test=np.zeros((2,2),dtype=np.uint32)
if type(labels) != type(test):
raise Exception( 'In type_components, labels is not *NumPy* array')
if len(labels.shape) != 3:
raise Exception( 'In type_components, labels is not 3 dimensional')
if not labels.flags.contiguous or np.isfortran(labels):
raise Exception( 'In type_components, labels not C-order contiguous')
if labels.dtype != np.uint32:
raise Exception( 'In type_components, labels not uint32')
if type(voxel_type) != type(test):
raise Exception( 'In type_components, voxel_type is not *NumPy* array')
if len(voxel_type.shape) != 3:
raise Exception( 'In type_components, voxel_type is not 3 dimensional')
if not voxel_type.flags.contiguous or np.isfortran(voxel_type):
raise Exception( 'In type_components, voxel_type not C-order contiguous')
if voxel_type.dtype != np.uint8:
raise Exception( 'In type_components, voxel_type not uint8')
if type(supervoxel_type) != type(test):
raise Exception( 'In type_components, supervoxel_type is not *NumPy* array')
if len(supervoxel_type.shape) != 1:
raise Exception( 'In type_components, supervoxel_type is not array')
if not supervoxel_type.flags.contiguous:
raise Exception( 'In type_components, supervoxel_type is not contiguous')
if supervoxel_type.dtype != np.uint8:
raise Exception( 'In type_components, supervoxel_type not uint8')
if not np.array_equal(labels.shape, voxel_type.shape):
raise Exception( 'In type_components, labels and voxel_type not same shape')
if type(voxel_out_type) != type(test):
raise Exception( 'In type_components, voxel_out_type is not *NumPy* array')
if len(voxel_out_type.shape) != 3:
raise Exception( 'In type_components, voxel_out_type is not 3 dimensional')
if not voxel_out_type.flags.contiguous or np.isfortran(voxel_out_type):
raise Exception( 'In type_components, voxel_out_type not C-order contiguous')
if voxel_out_type.dtype != np.uint8:
raise Exception( 'In type_components, voxel_out_type not uint8')
# max operation is slow, assumer caller did this correctly... segfault'able tho if wrong
#if supervoxel_type.size != labels.max():
# raise Exception( 'In type_components, supervoxel_type size not equal to number of supervoxels in labels')
if type(num_types) != type(1):
raise Exception( 'In type_components, num_types argument is not an integer')
if num_types < 2:
raise Exception( 'In type_components, num_types < 2, need at least two types')
return _pyCext.type_components(labels, voxel_type, supervoxel_type, voxel_out_type, num_types)
def verify_is_c_contiguous_and_is_not_fortran(cls,arr):
"""
arr is a numpy array.
"""
if np.isfortran(arr):
raise ValueError("Must be 'C' order")
if not arr.flags.c_contiguous:
msg="Must be 'C'-contiguous. Consider passing arr.copy() instead."
raise ValueError(msg)
def _initialize_with_attributes(self,
coordinates=None,
box=None,
cell=None,
timestep=None,
integstep=None,
step=None,
time=None):
self.coordinates = coordinates
self.box = box
self.cell = cell
self.timestep = timestep
self.integstep = integstep
self.step = step
self.time = time
if box is not None:
if box[0] is None:
self.box = None
if cell is not None:
if cell[0] is None:
self.cell = None
if self.coordinates is not None:
self.n_structures = self.coordinates.shape[0]
self.n_atoms = self.coordinates.shape[1]
ii = self.coordinates
if ii is not None:
if _np.isfortran(ii) == False:
ii = _np.asfortranarray(ii)
ii = self.box
if ii is not None:
if _np.isfortran(ii) == False:
ii = _np.asfortranarray(ii)
ii = self.cell
if ii is not None:
if _np.isfortran(ii) == False:
ii = _np.asfortranarray(ii)
ii = self.time
if ii is not None:
if _np.isfortran(ii) == False:
ii = _np.asfortranarray(ii)
ii = self.step
if ii is not None:
if _np.isfortran(ii) == False:
ii = _np.asfortranarray(ii)
if (self.cell is None) and (self.box is not None):
self.box2cell()
if (self.cell is not None) and (self.box is None):
self.cell2box()
if (self.box is not None):
self.box2invbox()
pass
def norm(a, ord=None):
"""
Matrix or vector norm.
This function is able to return one of seven different matrix norms,
or one of an infinite number of vector norms (described below), depending
on the value of the ``ord`` parameter.
Parameters
----------
a : (M,) or (M, N) array_like
Input array.
ord : {non-zero int, inf, -inf, 'fro'}, optional
Order of the norm (see table under ``Notes``). inf means numpy's
`inf` object.
Returns
-------
norm : float
Norm of the matrix or vector.
Notes
-----
For values of ``ord <= 0``, the result is, strictly speaking, not a
mathematical 'norm', but it may still be useful for various numerical
purposes.
The following norms can be calculated:
===== ============================ ==========================
ord norm for matrices norm for vectors
===== ============================ ==========================
None Frobenius norm 2-norm
'fro' Frobenius norm --
inf max(sum(abs(x), axis=1)) max(abs(x))
-inf min(sum(abs(x), axis=1)) min(abs(x))
0 -- sum(x != 0)
1 max(sum(abs(x), axis=0)) as below
-1 min(sum(abs(x), axis=0)) as below
2 2-norm (largest sing. value) as below
-2 smallest singular value as below
other -- sum(abs(x)**ord)**(1./ord)
===== ============================ ==========================
The Frobenius norm is given by [1]_:
:math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}`
References
----------
.. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,
Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15
Examples
--------
>>> from scipy.linalg import norm
>>> a = np.arange(9) - 4
>>> a
array([-4, -3, -2, -1, 0, 1, 2, 3, 4])
>>> b = a.reshape((3, 3))
>>> b
array([[-4, -3, -2],
[-1, 0, 1],
[ 2, 3, 4]])
>>> norm(a)
7.745966692414834
>>> norm(b)
7.745966692414834
>>> norm(b, 'fro')
7.745966692414834
>>> norm(a, np.inf)
4
>>> norm(b, np.inf)
9
>>> norm(a, -np.inf)
0
>>> norm(b, -np.inf)
2
>>> norm(a, 1)
20
>>> norm(b, 1)
7
>>> norm(a, -1)
-4.6566128774142013e-010
>>> norm(b, -1)
6
>>> norm(a, 2)
7.745966692414834
>>> norm(b, 2)
7.3484692283495345
>>> norm(a, -2)
nan
>>> norm(b, -2)
1.8570331885190563e-016
>>> norm(a, 3)
5.8480354764257312
>>> norm(a, -3)
nan
"""
# Differs from numpy only in non-finite handling and the use of blas.
a = np.asarray_chkfinite(a)
if a.dtype.char in 'fdFD':
if ord in (None, 2) and (a.ndim == 1):
# use blas for fast and stable euclidean norm
nrm2 = get_blas_funcs('nrm2', dtype=a.dtype)
return nrm2(a)
if a.ndim == 2:
# Use lapack for a couple fast matrix norms.
# For some reason the *lange frobenius norm is slow.
lange_args = None
if ord == 1:
if np.isfortran(a):
lange_args = '1', a
elif np.isfortran(a.T):
lange_args = 'i', a.T
elif ord == np.inf:
if np.isfortran(a):
lange_args = 'i', a
elif np.isfortran(a.T):
lange_args = '1', a.T
if lange_args:
lange = get_lapack_funcs('lange', dtype=a.dtype)
return lange(*lange_args)
return np.linalg.norm(a, ord=ord)
def _build_tree(X,
y,
is_classification,
criterion,
max_depth,
min_split,
min_density,
max_features,
random_state,
n_classes,
find_split,
sample_mask=None,
X_argsorted=None):
"""Build a tree by recursively partitioning the data."""
if max_depth <= 10:
init_capacity = (2**(max_depth + 1)) - 1
else:
init_capacity = 2047 # num nodes of tree with depth 10
tree = Tree(n_classes, init_capacity)
# Recursively partition X
def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent,
is_left_child):
# Count samples
n_node_samples = sample_mask.sum()
if n_node_samples == 0:
raise ValueError("Attempting to find a split "
"with an empty sample_mask")
# Split samples
if depth < max_depth and n_node_samples >= min_split:
feature, threshold, best_error, init_error = find_split(
X, y, X_argsorted, sample_mask, n_node_samples, max_features,
criterion, random_state)
else:
feature = -1
# Value at this node
current_y = y[sample_mask]
if is_classification:
value = np.zeros((n_classes, ))
t = current_y.max() + 1
value[:t] = np.bincount(current_y.astype(np.int))
else:
value = np.asarray(np.mean(current_y))
# Terminal node
if feature == -1:
# compute error at leaf
error = _tree._error_at_leaf(y, sample_mask, criterion,
n_node_samples)
tree.add_leaf(parent, is_left_child, value, error, n_node_samples)
# Internal node
else:
# Sample mask is too sparse?
if n_node_samples / X.shape[0] <= min_density:
X = X[sample_mask]
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
y = current_y
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
# Split and and recurse
split = X[:, feature] <= threshold
node_id = tree.add_split_node(parent, is_left_child, feature,
threshold, best_error, init_error,
n_node_samples, value)
# left child recursion
recursive_partition(X, X_argsorted, y, split & sample_mask,
depth + 1, node_id, True)
# right child recursion
recursive_partition(X, X_argsorted, y, ~split & sample_mask,
depth + 1, node_id, False)
# Launch the construction
if X.dtype != DTYPE or not np.isfortran(X):
X = np.asanyarray(X, dtype=DTYPE, order="F")
if y.dtype != DTYPE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DTYPE)
if sample_mask is None:
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
if X_argsorted is None:
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False)
tree.resize(tree.node_count)
return tree
def _build_tree(X,
y,
criterion,
max_depth,
min_samples_split,
min_samples_leaf,
min_density,
max_features,
random_state,
n_classes,
find_split,
sample_mask=None,
X_argsorted=None,
store_terminal_region=False):
"""Build a tree by recursively partitioning the data. """
if max_depth <= 10:
init_capacity = (2**(max_depth + 1)) - 1
else:
init_capacity = 2047 # num nodes of tree with depth 10
tree = Tree(n_classes, init_capacity)
# Recursively partition X
def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent,
is_left_child, sample_indices):
# Count samples
n_node_samples = sample_mask.sum()
if n_node_samples == 0:
raise ValueError("Attempting to find a split "
"with an empty sample_mask")
# Split samples
if depth < max_depth and n_node_samples >= min_samples_split \
and n_node_samples >= 2 * min_samples_leaf:
feature, threshold, best_error, init_error = find_split(
X, y, X_argsorted, sample_mask, n_node_samples,
min_samples_leaf, max_features, criterion, random_state)
else:
feature = -1
init_error = _tree._error_at_leaf(y, sample_mask, criterion,
n_node_samples)
value = criterion.init_value()
# Current node is leaf
if feature == -1:
node_id = tree.add_leaf(parent, is_left_child, value, init_error,
n_node_samples)
if store_terminal_region:
# remember which samples ended up in this leaf
tree.terminal_region[sample_indices[sample_mask]] = node_id
# Current node is internal node (= split node)
else:
# Sample mask is too sparse?
if n_node_samples / X.shape[0] <= min_density:
X = X[sample_mask]
sample_indices = sample_indices[sample_mask]
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
y = y[sample_mask]
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
# Split and and recurse
split = X[:, feature] <= threshold
node_id = tree.add_split_node(parent, is_left_child, feature,
threshold, best_error, init_error,
n_node_samples, value)
# left child recursion
recursive_partition(X, X_argsorted, y,
np.logical_and(split, sample_mask), depth + 1,
node_id, True, sample_indices)
# right child recursion
recursive_partition(
X, X_argsorted, y,
np.logical_and(np.logical_not(split), sample_mask), depth + 1,
node_id, False, sample_indices)
# setup auxiliary data structures and check input before
# recursive partitioning
if store_terminal_region:
tree.terminal_region = np.empty((X.shape[0], ), dtype=np.int32)
tree.terminal_region.fill(-1)
if X.dtype != DTYPE or not np.isfortran(X):
X = np.asanyarray(X, dtype=DTYPE, order="F")
if y.dtype != DTYPE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DTYPE)
if sample_mask is None:
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
if X_argsorted is None:
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
sample_indices = np.arange(X.shape[0])
# build the tree by recursive partitioning
recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False,
sample_indices)
# compactify the tree data structure
tree.resize(tree.node_count)
return tree
def _fit(self, data):
"""Actual method calling the underlying fitting implementation."""
data_ord = ord('c' if np.isfortran(data) else 'r')
data, c_data_ptr, data_ctype = self._to_cdata(data)
if self.init == "random" or self.init == "k-means++":
c_init = 1
else:
c_init = 0
if self.init_data == "random":
c_init_data = 0
elif self.init_data == "selectstrat":
c_init_data = 1
elif self.init_data == "randomselect":
c_init_data = 2
else:
print("""
Unknown init_data "%s", should be
"random", "selectstrat" or "randomselect".
""" % self.init_data)
sys.stdout.flush()
return
pred_centers = c_void_p(0)
pred_labels = c_void_p(0)
lib = self._load_lib()
rows = np.shape(data)[0]
cols = np.shape(data)[1]
if self.double_precision == 0:
status = lib.make_ptr_float_kmeans(
0, self.verbose,
self.random_state, self._gpu_id, self.n_gpus, rows, cols,
c_int(data_ord), self._n_clusters, self._max_iter, c_init,
c_init_data, self.tol, c_data_ptr, None, pointer(pred_centers),
pointer(pred_labels))
else:
status = lib.make_ptr_double_kmeans(
0, self.verbose,
self.random_state, self._gpu_id, self.n_gpus, rows, cols,
c_int(data_ord), self._n_clusters, self._max_iter, c_init,
c_init_data, self.tol, c_data_ptr, None, pointer(pred_centers),
pointer(pred_labels))
if status:
raise ValueError('KMeans failed in C++ library.')
centroids = np.fromiter(cast(pred_centers, POINTER(data_ctype)),
dtype=data_ctype,
count=self._n_clusters * cols)
centroids = np.reshape(centroids, (self._n_clusters, cols))
if np.isnan(centroids).any():
centroids = centroids[~np.isnan(centroids).any(axis=1)]
self._print_verbose(
0, "Removed %d empty centroids" %
(self._n_clusters - centroids.shape[0]))
self._n_clusters = centroids.shape[0]
self.cluster_centers_ = centroids
labels = np.fromiter(cast(pred_labels, POINTER(c_int)),
dtype=np.int32,
count=rows)
self.labels_ = np.reshape(labels, rows)
return self.cluster_centers_, self.labels_
def elastic_net(predictors,
target,
balance,
memlimit=None,
largest=None,
**kwargs):
"""
Raw-output wrapper for elastic net linear regression.
"""
# Mandatory parameters
predictors = np.asanyarray(predictors)
target = np.asanyarray(target)
# Decide on largest allowable models for memory/convergence.
memlimit = predictors.shape[1] if memlimit is None else memlimit
# If largest isn't specified use memlimit.
largest = memlimit if largest is None else largest
if memlimit < largest:
raise ValueError('Need largest <= memlimit')
# Flags determining overwrite behavior
overwrite_pred_ok = False
overwrite_targ_ok = False
thr = _DEFAULT_THRESH # Minimum change in largest coefficient
weights = None # Relative weighting per observation case
vp = None # Relative penalties per predictor (0 = no penalty)
isd = True # Standardize input variables before proceeding?
jd = np.zeros(1) # Predictors to exclude altogether from fitting
ulam = None # User-specified lambda values
flmin = _DEFAULT_FLMIN # Fraction of largest lambda at which to stop
nlam = _DEFAULT_NLAM # The (maximum) number of lambdas to try.
for keyword in kwargs:
if keyword == 'overwrite_pred_ok':
overwrite_pred_ok = kwargs[keyword]
elif keyword == 'overwrite_targ_ok':
overwrite_targ_ok = kwargs[keyword]
elif keyword == 'threshold':
thr = kwargs[keyword]
elif keyword == 'weights':
weights = np.asarray(kwargs[keyword]).copy()
elif keyword == 'penalties':
vp = kwargs[keyword].copy()
elif keyword == 'standardize':
isd = bool(kwargs[keyword])
elif keyword == 'exclude':
# Add one since Fortran indices start at 1
exclude = (np.asarray(kwargs[keyword]) + 1).tolist()
jd = np.array([len(exclude)] + exclude)
elif keyword == 'lambdas':
if 'flmin' in kwargs:
raise ValueError("Can't specify both lambdas & flmin keywords")
ulam = np.asarray(kwargs[keyword])
flmin = 2. # Pass flmin > 1.0 indicating to use the user-supplied.
nlam = len(ulam)
elif keyword == 'flmin':
flmin = kwargs[keyword]
ulam = None
elif keyword == 'nlam':
if 'lambdas' in kwargs:
raise ValueError("Can't specify both lambdas & nlam keywords")
nlam = kwargs[keyword]
else:
raise ValueError("Unknown keyword argument '%s'" % keyword)
# If predictors is a Fortran contiguous array, it will be overwritten.
# Decide whether we want this. If it's not Fortran contiguous it will
# be copied into that form anyway so there's no chance of overwriting.
if np.isfortran(predictors):
if not overwrite_pred_ok:
# Might as well make it F-ordered to avoid ANOTHER copy.
predictors = predictors.copy(order='F')
# target being a 1-dimensional array will usually be overwritten
# with the standardized version unless we take steps to copy it.
if not overwrite_targ_ok:
target = target.copy()
# Uniform weighting if no weights are specified.
if weights is None:
weights = np.ones(predictors.shape[0])
# Uniform penalties if none were specified.
if vp is None:
vp = np.ones(predictors.shape[1])
# Call the Fortran wrapper.
lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr = \
_glmnet.elnet(balance, predictors, target, weights, jd, vp,
memlimit, flmin, ulam, thr, nlam=nlam)
# Check for errors, documented in glmnet.f.
if jerr != 0:
if jerr == 10000:
raise ValueError('cannot have max(vp) < 0.0')
elif jerr == 7777:
raise ValueError('all used predictors have 0 variance')
elif jerr < 7777:
raise MemoryError('elnet() returned error code %d' % jerr)
else:
raise Exception('unknown error: %d' % jerr)
return lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr
#a=(array[:]).reshape(224,224)
# Unpacking the array from the vector shape
imageTmp = np.reshape(array,
(imageSize1, imageSize2), order='F') / 255.0
#imageTmp = np.reshape(array, (imageSize1, imageSize2), order='F')
cv2.imshow('BOSS image', imageTmp)
#cv2.waitKey(0)
#imageTmp = imageTmp.astype(np.uint8)
#imageTmp = skimage.exposure.rescale_intensity(imageTmp * 1.0, out_range=np.float32)
im = Image.fromarray(imageTmp)
#im.show()
print(np.isfortran(imageTmp))
#
#print(image.shape)
image = np.zeros((imageSize1, imageSize2, 3), float, 'C')
image = np.stack((imageTmp, ) * 3, -1)
'''
image[:, :, 0] = imageTmp
image[:, :, 1] = imageTmp
image[:, :, 2] = imageTmp
'''
'''
image[:, :, 1] = imageTmp
image[:, :, 2] = imageTmp
'''
def solver_group(X,
y,
alpha,
grp_size,
max_iter=10000,
tol=1e-4,
f_gap=10,
K=5,
use_acc=False,
algo='bcd'):
"""Solve the GroupLasso with BCD/ISTA/FISTA, eventually with extrapolation.
Groups are contiguous, of size grp_size.
Objective:
norm(y - Xw, ord=2)**2 / 2 + alpha * sum_g ||w_{[g]}||_2
Parameters:
algo: string
'bcd', 'pgd', 'fista'
alpha: strength of the group penalty
"""
is_sparse = sparse.issparse(X)
n_features = X.shape[1]
if n_features % grp_size != 0:
raise ValueError("n_features is not a multiple of group size")
n_groups = n_features // grp_size
if not is_sparse and not np.isfortran(X):
X = np.asfortranarray(X)
last_K_w = np.zeros([K + 1, n_features])
U = np.zeros([K, n_features])
if algo in ('pgd', 'fista'):
if is_sparse:
L = power_method(X, max_iter=1000)**2
else:
L = norm(X, ord=2)**2
lc = np.zeros(n_groups)
for g in range(n_groups):
X_g = X[:, g * grp_size:(g + 1) * grp_size]
if is_sparse:
gram = (X_g.T @ X_g).todense()
lc[g] = norm(gram, ord=2)
else:
lc[g] = norm(X_g, ord=2)**2
w = np.zeros(n_features)
if algo == 'fista':
z = np.zeros(n_features)
t_new = 1
R = y.copy()
E = []
gaps = np.zeros(max_iter // f_gap)
for it in range(max_iter):
if it % f_gap == 0:
if algo == 'fista':
R = y - X @ w
p_obj = primal_grp(R, w, alpha, grp_size)
E.append(p_obj)
theta = R / alpha
d_norm_theta = np.max(
norm((X.T @ theta).reshape(-1, grp_size), axis=1))
if d_norm_theta > 1.:
theta /= d_norm_theta
d_obj = dual_lasso(y, theta, alpha)
gap = p_obj - d_obj
print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" %
(it, p_obj, d_obj, gap))
gaps[it // f_gap] = gap
if gap < tol:
print("Early exit")
break
if algo == 'bcd':
if is_sparse:
_bcd_sparse(X.data, X.indices, X.indptr, w, R, alpha, lc)
else:
_bcd(X, w, R, alpha, lc)
elif algo == 'pgd':
w[:] = BST_vec(w + 1. / L * X.T @ R, alpha / L, grp_size)
R[:] = y - X @ w
elif algo == 'fista':
w_old = w.copy()
w[:] = BST_vec(z - X.T @ (X @ z - y) / L, alpha / L, grp_size)
t_old = t_new
t_new = (1. + np.sqrt(1 + 4 * t_old**2)) / 2.
z[:] = w + (t_old - 1.) / t_new * (w - w_old)
else:
raise ValueError("Unknown algo %s" % algo)
if use_acc:
if it < K + 1:
last_K_w[it] = w
else:
for k in range(K):
last_K_w[k] = last_K_w[k + 1]
last_K_w[K - 1] = w
for k in range(K):
U[k] = last_K_w[k + 1] - last_K_w[k]
C = np.dot(U, U.T)
try:
z = np.linalg.solve(C, np.ones(K))
c = z / z.sum()
w_acc = np.sum(last_K_w[:-1] * c[:, None], axis=0)
p_obj = primal_grp(R, w, alpha, grp_size)
R_acc = y - X @ w_acc
p_obj_acc = primal_grp(R_acc, w_acc, alpha, grp_size)
if p_obj_acc < p_obj:
w = w_acc
R = R_acc
except np.linalg.LinAlgError:
print("----------Linalg error")
return w, np.array(E), gaps[:it // f_gap + 1]
def sync(data, idx, aggregate=None, pad=True, axis=-1):
"""Synchronous aggregation of a multi-dimensional array between boundaries
.. note::
In order to ensure total coverage, boundary points may be added
to `idx`.
If synchronizing a feature matrix against beat tracker output, ensure
that frame index numbers are properly aligned and use the same hop length.
Parameters
----------
data : np.ndarray
multi-dimensional array of features
idx : iterable of ints or slices
Either an ordered array of boundary indices, or
an iterable collection of slice objects.
aggregate : function
aggregation function (default: `np.mean`)
pad : boolean
If `True`, `idx` is padded to span the full range `[0, data.shape[axis]]`
axis : int
The axis along which to aggregate data
Returns
-------
data_sync : ndarray
`data_sync` will have the same dimension as `data`, except that the `axis`
coordinate will be reduced according to `idx`.
For example, a 2-dimensional `data` with `axis=-1` should satisfy
`data_sync[:, i] = aggregate(data[:, idx[i-1]:idx[i]], axis=-1)`
Raises
------
ParameterError
If the index set is not of consistent type (all slices or all integers)
Examples
--------
Beat-synchronous CQT spectra
>>> y, sr = librosa.load(librosa.util.example_audio_file())
>>> tempo, beats = librosa.beat.beat_track(y=y, sr=sr, trim=False)
>>> cqt = librosa.cqt(y=y, sr=sr)
>>> beats = librosa.util.fix_frames(beats, x_max=cqt.shape[1])
By default, use mean aggregation
>>> cqt_avg = librosa.util.sync(cqt, beats)
Use median-aggregation instead of mean
>>> cqt_med = librosa.util.sync(cqt, beats,
... aggregate=np.median)
Or sub-beat synchronization
>>> sub_beats = librosa.segment.subsegment(cqt, beats)
>>> sub_beats = librosa.util.fix_frames(sub_beats, x_max=cqt.shape[1])
>>> cqt_med_sub = librosa.util.sync(cqt, sub_beats, aggregate=np.median)
Plot the results
>>> import matplotlib.pyplot as plt
>>> beat_t = librosa.frames_to_time(beats, sr=sr)
>>> subbeat_t = librosa.frames_to_time(sub_beats, sr=sr)
>>> plt.figure()
>>> plt.subplot(3, 1, 1)
>>> librosa.display.specshow(librosa.logamplitude(cqt**2,
... ref_power=np.max),
... x_axis='time')
>>> plt.colorbar(format='%+2.0f dB')
>>> plt.title('CQT power, shape={}'.format(cqt.shape))
>>> plt.subplot(3, 1, 2)
>>> librosa.display.specshow(librosa.logamplitude(cqt_med**2,
... ref_power=np.max),
... x_coords=beat_t, x_axis='time')
>>> plt.colorbar(format='%+2.0f dB')
>>> plt.title('Beat synchronous CQT power, '
... 'shape={}'.format(cqt_med.shape))
>>> plt.subplot(3, 1, 3)
>>> librosa.display.specshow(librosa.logamplitude(cqt_med_sub**2,
... ref_power=np.max),
... x_coords=subbeat_t, x_axis='time')
>>> plt.colorbar(format='%+2.0f dB')
>>> plt.title('Sub-beat synchronous CQT power, '
... 'shape={}'.format(cqt_med_sub.shape))
>>> plt.tight_layout()
"""
if aggregate is None:
aggregate = np.mean
shape = list(data.shape)
if np.all([isinstance(_, slice) for _ in idx]):
slices = idx
elif np.all([np.issubdtype(type(_), np.int) for _ in idx]):
slices = index_to_slice(np.asarray(idx), 0, shape[axis], pad=pad)
else:
raise ParameterError('Invalid index set: {}'.format(idx))
agg_shape = list(shape)
agg_shape[axis] = len(slices)
data_agg = np.empty(agg_shape, order='F' if np.isfortran(data) else 'C', dtype=data.dtype)
idx_in = [slice(None)] * data.ndim
idx_agg = [slice(None)] * data_agg.ndim
for (i, segment) in enumerate(slices):
idx_in[axis] = segment
idx_agg[axis] = i
data_agg[idx_agg] = aggregate(data[idx_in], axis=axis)
return data_agg
def apcg_enet(X,
y,
alpha,
rho=0,
max_iter=10000,
tol=1e-4,
f_gap=10,
verbose=False):
"""Solve the Lasso with accelerated proximal coordinate gradient.
Parameters
----------
X : {array_like, sparse matrix}, shape (n_samples, n_features)
Design matrix
y : ndarray, shape (n_samples,)
Observation vector
alpha : float
Regularization strength for the l1 norm
rho: float
Regularization strength for the l2 (squared) norm
max_iter : int, default=1000
Maximum number of iterations
tol : float, default=1e-4
The algorithm early stops if the duality gap is less than tol.
f_gap: int, default=10
The gap is computed every f_gap iterations.
verbose : bool, default=False
Verbosity.
Returns
-------
W : array, shape (n_features,)
Estimated coefficients.
E : ndarray
Objectives every gap_freq iterations.
gaps : ndarray
Duality gaps every gap_freq iterations.
"""
np.random.seed(0)
n_samples, n_features = X.shape
is_sparse = sparse.issparse(X)
if not is_sparse and not np.isfortran(X):
X = np.asfortranarray(X)
if is_sparse:
lc = sparse.linalg.norm(X, axis=0)**2
else:
lc = (X**2).sum(axis=0)
# Algo 2 in Li, Lu, Xiao 2014
w = np.zeros(n_features)
u = np.zeros(n_features)
z = w.copy()
tau = 1. / n_features
Xu = np.zeros(n_samples)
Xz = np.zeros(n_samples)
E = []
gaps = np.zeros(max_iter // f_gap)
for it in range(max_iter):
if sparse.issparse(X):
tau, tau_old = _apcg_sparse(X.data, X.indices, X.indptr, z, u, tau,
Xu, Xz, y, alpha, rho, lc, n_features)
else:
tau, tau_old = _apcg(X, z, u, tau, Xu, Xz, y, alpha, rho, lc)
if it % f_gap == 0:
w = tau_old**2 * u + z
R = y - X @ w # MM: todo this is brutal if f_gap = 1
p_obj = primal_enet(R, w, alpha, rho)
E.append(p_obj)
if np.abs(p_obj) > np.abs(E[0] * 1e3):
break
if alpha != 0:
theta = R / alpha
if rho == 0:
d_norm_theta = np.max(np.abs(X.T @ theta))
if d_norm_theta > 1.:
theta /= d_norm_theta
d_obj = dual_lasso(y, theta, alpha)
else:
XTR = X.T @ R
d_obj = dual_enet(XTR, y - R, y, alpha, rho)
gap = p_obj - d_obj
if verbose:
print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" %
(it, p_obj, d_obj, gap))
gaps[it // f_gap] = gap
if gap < tol:
print("Early exit")
break
else:
if verbose:
print("Iteration %d, p_obj::%.10f" % (it, p_obj))
return w, np.array(E), gaps[:it // f_gap + 1]
def convert_to_float32(data):
if (not np.issubdtype(data.dtype, np.float32) or np.isfortran(data)):
print "Converting H0"
return data.astype(np.float32, order='C')
else:
return data
def set_array(self, new_array):
if not np.isfortran(new_array):
new_array = np.asfortranarray(new_array)
self.array = new_array
def set_scalars(self, new_scalars):
order = 'C'
if np.isfortran(self.array):
order = 'F'
new_scalars = np.reshape(new_scalars, self.array.shape, order=order)
self.array = np.asfortranarray(new_scalars)
i8: np.int64
A: np.ndarray
B: List[int]
C: SubClass
reveal_type(np.count_nonzero(i8)) # E: int
reveal_type(np.count_nonzero(A)) # E: int
reveal_type(np.count_nonzero(B)) # E: int
reveal_type(np.count_nonzero(
A, keepdims=True)) # E: Union[numpy.signedinteger[Any], numpy.ndarray]
reveal_type(np.count_nonzero(
A, axis=0)) # E: Union[numpy.signedinteger[Any], numpy.ndarray]
reveal_type(np.isfortran(i8)) # E: bool
reveal_type(np.isfortran(A)) # E: bool
reveal_type(np.argwhere(i8)) # E: numpy.ndarray
reveal_type(np.argwhere(A)) # E: numpy.ndarray
reveal_type(np.flatnonzero(i8)) # E: numpy.ndarray
reveal_type(np.flatnonzero(A)) # E: numpy.ndarray
reveal_type(np.correlate(B, A, mode="valid")) # E: numpy.ndarray
reveal_type(np.correlate(A, A, mode="same")) # E: numpy.ndarray
reveal_type(np.convolve(B, A, mode="valid")) # E: numpy.ndarray
reveal_type(np.convolve(A, A, mode="same")) # E: numpy.ndarray
reveal_type(np.outer(i8, A)) # E: numpy.ndarray
def neldermead(fcn,
x0,
xmin,
xmax,
ftol=EPSILON,
maxfev=None,
initsimplex=0,
finalsimplex=9,
step=None,
iquad=1,
verbose=0):
x, xmin, xmax = _check_args(x0, xmin, xmax)
order = 'F' if numpy.isfortran(x) else 'C'
if step is None or (numpy.iterable(step) and len(step) != len(x)):
step = 1.2 * numpy.ones(x.shape, numpy.float_, order)
elif numpy.isscalar(step):
step = step * numpy.ones(x.shape, numpy.float_, order)
def stat_cb0(pars):
return fcn(pars)[0]
#
# A safeguard just in case the initial simplex is outside the bounds
#
orig_fcn = stat_cb0
def stat_cb0(x_new):
if _my_is_nan(x_new) or _outside_limits(x_new, xmin, xmax):
return FUNC_MAX
return orig_fcn(x_new)
debug = False # for internal use only
if numpy.isscalar(finalsimplex) and 0 == numpy.iterable(finalsimplex):
finalsimplex = int(finalsimplex)
if 0 == finalsimplex:
finalsimplex = [1]
elif 1 == finalsimplex:
finalsimplex = [2]
elif 2 == finalsimplex:
finalsimplex = [0, 0]
elif 3 == finalsimplex:
finalsimplex = [0, 1]
elif 4 == finalsimplex:
finalsimplex = [0, 1, 0]
elif 5 == finalsimplex:
finalsimplex = [0, 2, 0]
elif 6 == finalsimplex:
finalsimplex = [1, 1, 0]
elif 7 == finalsimplex:
finalsimplex = [2, 1, 0]
elif 8 == finalsimplex:
finalsimplex = [1, 2, 0]
elif 9 == finalsimplex:
finalsimplex = [0, 1, 1]
elif 10 == finalsimplex:
finalsimplex = [0, 2, 1]
elif 11 == finalsimplex:
finalsimplex = [1, 1, 1]
elif 12 == finalsimplex:
finalsimplex = [1, 2, 1]
elif 13 == finalsimplex:
finalsimplex = [2, 1, 1]
else:
finalsimplex = [2, 2, 2]
elif (False == numpy.isscalar(finalsimplex)
and 1 == numpy.iterable(finalsimplex)):
pass
else:
finalsimplex = [2, 2, 2]
finalsimplex = numpy.asarray(finalsimplex, numpy.int_)
if maxfev is None:
maxfev = 1024 * len(x)
if debug:
print(
'opfcts.py neldermead() finalsimplex=%s\tisscalar=%s\titerable=%d'
% (finalsimplex, numpy.isscalar(finalsimplex),
numpy.iterable(finalsimplex)))
def simplex(verbose,
maxfev,
init,
final,
tol,
step,
xmin,
xmax,
x,
myfcn,
debug,
ofval=FUNC_MAX):
tmpfinal = final[:]
if len(final) >= 3:
tmpfinal = final[0:-1] # get rid of the last entry in the list
xx, ff, nf, er = _saoopt.neldermead(verbose, maxfev, init, tmpfinal,
tol, step, xmin, xmax, x, myfcn)
if debug:
print('finalsimplex=%s, nfev=%d:\tf%s=%.20e' %
(tmpfinal, nf, xx, ff))
if len(final) >= 3 and ff < 0.995 * ofval and nf < maxfev:
myfinal = [final[-1]]
x, fval, nfev, err = simplex(verbose,
maxfev - nf,
init,
myfinal,
tol,
step,
xmin,
xmax,
x,
myfcn,
debug,
ofval=ff)
return x, fval, nfev + nf, err
else:
return xx, ff, nf, er
x, fval, nfev, ier = simplex(verbose, maxfev, initsimplex, finalsimplex,
ftol, step, xmin, xmax, x, stat_cb0, debug)
if debug:
print('f%s=%e in %d nfev' % (x, fval, nfev))
info = 1
covarerr = None
if len(finalsimplex) >= 3 and 0 != iquad:
nelmea = minim(fcn,
x,
xmin,
xmax,
ftol=10.0 * ftol,
maxfev=maxfev - nfev - 12,
iquad=1)
nelmea_x = numpy.asarray(nelmea[1], numpy.float_)
nelmea_nfev = nelmea[4].get('nfev')
info = nelmea[4].get('info')
covarerr = nelmea[4].get('covarerr')
nfev += nelmea_nfev
minim_fval = nelmea[2]
if minim_fval < fval:
x = nelmea_x
fval = minim_fval
if debug:
print('minim: f%s=%e %d nfev, info=%d' %
(x, fval, nelmea_nfev, info))
if nfev >= maxfev:
ier = 3
key = {
0: (True, 'Optimization terminated successfully'),
1: (False, 'improper input parameters'),
2: (False, 'improper values for x, xmin or xmax'),
3: (False, 'number of function evaluations has exceeded %d' % maxfev)
}
status, msg = key.get(ier, (False, 'unknown status flag (%d)' % ier))
rv = (status, x, fval)
print_covar_err = False
if print_covar_err and None != covarerr:
rv += (msg, {'covarerr': covarerr, 'info': status, 'nfev': nfev})
else:
rv += (msg, {'info': status, 'nfev': nfev})
return rv
def verify_is_c_contiguous_and_is_not_fortran(x):
if np.isfortran(x):
raise ValueError("Must be 'C' order")
if not x.flags.c_contiguous:
raise ValueError("Must be 'C'-contiguous")
def norm(a, ord=None, axis=None, keepdims=False, check_finite=True):
"""
Matrix or vector norm.
This function is able to return one of seven different matrix norms,
or one of an infinite number of vector norms (described below), depending
on the value of the ``ord`` parameter.
Parameters
----------
a : (M,) or (M, N) array_like
Input array. If `axis` is None, `a` must be 1D or 2D.
ord : {non-zero int, inf, -inf, 'fro'}, optional
Order of the norm (see table under ``Notes``). inf means NumPy's
`inf` object
axis : {int, 2-tuple of ints, None}, optional
If `axis` is an integer, it specifies the axis of `a` along which to
compute the vector norms. If `axis` is a 2-tuple, it specifies the
axes that hold 2-D matrices, and the matrix norms of these matrices
are computed. If `axis` is None then either a vector norm (when `a`
is 1-D) or a matrix norm (when `a` is 2-D) is returned.
keepdims : bool, optional
If this is set to True, the axes which are normed over are left in the
result as dimensions with size one. With this option the result will
broadcast correctly against the original `a`.
check_finite : bool, optional
Whether to check that the input matrix contains only finite numbers.
Disabling may give a performance gain, but may result in problems
(crashes, non-termination) if the inputs do contain infinities or NaNs.
Returns
-------
n : float or ndarray
Norm of the matrix or vector(s).
Notes
-----
For values of ``ord <= 0``, the result is, strictly speaking, not a
mathematical 'norm', but it may still be useful for various numerical
purposes.
The following norms can be calculated:
===== ============================ ==========================
ord norm for matrices norm for vectors
===== ============================ ==========================
None Frobenius norm 2-norm
'fro' Frobenius norm --
inf max(sum(abs(x), axis=1)) max(abs(x))
-inf min(sum(abs(x), axis=1)) min(abs(x))
0 -- sum(x != 0)
1 max(sum(abs(x), axis=0)) as below
-1 min(sum(abs(x), axis=0)) as below
2 2-norm (largest sing. value) as below
-2 smallest singular value as below
other -- sum(abs(x)**ord)**(1./ord)
===== ============================ ==========================
The Frobenius norm is given by [1]_:
:math:`||A||_F = [\\sum_{i,j} abs(a_{i,j})^2]^{1/2}`
The ``axis`` and ``keepdims`` arguments are passed directly to
``numpy.linalg.norm`` and are only usable if they are supported
by the version of numpy in use.
References
----------
.. [1] G. H. Golub and C. F. Van Loan, *Matrix Computations*,
Baltimore, MD, Johns Hopkins University Press, 1985, pg. 15
Examples
--------
>>> from scipy.linalg import norm
>>> a = np.arange(9) - 4.0
>>> a
array([-4., -3., -2., -1., 0., 1., 2., 3., 4.])
>>> b = a.reshape((3, 3))
>>> b
array([[-4., -3., -2.],
[-1., 0., 1.],
[ 2., 3., 4.]])
>>> norm(a)
7.745966692414834
>>> norm(b)
7.745966692414834
>>> norm(b, 'fro')
7.745966692414834
>>> norm(a, np.inf)
4
>>> norm(b, np.inf)
9
>>> norm(a, -np.inf)
0
>>> norm(b, -np.inf)
2
>>> norm(a, 1)
20
>>> norm(b, 1)
7
>>> norm(a, -1)
-4.6566128774142013e-010
>>> norm(b, -1)
6
>>> norm(a, 2)
7.745966692414834
>>> norm(b, 2)
7.3484692283495345
>>> norm(a, -2)
0
>>> norm(b, -2)
1.8570331885190563e-016
>>> norm(a, 3)
5.8480354764257312
>>> norm(a, -3)
0
"""
# Differs from numpy only in non-finite handling and the use of blas.
if check_finite:
a = np.asarray_chkfinite(a)
else:
a = np.asarray(a)
# Only use optimized norms if axis and keepdims are not specified.
if a.dtype.char in 'fdFD' and axis is None and not keepdims:
if ord in (None, 2) and (a.ndim == 1):
# use blas for fast and stable euclidean norm
nrm2 = get_blas_funcs('nrm2', dtype=a.dtype)
return nrm2(a)
if a.ndim == 2 and axis is None and not keepdims:
# Use lapack for a couple fast matrix norms.
# For some reason the *lange frobenius norm is slow.
lange_args = None
# Make sure this works if the user uses the axis keywords
# to apply the norm to the transpose.
if ord == 1:
if np.isfortran(a):
lange_args = '1', a
elif np.isfortran(a.T):
lange_args = 'i', a.T
elif ord == np.inf:
if np.isfortran(a):
lange_args = 'i', a
elif np.isfortran(a.T):
lange_args = '1', a.T
if lange_args:
lange = get_lapack_funcs('lange', dtype=a.dtype)
return lange(*lange_args)
# Filter out the axis and keepdims arguments if they aren't used so they
# are never inadvertently passed to a version of numpy that doesn't
# support them.
if axis is not None:
if keepdims:
return np.linalg.norm(a, ord=ord, axis=axis, keepdims=keepdims)
return np.linalg.norm(a, ord=ord, axis=axis)
return np.linalg.norm(a, ord=ord)
def from_array(cls, arr: np.ndarray):
return cls(arr.shape, arr.dtype, strides=arr.strides, order='CF'[np.isfortran(arr)])
def GBDT_test(data, fold_n, num_rounds=100000, bf=1, ff=1):
model_type = "mort" if isMORT else "lgb"
nFeatures = data.X_train.shape[1]
early_stop = 100
verbose_eval = 20
#lr = 0.01;
bf = bf
ff = ff
if data.problem() == "classification":
metric = 'auc' #"rmse"
params = {
"objective": "binary",
"metric": metric,
'n_estimators': num_rounds,
"bagging_fraction": bf,
"feature_fraction": ff,
'verbose_eval': verbose_eval,
"early_stopping_rounds": early_stop,
'n_jobs': -1,
}
else:
metric = 'l2' #"rmse"
params = {
"objective": "regression",
"metric": metric,
'n_estimators': num_rounds,
"bagging_fraction": bf,
"feature_fraction": ff,
'verbose_eval': verbose_eval,
"early_stopping_rounds": early_stop,
'n_jobs': -1,
}
print(f"====== GBDT_test\tparams={params}")
X_train, y_train = data.X_train, data.y_train
X_valid, y_valid = data.X_valid, data.y_valid
X_test, y_test = data.X_test, data.y_test
if not np.isfortran(
X_train): #Very important!!! mort need COLUMN-MAJOR format
X_train = np.asfortranarray(X_train)
X_valid = np.asfortranarray(X_valid)
#X_train, X_valid = pd.DataFrame(X_train), pd.DataFrame(X_valid)
print(f"GBDT_test\ttrain={X_train.shape} valid={X_valid.shape}")
#print(f"X_train=\n{X_train.head()}\n{X_train.tail()}")
if model_type == 'mort':
params['verbose'] = 667
model = LiteMORT(params).fit(X_train,
y_train,
eval_set=[(X_valid, y_valid)])
#y_pred_valid = model.predict(X_valid)
#y_pred = model.predict(X_test)
if model_type == 'lgb':
if data.problem() == "classification":
model = lgb.LGBMClassifier(**params)
else:
model = lgb.LGBMRegressor(**params)
model.fit(X_train,
y_train,
eval_set=[(X_train, y_train), (X_valid, y_valid)],
verbose=min(num_rounds // 10, 1000))
pred_val = model.predict(data.X_test)
#plot_importance(model)
lgb.plot_importance(model, max_num_features=32)
plt.title("Featurertances")
plt.savefig(f"./results/{dataset}_feat_importance_.jpg")
#plt.show(block=False)
plt.close()
fold_importance = pd.DataFrame()
fold_importance["importance"] = model.feature_importances_
fold_importance["feature"] = [i for i in range(nFeatures)]
fold_importance["fold"] = fold_n
#fold_importance.to_pickle(f"./results/{dataset}_feat_{fold_n}.pickle")
print('best_score', model.best_score_)
acc_train, acc_ = model.best_score_['training'][
metric], model.best_score_['valid_1'][metric]
if data.X_test is not None:
pred_val = model.predict(data.X_test)
acc_ = ((data.y_test - pred_val)**2).mean()
print(
f'====== Best step: test={data.X_test.shape} ACCU@Test={acc_:.5f}')
return acc_, fold_importance
def write_edf(edf_file,
signals,
signal_headers,
header=None,
digital=False,
file_type=-1,
block_size=1):
"""
Write signals to an edf_file. Header can be generated on the fly with
generic values. EDF+/BDF+ is selected based on the filename extension,
but can be overwritten by setting filetype to pyedflib.FILETYPE_XXX
Parameters
----------
edf_file : np.ndarray or list
where to save the EDF file
signals : list
The signals as a list of arrays or a ndarray.
signal_headers : list of dict
a list with one signal header(dict) for each signal.
See pyedflib.EdfWriter.setSignalHeader..
header : dict
a main header (dict) for the EDF file, see
pyedflib.EdfWriter.setHeader for details.
If no header present, will create an empty header
digital : bool, optional
whether the signals are in digital format (ADC). The default is False.
filetype: int, optional
choose filetype for saving.
EDF = 0, EDF+ = 1, BDF = 2, BDF+ = 3, automatic from extension = -1
block_size : int
set the block size for writing. Should be divisor of signal length
in seconds. Higher values mean faster writing speed, but if it
is not a divisor of the signal duration, it will append zeros.
Can be any value between 1=><=60, -1 will auto-infer the fastest value.
Returns
-------
bool
True if successful, False if failed.
"""
assert header is None or isinstance(header, dict), \
'header must be dictioniary or None'
assert isinstance(signal_headers, list), \
'signal headers must be list'
assert len(signal_headers)==len(signals), \
'signals and signal_headers must be same length'
assert file_type in [-1, 0, 1, 2, 3], \
'filetype must be in range -1, 3'
assert block_size<=60 and block_size>=-1 and block_size!=0, \
'blocksize must be smaller or equal to 60'
# copy objects to prevent accidential changes to mutable objects
header = deepcopy(header)
signal_headers = deepcopy(signal_headers)
if file_type == -1:
ext = os.path.splitext(edf_file)[-1]
if ext.lower() == '.edf':
file_type = pyedflib.FILETYPE_EDFPLUS
elif ext.lower() == '.bdf':
file_type = pyedflib.FILETYPE_BDFPLUS
else:
raise ValueError('Unknown extension {}'.format(ext))
n_channels = len(signals)
# if there is no header, we create one with dummy values
if header is None:
header = {}
default_header = make_header()
default_header.update(header)
header = default_header
# block_size sets the size of each writing block and should be a divisor
# of the length of the signal. If it is not, the remainder of the file
# will be filled with zeros.
signal_duration = len(signals[0]) / signal_headers[0]['sample_rate']
if block_size == -1:
samplefrequencies = np.array(
[hdr["sample_rate"] for hdr in signal_headers])
block_size = min([
d for d in range(1, 61) if ((signal_duration % d == 0) and np.all(
((samplefrequencies * d) % 1) == 0))
])
elif signal_duration % block_size != 0:
warnings.warn('Signal length is not dividable by block_size. ' +
'The file will have a zeros appended.')
# check dmin, dmax and pmin, pmax dont exceed signal min/max
for sig, shead in zip(signals, signal_headers):
dmin, dmax = shead['digital_min'], shead['digital_max']
pmin, pmax = shead['physical_min'], shead['physical_max']
label = shead['label']
if digital: # exception as it will lead to clipping
assert dmin<=sig.min(), \
'digital_min is {}, but signal_min is {}' \
'for channel {}'.format(dmin, sig.min(), label)
assert dmax>=sig.max(), \
'digital_min is {}, but signal_min is {}' \
'for channel {}'.format(dmax, sig.max(), label)
assert pmin != pmax, \
'physical_min {} should be different from physical_max {}'.format(pmin,pmax)
else: # only warning, as this will not lead to clipping
assert pmin<=sig.min(), \
'phys_min is {}, but signal_min is {} ' \
'for channel {}'.format(pmin, sig.min(), label)
assert pmax>=sig.max(), \
'phys_max is {}, but signal_max is {} ' \
'for channel {}'.format(pmax, sig.max(), label)
shead['sample_rate'] *= block_size
# get annotations, in format [[timepoint, duration, description], [...]]
annotations = header.get('annotations', [])
if any([np.isfortran(s) for s in signals]) or \
(isinstance(signals, np.ndarray) and np.isfortran(signals)):
warnings.warn('signals are in Fortran order. Will automatically ' \
'transfer to C order for compatibility with edflib.')
if isinstance(signals, list):
signals = [
s.copy(order='C') if np.isfortran(s) else s for s in signals
]
elif isinstance(signals, np.ndarray) and np.isfortran(signals):
signals = signals.copy(order='C')
with pyedflib.EdfWriter(edf_file,
n_channels=n_channels,
file_type=file_type) as f:
f.setDatarecordDuration(int(100000 * block_size))
f.setSignalHeaders(signal_headers)
f.setHeader(header)
f.writeSamples(signals, digital=digital)
for annotation in annotations:
f.writeAnnotation(*annotation)
del f
return os.path.isfile(edf_file)
def build(self,
X,
y,
criterion,
max_depth,
min_samples_split,
min_samples_leaf,
min_density,
max_features,
random_state,
find_split,
sample_mask=None,
X_argsorted=None):
# Recursive algorithm
def recursive_partition(X, X_argsorted, y, sample_mask, depth, parent,
is_left_child):
# Count samples
n_node_samples = sample_mask.sum()
if n_node_samples == 0:
raise ValueError("Attempting to find a split "
"with an empty sample_mask")
# Split samples
if depth < max_depth and n_node_samples >= min_samples_split \
and n_node_samples >= 2 * min_samples_leaf:
feature, threshold, best_error, init_error = find_split(
X, y, X_argsorted, sample_mask, n_node_samples,
min_samples_leaf, max_features, criterion, random_state)
else:
feature = -1
init_error = _tree._error_at_leaf(y, sample_mask, criterion,
n_node_samples)
value = criterion.init_value()
# Current node is leaf
if feature == -1:
self._add_leaf(parent, is_left_child, value, init_error,
n_node_samples)
# Current node is internal node (= split node)
else:
# Sample mask is too sparse?
if n_node_samples / X.shape[0] <= min_density:
X = X[sample_mask]
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
y = y[sample_mask]
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
# Split and and recurse
split = X[:, feature] <= threshold
node_id = self._add_split_node(parent, is_left_child, feature,
threshold, best_error,
init_error, n_node_samples,
value)
# left child recursion
recursive_partition(X, X_argsorted, y,
np.logical_and(split, sample_mask),
depth + 1, node_id, True)
# right child recursion
recursive_partition(
X, X_argsorted, y,
np.logical_and(np.logical_not(split), sample_mask),
depth + 1, node_id, False)
# Setup auxiliary data structures and check input before
# recursive partitioning
if X.dtype != DTYPE or not np.isfortran(X):
X = np.asanyarray(X, dtype=DTYPE, order="F")
if y.dtype != DTYPE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DTYPE)
if sample_mask is None:
sample_mask = np.ones((X.shape[0], ), dtype=np.bool)
if X_argsorted is None:
X_argsorted = np.asfortranarray(
np.argsort(X.T, axis=1).astype(np.int32).T)
# Pre-allocate some space
if max_depth <= 10:
# allocate space for complete binary tree
init_capacity = (2**(max_depth + 1)) - 1
else:
# allocate fixed size and dynamically resize later
init_capacity = 2047
self._resize(init_capacity)
# Build the tree by recursive partitioning
recursive_partition(X, X_argsorted, y, sample_mask, 0, -1, False)
# Compactify the tree data structure
self._resize(self.node_count)
return self
def fit_file(self, motion_correct=False, indices=[slice(None)] * 2):
"""
This method packages the analysis pipeline (motion correction, memory
mapping, patch based CNMF processing) in a single method that can be
called on a specific (sequence of) file(s). It is assumed that the CNMF
object already contains a params object where the location of the files
and all the relevant parameters have been specified. The method does
not perform the quality evaluation step. Consult demo_pipeline for an
example.
Args:
motion_correct (bool)
flag for performing motion correction
indices (list of slice objects)
perform analysis only on a part of the FOV
Returns:
cnmf object with the current estimates
"""
fnames = self.params.get('data', 'fnames')
if os.path.exists(fnames[0]):
_, extension = os.path.splitext(fnames[0])[:2]
extension = extension.lower()
else:
logging.warning("Error: File not found, with file list:\n" +
fnames[0])
raise Exception('File not found!')
if extension == '.mmap':
fname_new = fnames[0]
Yr, dims, T = mmapping.load_memmap(fnames[0])
if np.isfortran(Yr):
raise Exception(
'The file should be in C order (see save_memmap function)')
else:
if motion_correct:
mc = MotionCorrect(fnames,
dview=self.dview,
**self.params.motion)
mc.motion_correct(save_movie=True)
fname_mc = mc.fname_tot_els if self.params.motion[
'pw_rigid'] else mc.fname_tot_rig
if self.params.get('motion', 'pw_rigid'):
b0 = np.ceil(
np.maximum(np.max(np.abs(mc.x_shifts_els)),
np.max(np.abs(mc.y_shifts_els)))).astype(
np.int)
self.estimates.shifts = [mc.x_shifts_els, mc.y_shifts_els]
else:
b0 = np.ceil(np.max(np.abs(mc.shifts_rig))).astype(np.int)
self.estimates.shifts = mc.shifts_rig
b0 = 0 if self.params.get('motion',
'border_nan') is 'copy' else 0
fname_new = mmapping.save_memmap(fname_mc,
base_name='memmap_',
order='C',
border_to_0=b0)
else:
fname_new = mmapping.save_memmap(fnames,
base_name='memmap_',
order='C')
# now load the file
Yr, dims, T = mmapping.load_memmap(fname_new)
images = np.reshape(Yr.T, [T] + list(dims), order='F')
self.mmap_file = fname_new
return self.fit(images, indices=indices)
def bin_array(arraytotcorr, indZ, threeD_array):
"""
Function to reduce the size of large datasets. Data placed into equidistant bins for each x,y coordinate and
new vector created from the mean of each bin. Size of equidistant bins determined by 0.01 nm increments of
Zsensor data.
:param arraytotcorr: Zsensor data corrected for sample tilt using correct_slope function. Important to use this
and not raw Zsensor data so as to get an accurate Zmax value.
:param indZ: Index of Zmax for each x,y coordinate to cut data set into approach and retract.
:param rawarray: 3D numpy array the user wishes to reduce in size (e.g. phase, amp)
:return: 3D numpy array of binned approach values, 3D numpy array of binned retract values.
"""
assert np.isfortran(
threeD_array
) == True, "Input array not passed through generate_array fucntion. \
Needs to be column-major indexing."
# Generate empty numpy array to populate.
global reduced_array_approach
global reduced_array_retract
global linearized
arraymean = np.zeros(len(arraytotcorr[:, 1, 1]))
digitized = np.empty_like(arraymean)
# Create list of the mean Zsensor value for each horizontal slice of Zsensor array.
for z in range(len(arraymean)):
arraymean[z] = np.mean(arraytotcorr[z, :, :])
# Turn mean Zsensor data into a linear vector with a step size of 0.02 nm.
linearized = np.arange(-0.2, arraymean.max(), 0.02)
# Generate empty array to populate
reduced_array_approach = np.zeros(
(len(linearized), len(arraytotcorr[1, :, 1]), len(arraytotcorr[1,
1, :])))
reduced_array_retract = np.zeros(
(len(linearized), len(arraytotcorr[1, :, 1]), len(arraytotcorr[1,
1, :])))
# Cut raw phase/amp datasets into approach and retract, then bin data according to the linearized Zsensor data.
# Generate new arrays from the means of each bin. Perform on both approach and retract data.
for j in range(len(arraytotcorr[1, :, 1])):
for i in range(len(arraytotcorr[1, 1, :])):
z = arraytotcorr[:(int(indZ[i, j])), i,
j] # Create dataset with just retract data
digitized = np.digitize(
z, linearized
) # Bin Z data based on standardized linearized vector.
for n in range(len(linearized)):
ind = list(np.where(digitized == n)
[0]) # Find which indices belong to which bins
reduced_array_approach[n, i, j] = np.mean(
threeD_array[ind, i, j]) # Find the mean of the bins and
# populate new array.
for j in range(len(arraytotcorr[1, :, 1])):
for i in range(len(arraytotcorr[1, 1, :])):
z = arraytotcorr[-(int(indZ[i, j])):, i,
j] # Create dataset with just approach data.
z = np.flipud(
z) # Flip array so surface is at the bottom on the plot.
digitized = np.digitize(
z, linearized
) # Bin Z data based on standardized linearized vector.
for n in range(len(linearized)):
ind = list(np.where(digitized == n)
[0]) # Find which indices belong to which bins
reduced_array_retract[n, i, j] = np.mean(
threeD_array[ind, i, j]) # Find the mean of the bins and
# populate new array.
return linearized, reduced_array_approach, reduced_array_retract
#b = numpy.ones(4)
#err = libzquatev.test(a,b)
#exit(1)
n = 2 #1000
print("n=",n)
fock = libzquatev.gen_array(n) # return a Hermitian matrix
print(numpy.linalg.norm(fock - fock.conj().T))
print("F=\n",fock)
print("\nzquatev")
e,v = zquatev.eigh(fock,iop=1)
print("e=",e)
numpy.set_printoptions(4, linewidth=120)
print("v=",v)
print("Fc0-c0*E0 Should be clouse to zero", e[0])
#print(numpy.dot(fock, v[:,0]) - e[0]*v[:,0])
print(abs(numpy.dot(fock, v) - e*v).max())
print("v0=",v[0,:])
print(numpy.isfortran(fock))
print(fock.flags)
print("\nnumpy")
e,v = numpy.linalg.eigh(fock)
print("e=",e.shape,e)
print("v0=",v[:,0])
print("v=\n",v)
print("Should be clouse to zero", e[0])
#print(abs(numpy.dot(fock, v[:,0]) - e[0]*v[:,0]).max())
print(abs(numpy.dot(fock, v) - e*v).max())
def apcg_logreg(X, y, alpha, max_iter=10000, tol=1e-4, f_gap=10, seed=0,
verbose=False):
"""Solve the l1 regularized logistic regression with with accelerated
proximal coordinate gradient.
Parameters
----------
X : {array_like, sparse matrix}, shape (n_samples, n_features)
Design matrix
y : ndarray, shape (n_samples,)
Observation vector
alpha : float
Regularization strength
max_iter : int, default=1000
Maximum number of iterations
tol : float, default=1e-4
The algorithm early stops if the duality gap is less than tol.
f_gap: int, default=10
The gap is computed every f_gap iterations.
seed : int (default=0)
Seed for randomness.
verbose : bool, default=False
Verbosity.
Returns
-------
w : array, shape (n_features,)
Estimated coefficients.
E : ndarray
Objectives every gap_freq iterations.
gaps : ndarray
Duality gaps every gap_freq iterations.
"""
np.random.seed(seed)
n_samples, n_features = X.shape
is_sparse = sparse.issparse(X)
if not is_sparse and not np.isfortran(X):
X = np.asfortranarray(X)
if is_sparse:
lc = sparse.linalg.norm(X, axis=0) ** 2 / 4.
else:
lc = (X ** 2).sum(axis=0) / 4.
w = np.zeros(n_features)
u = np.zeros(n_features)
z = w.copy()
tau = 1. / n_features
Xu = np.zeros(n_samples)
Xz = np.zeros(n_samples)
E = []
gaps = np.zeros(max_iter // f_gap)
for it in range(max_iter):
if is_sparse:
tau, tau_old = _apcg_sparse(
X.data, X.indices, X.indptr, z, u, tau, Xu, Xz, y, alpha, lc,
n_features)
else:
tau, tau_old = _apcg(X, z, u, tau, Xu, Xz, y, alpha, lc)
if it % f_gap == 0:
w = tau_old ** 2 * u + z
Xw = X @ w
p_obj = primal_logreg(Xw, y, w, alpha)
E.append(p_obj)
if np.abs(p_obj) > np.abs(E[0] * 1e3):
break
if alpha != 0:
theta = y * sigmoid(-y * Xw) / alpha
d_norm_theta = np.max(np.abs(X.T @ theta))
if d_norm_theta > 1.:
theta /= d_norm_theta
d_obj = dual_logreg(y, theta, alpha)
gap = p_obj - d_obj
if verbose:
print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" %
(it, p_obj, d_obj, gap))
gaps[it // f_gap] = gap
if gap < tol:
print("Early exit")
break
else:
if verbose:
print("Iteration %d, p_obj::%.10f" % (it, p_obj))
return w, np.array(E), gaps[:it // f_gap + 1]
def elastic_net(predictors,
target,
balance,
memlimit=None,
largest=None,
thr=_DEFAULT_THRESH,
weights=None,
penalties=None,
standardize=True,
exclude=None,
lambdas=None,
flmin=None,
nlam=None,
overwrite_pred_ok=False,
overwrite_targ_ok=True):
""" Raw-output wrapper for elastic net linear regression.
Args:
predictors (np.ndarray): X design matrix
target (np.ndarray): y dependent variables
balance (float): Family member index (0 is ridge, 1 is Lasso)
memlimit (int): Maximum number of variables allowed to enter all models
along path
largest (int): Maximum number of variables allowed to enter largest
model
thr (float): Minimum change in largest coefficient
weights (np.ndarray): Relative weighting per observation case
penalties (np.ndarray): Relative penalties per predictor (0 = no
penalty)(vp in Fortran code)
standardize (bool): Standardize input variables before proceeding?
(isd in Fortran code)
exclude (np.ndarray): Predictors to exclude altogether from fitting
(jd in Fortran code)
lambdas (np.ndarray): User specified lambda values (ulam in Fortran
code). Do not specify ``lambdas`` if ``flmin`` or ``nlam`` are also
provided.
flmin (float): Fraction of largest lambda at which to stop
(default: 0.001)
nlam (int): The (maximum) number of lambdas to try (default: 100)
overwrite_pred_ok (bool): Allow overwriting of X
overwrite_targ_ok (bool): Allow overwriting of y
"""
# Decide on largest allowable models for memory/convergence.
memlimit = predictors.shape[1] if memlimit is None else memlimit
# If largest isn't specified use memlimit.
largest = memlimit if largest is None else largest
if memlimit < largest:
raise ValueError('Need largest <= memlimit')
if exclude is not None:
# Add one since Fortran indices start at 1
exclude += 1
jd = np.array([len(exclude)] + exclude)
else:
jd = np.zeros(1)
if lambdas is not None and flmin is not None:
raise ValueError("Can't specify both lambdas & flmin keywords")
elif lambdas is not None:
lambdas = np.atleast_1d(lambdas)
flmin = 2. # Pass flmin > 1.0 indicating to use the user-supplied.
nlam = len(lambdas)
else:
lambdas = None
flmin = _DEFAULT_FLMIN
nlam = _DEFAULT_NLAM
# If predictors is a Fortran contiguous array, it will be overwritten.
# Decide whether we want this. If it's not Fortran contiguous it will
# be copied into that form anyway so there's no chance of overwriting.
if np.isfortran(predictors):
if not overwrite_pred_ok:
# Might as well make it F-ordered to avoid ANOTHER copy.
predictors = predictors.copy(order='F')
# Target being a 1-dimensional array will usually be overwritten
# with the standardized version unless we take steps to copy it.
if not overwrite_targ_ok:
target = target.copy()
# Uniform weighting if no weights are specified.
if weights is None:
weights = np.ones(predictors.shape[0])
# Uniform penalties if none were specified.
if penalties is None:
penalties = np.ones(predictors.shape[1])
# Call the Fortran wrapper.
lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr = \
_glmnet.elnet(balance, predictors, target, weights, jd, penalties,
memlimit, flmin, lambdas, thr,
nlam=nlam, isd=standardize)
# Check for errors, documented in glmnet.f.
if jerr != 0:
if jerr == 10000:
raise ValueError('cannot have max(vp) < 0.0')
elif jerr == 7777:
raise ValueError('all used predictors have 0 variance')
elif jerr < 7777:
raise MemoryError('elnet() returned error code %d' % jerr)
else:
raise Exception('unknown error: %d' % jerr)
return lmu, a0, ca, ia, nin, rsq, alm, nlp, jerr
def solver_logreg(
X, y, alpha, rho=0, max_iter=10000, tol=1e-4, f_gap=10, K=5,
use_acc=False, algo='cd', reg_amount=None, seed=0, verbose=False):
"""Solve l1+l2 regularized logistic regression with CD/ISTA/FISTA,
eventually using Anderson extrapolation.
The minimized objective is:
np.sum(np.log(1 + np.exp(- y * Xw))) / 2 + alpha * norm(x, ord=1) +
rho/2 * norm(w) ** 2
Parameters
----------
X : {array_like, sparse matrix}, shape (n_samples, n_features)
Design matrix
y : ndarray, shape (n_samples,)
Observation vector
alpha : float
L1 regularization strength
rho : float (optional, default=0)
L2 regularization strength.
max_iter : int, default=1000
Maximum number of iterations
tol : float, default=1e-4
The algorithm early stops if the duality gap is less than tol.
f_gap: int, default=10
The gap is computed every f_gap iterations.
K : int, default=5
Number of points used for Anderson extrapolation.
use_acc : bool, default=False TODO change to True?
Whether or not to use Anderson acceleration.
algo : {'cd', 'pgd', 'fista'}
Algorithm used to solve the Elastic net problem.
reg_amount : float or None (default=None)
Amount of regularization used when solving for the extrapolation
coefficients. None means 0 and should be preferred.
seed : int (default=0)
Seed for randomness.
verbose : bool, default=False
Verbosity.
Returns
-------
W : array, shape (n_features,)
Estimated coefficients.
E : ndarray
Objectives every gap_freq iterations.
gaps : ndarray
Duality gaps every gap_freq iterations.
"""
np.random.seed(seed)
is_sparse = sparse.issparse(X)
n_features = X.shape[1]
_range = np.arange(n_features)
feats = dict(cd=lambda: _range,
cd2=lambda: np.hstack((_range, _range)),
cdsym=lambda: np.hstack((_range, _range[::-1])),
cdshuf=lambda: np.random.choice(n_features, n_features,
replace=False))
if not is_sparse and not np.isfortran(X):
X = np.asfortranarray(X)
if use_acc:
last_K_w = np.zeros([K + 1, n_features])
U = np.zeros([K, n_features])
if algo == 'pgd' or algo == 'fista':
if is_sparse:
L = power_method(X, max_iter=1000) ** 2 / 4
else:
L = norm(X, ord=2) ** 2 / 4
if is_sparse:
lc = sparse.linalg.norm(X, axis=0) ** 2 / 4
else:
lc = (X ** 2).sum(axis=0) / 4
w = np.zeros(n_features)
if algo == 'fista':
z = np.zeros(n_features)
t_new = 1
Xw = np.zeros(len(y))
E = []
gaps = np.zeros(max_iter // f_gap)
for it in range(max_iter):
if it % f_gap == 0:
if algo == 'fista':
Xw = X @ w
p_obj = primal_logreg(Xw, y, w, alpha, rho)
E.append(p_obj)
if alpha != 0:
theta = y * sigmoid(-y * Xw) / alpha
d_norm_theta = np.max(np.abs(X.T @ theta))
if d_norm_theta > 1.:
theta /= d_norm_theta
d_obj = dual_logreg(y, theta, alpha)
gap = p_obj - d_obj
if verbose:
print("Iteration %d, p_obj::%.5f, d_obj::%.5f, gap::%.2e" %
(it, p_obj, d_obj, gap))
gaps[it // f_gap] = gap
if gap < tol:
print("Early exit")
break
else:
if verbose:
print("Iteration %d, p_obj::%.10f" % (it, p_obj))
if algo.startswith('cd'):
if is_sparse:
_cd_logreg_sparse(X.data, X.indices, X.indptr, w, Xw, y,
alpha, rho, lc, feats[algo]())
else:
_cd_logreg(X, w, Xw, y, alpha, rho, lc, feats[algo]())
elif algo == 'pgd':
w[:] = ST_vec(
w + 1. / L * X.T @ (y / (1. + np.exp(y * Xw))), alpha / L)
if rho != 0:
w /= 1. + rho / L
Xw[:] = X @ w
elif algo == 'fista':
w_old = w.copy()
w[:] = ST_vec(
z + X.T @ (y / (1. + np.exp(y * (X @ z)))) / L, alpha / L)
if rho != 0:
w /= 1. + rho / L
t_old = t_new
t_new = (1. + np.sqrt(1 + 4 * t_old ** 2)) / 2.
z[:] = w + (t_old - 1.) / t_new * (w - w_old)
else:
raise ValueError("Unknown algo %s" % algo)
if use_acc:
if it < K + 1:
last_K_w[it] = w
else:
for k in range(K):
last_K_w[k] = last_K_w[k + 1]
last_K_w[K] = w
for k in range(K):
U[k] = last_K_w[k + 1] - last_K_w[k]
C = np.dot(U, U.T)
if reg_amount is not None:
C += reg_amount * norm(C, ord=2) * np.eye(C.shape[0])
try:
z = np.linalg.solve(C, np.ones(K))
c = z / z.sum()
w_acc = np.sum(last_K_w[:-1] * c[:, None],
axis=0)
p_obj = primal_logreg(Xw, y, w, alpha, rho)
Xw_acc = X @ w_acc
p_obj_acc = primal_logreg(Xw_acc, y, w_acc, alpha, rho)
if p_obj_acc < p_obj:
w = w_acc
Xw = Xw_acc
except np.linalg.LinAlgError:
if verbose:
print("----------Linalg error")
return w, np.array(E), gaps[:it // f_gap + 1]
def fit(self, images, indices=[slice(None), slice(None)]):
"""
This method uses the cnmf algorithm to find sources in data.
it is calling every function from the cnmf folder
you can find out more at how the functions are called and how they are laid out at the ipython notebook
Args:
images : mapped np.ndarray of shape (t,x,y[,z]) containing the images that vary over time.
indices: list of slice objects along dimensions (x,y[,z]) for processing only part of the FOV
Returns:
self: updated using the cnmf algorithm with C,A,S,b,f computed according to the given initial values
Raises:
Exception 'You need to provide a memory mapped file as input if you use patches!!'
See Also:
..image::docs/img/quickintro.png
http://www.cell.com/neuron/fulltext/S0896-6273(15)01084-3
"""
# Todo : to compartment
if isinstance(indices, slice):
indices = [indices]
indices = [slice(None)] + indices
if len(indices) < len(images.shape):
indices = indices + [slice(None)
] * (len(images.shape) - len(indices))
dims_orig = images.shape[1:]
dims_sliced = images[tuple(indices)].shape[1:]
is_sliced = (dims_orig != dims_sliced)
if self.params.get('patch', 'rf') is None and (is_sliced or 'ndarray'
in str(type(images))):
images = images[tuple(indices)]
self.dview = None
logging.info("Parallel processing in a single patch "
"is not available for loaded in memory or sliced" +
" data.")
T = images.shape[0]
self.params.set('online', {'init_batch': T})
self.dims = images.shape[1:]
#self.params.data['dims'] = images.shape[1:]
Y = np.transpose(images, list(range(1, len(self.dims) + 1)) + [0])
Yr = np.transpose(np.reshape(images, (T, -1), order='F'))
if np.isfortran(Yr):
raise Exception(
'The file is in F order, it should be in C order (see save_memmap function)'
)
logging.info((T, ) + self.dims)
# Make sure filename is pointed correctly (numpy sets it to None sometimes)
try:
Y.filename = images.filename
Yr.filename = images.filename
self.mmap_file = images.filename
except AttributeError: # if no memmapping cause working with small data
pass
# update/set all options that depend on data dimensions
# number of rows, columns [and depths]
self.params.set(
'spatial', {
'medw': (3, ) * len(self.dims),
'se': np.ones((3, ) * len(self.dims), dtype=np.uint8),
'ss': np.ones((3, ) * len(self.dims), dtype=np.uint8)
})
logging.info(('Using ' + str(self.params.get('patch', 'n_processes')) +
' processes'))
if self.params.get('preprocess', 'n_pixels_per_process') is None:
avail_memory_per_process = psutil.virtual_memory(
)[1] / 2.**30 / self.params.get('patch', 'n_processes')
mem_per_pix = 3.6977678498329843e-09
npx_per_proc = np.int(avail_memory_per_process / 8. / mem_per_pix /
T)
npx_per_proc = np.int(
np.minimum(
npx_per_proc,
np.prod(self.dims) //
self.params.get('patch', 'n_processes')))
self.params.set('preprocess',
{'n_pixels_per_process': npx_per_proc})
self.params.set(
'spatial', {
'n_pixels_per_process':
self.params.get('preprocess', 'n_pixels_per_process')
})
logging.info(
'using ' +
str(self.params.get('preprocess', 'n_pixels_per_process')) +
' pixels per process')
logging.info('using ' +
str(self.params.get('spatial', 'block_size_spat')) +
' block_size_spat')
logging.info('using ' +
str(self.params.get('temporal', 'block_size_temp')) +
' block_size_temp')
if self.params.get('patch', 'rf') is None: # no patches
logging.info('preprocessing ...')
Yr = self.preprocess(Yr)
if self.estimates.A is None:
logging.info('initializing ...')
self.initialize(Y)
if self.params.get(
'patch',
'only_init'): # only return values after initialization
if not (self.params.get('init', 'method_init') == 'corr_pnr'
and self.params.get('init',
'ring_size_factor') is not None):
self.compute_residuals(Yr)
self.estimates.bl = None
self.estimates.c1 = None
self.estimates.neurons_sn = None
if self.remove_very_bad_comps:
logging.info('removing bad components : ')
final_frate = 10
r_values_min = 0.5 # threshold on space consistency
fitness_min = -15 # threshold on time variability
fitness_delta_min = -15
Npeaks = 10
traces = np.array(self.C)
logging.info('estimating the quality...')
idx_components, idx_components_bad, fitness_raw,\
fitness_delta, r_values = estimate_components_quality(
traces, Y, self.estimates.A, self.estimates.C, self.estimates.b, self.estimates.f,
final_frate=final_frate, Npeaks=Npeaks, r_values_min=r_values_min,
fitness_min=fitness_min, fitness_delta_min=fitness_delta_min, return_all=True, N=5)
logging.info(
('Keeping ' + str(len(idx_components)) +
' and discarding ' + str(len(idx_components_bad))))
self.estimates.C = self.estimates.C[idx_components]
self.estimates.A = self.estimates.A[:,
idx_components] # type: ignore # not provable that self.initialise provides a value
self.estimates.YrA = self.estimates.YrA[idx_components]
self.estimates.normalize_components()
return self
logging.info('update spatial ...')
self.update_spatial(Yr, use_init=True)
logging.info('update temporal ...')
if not self.skip_refinement:
# set this to zero for fast updating without deconvolution
self.params.set('temporal', {'p': 0})
else:
self.params.set('temporal',
{'p': self.params.get('preprocess', 'p')})
logging.info('deconvolution ...')
self.update_temporal(Yr)
if not self.skip_refinement:
logging.info('refinement...')
if self.params.get('merging', 'do_merge'):
logging.info('merging components ...')
self.merge_comps(Yr,
mx=50,
fast_merge=True,
max_merge_area=self.params.get(
'merging', 'max_merge_area'))
logging.info('Updating spatial ...')
self.update_spatial(Yr, use_init=False)
# set it back to original value to perform full deconvolution
self.params.set('temporal',
{'p': self.params.get('preprocess', 'p')})
logging.info('update temporal ...')
self.update_temporal(Yr, use_init=False)
# else:
# todo : ask for those..
# C, f, S, bl, c1, neurons_sn, g1, YrA, lam = self.estimates.C, self.estimates.f, self.estimates.S, self.estimates.bl, self.estimates.c1, self.estimates.neurons_sn, self.estimates.g, self.estimates.YrA, self.estimates.lam
# embed in the whole FOV
if is_sliced:
FOV = np.zeros(dims_orig, order='C')
FOV[indices[1:]] = 1
FOV = FOV.flatten(order='F')
ind_nz = np.where(FOV > 0)[0].tolist()
self.estimates.A = self.estimates.A.tocsc()
A_data = self.estimates.A.data
A_ind = np.array(ind_nz)[self.estimates.A.indices]
A_ptr = self.estimates.A.indptr
A_FOV = scipy.sparse.csc_matrix(
(A_data, A_ind, A_ptr),
shape=(FOV.shape[0], self.estimates.A.shape[-1]))
b_FOV = np.zeros((FOV.shape[0], self.estimates.b.shape[-1]))
b_FOV[ind_nz] = self.estimates.b
self.estimates.A = A_FOV
self.estimates.b = b_FOV
else: # use patches
if self.params.get('patch', 'stride') is None:
self.params.set('patch', {
'stride':
np.int(self.params.get('patch', 'rf') * 2 * .1)
})
logging.info(
('Setting the stride to 10% of 2*rf automatically:' +
str(self.params.get('patch', 'stride'))))
if type(images) is np.ndarray:
raise Exception(
'You need to provide a memory mapped file as input if you use patches!!'
)
self.estimates.A, self.estimates.C, self.estimates.YrA, self.estimates.b, self.estimates.f, \
self.estimates.sn, self.estimates.optional_outputs = run_CNMF_patches(
images.filename, self.dims + (T,), self.params,
dview=self.dview, memory_fact=self.params.get('patch', 'memory_fact'),
gnb=self.params.get('init', 'nb'), border_pix=self.params.get('patch', 'border_pix'),
low_rank_background=self.params.get('patch', 'low_rank_background'),
del_duplicates=self.params.get('patch', 'del_duplicates'),
indices=indices)
self.estimates.bl, self.estimates.c1, self.estimates.g, self.estimates.neurons_sn = None, None, None, None
logging.info("merging")
self.estimates.merged_ROIs = [0]
if self.params.get('init',
'center_psf'): # merge taking best neuron
if self.params.get('patch', 'nb_patch') > 0:
while len(self.estimates.merged_ROIs) > 0:
self.merge_comps(Yr, mx=np.Inf, fast_merge=True)
logging.info("update temporal")
self.update_temporal(Yr, use_init=False)
self.params.set(
'spatial', {
'se': np.ones(
(1, ) * len(self.dims), dtype=np.uint8)
})
logging.info('update spatial ...')
self.update_spatial(Yr, use_init=False)
logging.info("update temporal")
self.update_temporal(Yr, use_init=False)
else:
while len(self.estimates.merged_ROIs) > 0:
self.merge_comps(Yr, mx=np.Inf, fast_merge=True)
if len(self.estimates.merged_ROIs) > 0:
not_merged = np.setdiff1d(
list(range(len(self.estimates.YrA))),
np.unique(
np.concatenate(
self.estimates.merged_ROIs)))
self.estimates.YrA = np.concatenate([
self.estimates.YrA[not_merged],
np.array([
self.estimates.YrA[m].mean(0) for ind, m in
enumerate(self.estimates.merged_ROIs)
if not self.empty_merged[ind]
])
])
if self.params.get('init', 'nb') == 0:
self.estimates.W, self.estimates.b0 = compute_W(
Yr,
self.estimates.A.toarray(),
self.estimates.C,
self.dims,
self.params.get('init', 'ring_size_factor') *
self.params.get('init', 'gSiz')[0],
ssub=self.params.get('init', 'ssub_B'))
if len(self.estimates.C):
self.deconvolve()
else:
self.estimates.S = self.estimates.C
else:
while len(self.estimates.merged_ROIs) > 0:
self.merge_comps(Yr, mx=np.Inf)
logging.info("update temporal")
self.update_temporal(Yr, use_init=False)
self.estimates.normalize_components()
return self
def is_c_contiguous(a):
return np.isfortran(a.T)
def nnls_regularized(A, b, l=0, maxiter=None):
"""
Solve math:`argmin_x || Ax - b ||_2^2 + \lambda^2 ||x||_2^2` for ``x>=0``.
This is a wrapper for a FORTRAN non-negative least squares solver,
with regularization (added by stacking $A$ on top an identity matrix
times $\lambda$ and $b$ on top of a matching array of zero.
Parameters
----------
A : ndarray
Matrix ``A`` as shown above.
b : ndarray
Right-hand side vector.
l : double (default 0)
:math:`lambda` -- if this is set to 0, the algorithm reverts to
standard nnls (rather than stacking on top of two zero matrices
for no reason)
maxiter: int, optional
Maximum number of iterations, optional.
Default is ``3 * A.shape[1]``.
Returns
-------
x : ndarray
Solution vector.
rnorm : float
The residual, ``|| Ax-b ||_2``.
Notes
-----
The FORTRAN code was published in the book below. The algorithm
is an active set method. It solves the KKT (Karush-Kuhn-Tucker)
conditions for the non-negative least squares problem.
This was adapted from the source distributed with scipy --
see scipy for relevant licensing.
References
----------
Lawson C., Hanson R.J., (1987) Solving Least Squares Problems, SIAM
"""
logger.debug(strm("isfortran result",isfortran(A),isfortran(b)))
A, b = list(map(asarray_chkfinite, (A, b)))
if len(A.shape) != 2:
raise ValueError("expected matrix")
if len(b.shape) > 2:
raise ValueError("expected vector")
m, n = A.shape
if m != b.shape[-1]:
raise ValueError(strm("incompatible dimensions (rows of A",m," do not match size of data",b.shape,")"))
maxiter = -1 if maxiter is None else int(maxiter)
if isscalar(l):
if l == 0.0:
w = zeros((n,), dtype=double)
zz = zeros((m,), dtype=double)
index = zeros((n,), dtype=int)
x, rnorm, mode = _nnls.nnls(A, b, w, zz, index, maxiter)
else:
w = zeros((n,), dtype=double)
zz = zeros((m+n,), dtype=double)
index = zeros((n,), dtype=int)
# choose the correct subroutine based on the dimension
if len(b.shape) == 1:
x, rnorm, mode = _nnls.nnls_regularized(A, b, w, zz, index, maxiter, l)
if len(b.shape) == 2:
x, rnorm, mode = _nnls.nnls_regularized_loop(A, redim_C_to_F(b), w, zz, index, maxiter, l)
x = redim_F_to_C(x)
else:
nCPU = cpu_count()
#print("I found",nCPU,"CPU's")
p = mpd.Pool(nCPU)
if len(b.shape) == 1:
def nnls_func(l):
w = zeros((n,), dtype=double)
zz = zeros((m+n,), dtype=double)
index = zeros((n,), dtype=int)
return _nnls.nnls_regularized(A, b, w, zz, index, maxiter, l)
if len(b.shape) == 2:
def nnls_func(l):
w = zeros((n,), dtype=double)
zz = zeros((m+n,), dtype=double)
index = zeros((n,), dtype=int)
x, rnorm, mode = _nnls.nnls_regularized_loop(A, redim_C_to_F(b), w, zz, index, maxiter, l)
return redim_F_to_C(x), rnorm, mode
retval = p.map(nnls_func,l)
x,rnorm,mode = list(map(np_array,list(zip(*retval))))
if (isscalar(mode) and mode != 1):
# need something for the multiple lambda
raise RuntimeError("too many iterations")
return x, rnorm
