def SelectLeastsq(array, ref=None, axis=0, cut=None):
'''
array:
N-D array, for example, a.shape=4x5x6
I want to get a[:,i,:] which is the most resonable one along the second axis
ref:
reference to judge
(1) ==ndarray: array-ref, can broadcast
(2) =='mean': array-array.mean(axis)
(3) =='eachother': use the difference of each other to judge
axis:
along which axis of array
cut:
(1) None/False: return all
(2) True/int: cut automatically
vmax = m.max() + cut*(m.max()-std.min())
return:
norder.size = array.shape[axis]
'''
import numpy as np
from jizhipy.Array import ArrayAxis
from jizhipy.Basic import IsType
from jizhipy.Optimize import Medfilt
# check axis
Ntot = array.shape[axis]
# ref
try:
array = array - ref
ref = 'give'
except:
ref = str(ref).lower()
if (ref != 'mean'): ref = 'eachother'
# Move axis to 0
array = ArrayAxis(array, axis, 0, 'move')
if (ref != 'eachother'):
if (ref == 'mean'): array = array - array.mean(0)
sidx = tuple(range(len(array.shape)))
std = (array**2).mean(sidx[1:])
else:
std = []
for i in range(Ntot):
a = array - array[i]
a = (a**2).mean()
std.append(a)
std = np.array(std)
norder = std + 1j * np.arange(Ntot)
norder = np.sort(norder).imag.astype(int)
std = std[norder]
# cut
if (not IsType.isfalse(cut)):
m = Medfilt(std, 0, std.size / 5)
vmax = m.max() + cut * (m.max() - std.min())
norder = norder[std <= vmax]
std = std[std <= vmax]
return [norder, std]
def Sample(array, axis, per=None, size=None):
'''
Reduce the size of array
array:
Any shape, any dtype
int/float/complex ndarray
MaskedArray
axis:
Sampling along which axis?
per, size:
Use one of them
(1) per, size=None:
Sampling every per ponts
(2) per=None, size:
Sampled array.shape[axis] == size
'''
import numpy as np
from jizhipy.Basic import IsType, Raise
from jizhipy.Array import ArrayAxis
if (not IsType.isndarray(array) and not IsType.isnmaskedarray(array)
and not IsType.ismatrix(array)):
array = np.array(array)
if (len(array.shape) == 1): axis = 0
elif (axis < 0): axis += len(array.shape)
if (axis >= len(array.shape)):
Raise(Exception,
'axis=' + str(axis) + ' out of array.shape=' + str(array.shape))
#----------------------------------------
if (per is None and size is None):
return array
#----------------------------------------
elif (per is None and array.shape[axis] >= size):
return Smooth(array, axis, reduceshape=size)
#----------------------------------------
elif (per is not None):
per += 0
n = np.arange(0, array.shape[axis], per)
else:
size += 0
n = np.linspace(0, array.shape[axis] - 1, size)
n = n.astype(int)
#----------------------------------------
array = ArrayAxis(array, axis, 0)
array = array[n]
array = ArrayAxis(array, 0, axis)
return array
def Sort( array, along='[0,:]', l2s=False ) :
'''
array:
Can be any shape
along:
Must as format like '[n1,n2,:,n4]'
Must have ':', use [2,:] instead of [2]
array[n1,n2,:,n4] must 1D so that we can sort along this
along=[:,2] : second column
'[0,:]' => 0-row
'[:,0]' => 0-column
l2s:
l2s=False: from small to large (default)
l2s=True : from large to small
'''
import numpy as np
from jizhipy.Array import Asarray, ArrayAxis
along = along[1:-1].split(',')
axis = along.index(':')
along.pop(axis)
along = np.array(along, int)
#--------------------------------------------------
array = Asarray(array)
if (len(array.shape) == 1) :
array = np.sort(array)
if (l2s) : array = array[::-1]
return array
#--------------------------------------------------
if (array.shape[axis] == 1) : return array
array = ArrayAxis(array, axis, -1, 'move')
shape = array.shape
#--------------------------------------------------
cumprod = np.cumprod((shape[1:-1]+(1,))[::-1])[::-1]
along = (along*cumprod).sum()
a = array.reshape(np.prod(shape[:-1]), shape[-1])[along]
#--------------------------------------------------
a = a + 1j*np.arange(a.size)
a = np.sort(a).imag.astype(int)
if (l2s) : a = a[::-1]
#--------------------------------------------------
array = ArrayAxis(array, -1, 0, 'move')
array = array[a]
array = ArrayAxis(array, 0, axis, 'move')
return array
def ArraySplit( array, axis, which ) : ''' axis: split along which axis which: =='1G' | '1GB' ==int ''' import numpy as np from jizhipy.Array import Asarray, ArrayAxis, Repection from jizhipy.Basic import IsType array = Asarray(array) if (IsType.isint(which)) : mem, N = None, which elif (IsType.isstr(which)) : which = str(which).lower() if ((which[-1]=='g' and which[-2] in '0123456789')) : mem = float(which[:-1]) elif ((which[-2:]=='gb' and which[-3] in '0123456789')) : mem = float(which[:-2]) # GB else : array = ArrayAxis(array, axis, 0, 'move') array = array[None,:] array = ArrayAxis(array, 0, axis+1, 'move') return array #-------------------------------------------------- if (mem is not None) : bit, n = array.dtype.name, 0 while (bit[n] not in '0123456789') : n += 1 bit = float(bit[n:]) size = array.size memtot = size / 1e8 * 0.8 * bit/64 N = int(memtot / mem) + 1 m = np.linspace(0, array.shape[axis], N+1).astype(int) m = Repetition(m, renon=True) b = [] array = ArrayAxis(array, axis, 0, 'move') for i in range(len(m)-1) : a = ArrayAxis(array[m[i]:m[i+1]], 0, axis) b.append(a) return b
def _RotationMatrix(self, key, ang, islist):
''' return R, not call directly '''
from jizhipy.Array import ArrayAxis
import numpy as np
one = np.ones(ang.shape)
zero = np.zeros(ang.shape)
if (key == 'ax'):
R = np.array([[one, zero, zero], [zero,
np.cos(ang),
np.sin(ang)],
[zero, -np.sin(ang), np.cos(ang)]])
elif (key == 'ay'):
R = np.array([[np.cos(ang), zero, -np.sin(ang)], [zero, one, zero],
[np.sin(ang), zero, np.cos(ang)]])
elif (key == 'az'):
R = np.array([[np.cos(ang), np.sin(ang), zero],
[-np.sin(ang), np.cos(ang), zero], [zero, zero,
one]])
else:
Raise(Exception, "key in **kwargs not in ['ax', 'ay', 'az']")
R = ArrayAxis(R, 0, -1, 'move')
R = ArrayAxis(R, 0, -1, 'move')
if (not islist): R = R[0]
return R
def ResetMasked(maskedarray, axis, Nprocess=None, fast=0, nfast=None):
'''
Use the elements around the masked elements to calculate some values, then use these values to fill/reset the masked elements
Return a non-masked array
maskedarray:
Any dimension np.ma.MaskedArray()
axis:
Along which axis to guess the values
fast:
How fast to do ResetMasked()?
fast in [0, 1, 2]
(1) ==0/None: fastest, not do Smooth() nor Medfilt()
(2) ==1: middle, do Smooth()
(3) ==2: slowest, do Medfilt()
nfast:
For fast in [1, 2]
(1) fast==1: Smooth(a, 0, 3, nfast)
(2) fast==2: Medfilt(a, 0, nfast)
return:
resetvalue = ResetMasked(...)
Usage: maskedarray[maskedarray.mask] = resetvalue
'''
import numpy as np
from jizhipy.Process import PoolFor, NprocessCPU
from jizhipy.Array import ArrayAxis
if (maskedarray.mask[maskedarray.mask].size == 0): return np.array([])
Nprocess = NprocessCPU(Nprocess)[0]
mask = (1 - maskedarray.mask).astype(bool)
if (axis == 0):
maskedarray = maskedarray.data.T
mask = mask.T
else:
maskedarray = ArrayAxis(maskedarray.data, axis, -1, 'move')
mask = ArrayAxis(mask, axis, -1, 'move')
shape = mask.shape
maskedarray = maskedarray.reshape(int(np.prod(shape[:-1])), shape[-1])
mask = mask.reshape(int(np.prod(shape[:-1])), shape[-1])
#--------------------------------------------------
try:
fast = int(fast)
except:
fast = 0
try:
nfast = int(nfast)
except:
nfast = 3
send = (maskedarray, mask)
bcast = (fast, nfast)
if (Nprocess <= 1):
iterable = (None, send, bcast)
maskedarray = _DoMultiprocess_ResetMasked(iterable)
else:
pool = PoolFor(0, mask.shape[0], Nprocess)
maskedarray = pool.map_async(_DoMultiprocess_ResetMasked, send, bcast)
maskedarray = np.concatenate(maskedarray, 0)
#--------------------------------------------------
maskedarray, mask = maskedarray.reshape(shape), mask.reshape(shape)
if (axis == 0):
maskedarray = maskedarray.T
mask = mask.T
else:
maskedarray = ArrayAxis(maskedarray, -1, axis, 'move')
mask = ArrayAxis(mask, -1, axis, 'move')
mask = (1 - mask).astype(bool)
return maskedarray[mask]
def Smooth(array,
axis,
per=3,
times=None,
std=None,
filt=None,
sigma=False,
reduceshape=False,
nlr=True,
fft=False,
Nprocess=1):
'''
Smooth/Average/Mean array along one axis.
We can also use spsn.convolve() to do this, but spsn.convolve() will cost much more memory and time, so, the function written here is the best and fastest.
array:
Any shape, any dtype
(1) int/float/complex array: faster
(2) MaskedArray: slower
nlr:
Large times/std will make the left and right edge worse and worse
Select different nlr, the edge effect wil be different. Try different nlr and use the best case !!!
(1) ==None: append first/last "element"
(2) ==True: set nlr=[len(array)/100, len(array)/100]
(3) ==[int, int]: set nlr=[int, int] # interp1d
(4) isnum (float or int): as the outside value (nlr=0)
(5) =='periodic': append right end to the left head, left head to right end
(6) ==False: don't append, always filt.sum()==1, completely ==Convolve(array, filt)
(7) =='mirror': append mirror
axis:
array will be smoothed/averaged along this axis.
** First use (per, times), second use std, third use filt
per, times:
int, int/'stock'
Smooth times, for each time, the window size=per: equivalent to use GaussianFilter(per, times) to convolve array
std:
isnum, unit is "pixel"
The std of GaussianFilter
std = fwhm / (8*np.log(2))**0.5
filt:
Use this filter directly, any size
NOTE THAT must filt.sum()==1 (normalized), otherwise the result may be bad
sigma:
Used in case 1,2,3,4, NOT 5
False, True, int, np.array
(1) ==False: don't return the error
(2) ==True: calculate the error of the output
(3) ==isnum or np.array: use this sigma to calculate the error of the output
reduceshape:
(1) ==False: return.shape = array.shape
(2) ==True: used togather with per, reduce the data
(3) ==int: return.shape[axis]==this int, this int can >= or < array.shape[axis], any value you want
fft:
True | False
Use FFT to convolve or NOT?
Note that:
Wide of 2 windows: w1 < w2
a1 = Convolve(a, w1)
a2 = Convolve(a, w2)
a12 = Convolve(a1, w2)
=> a12 = Smooth(a2)
But the beam sizes of the result maps are similar (roughly the same), Beamsize(a12) >= Beamsize(a2).
(1) Smooth( a, axis, per, times )
(2) Smooth( a, axis, std=std )
(3) Smooth( a, axis, filt=filt )
(4) Smooth( a, axis, per, reduceshape=True )
(5) Smooth( a, axis, reduceshape=int number )
'''
import numpy as np
from jizhipy.Basic import IsType, Raise
from jizhipy.Array import ArrayAxis, Asarray
from jizhipy.Process import PoolFor, NprocessCPU
array = Asarray(array)
dtype, shape = array.dtype.name, array.shape
if (len(shape) == 1): axis = 0
if (axis < 0): axis = len(shape) + axis
if (axis >= len(shape)):
Raise(Exception,
'axis=' + str(axis) + ' out of array.shape=' + str(shape))
#--------------------------------------------------
if (per is not None and IsType.isstr(times)):
per, times = int(per), str(times).lower()
if ('stock' not in times):
Raise(Exception, 'per is int, but times = ' + times + ' != stock')
case = 6
#--------------------------------------------------
elif (per is not None and times is not None):
per, times, case = int(per), int(times), 1
elif (std is not None):
std, case = float(std), 2
elif (filt is not None):
filt, case = Asarray(filt), 3
elif (per is not None and reduceshape is True):
per, case = int(per), 4
elif (IsType.isnum(reduceshape)):
N, sigma, case = int(reduceshape), False, 5
#--------------------------------------------------
if (case == 1 and (per <= 1 or times <= 0)):
return array
# elif (case == 2 and std < 1) : return array
elif (case == 3 and abs(filt).max() < 1e-9):
return array
elif (case == 4 and per <= 1):
return array
elif (case == 5 and N == shape[axis]):
return array
#--------------------------------------------------
#--------------------------------------------------
# int array to float32
if ('int' in dtype):
dtype = 'float32'
array = array.astype(dtype)
#--------------------------------------------------
# Move axis to axis=0, smooth along axis=0
array = ArrayAxis(array, axis, 0, 'move')
shape0 = array.shape
# N-D array to 2D | matrix
if (len(shape0) == 1): array = array[:, None]
elif (len(shape0) > 2):
array = array.reshape(shape0[0], int(np.prod(shape0[1:])))
shape = array.shape
# shape0: after ArrayAxis(), smooth along shape0[0]
# shape: 2D
#--------------------------------------------------
Nprocess = NprocessCPU(Nprocess)[0]
if (Nprocess > shape[1]): Nprocess = shape[1]
#--------------------------------------------------
#--------------------------------------------------
if (case == 5): # Smooth(array, axis, reduceshape=int)
# Now smooth along axis=0
if (shape[1] > N):
n = np.linspace(0, shape[0], N + 1).astype(int)
n = np.array([n[:-1], n[1:]]).T
if (Nprocess <= 1):
iterable = (None, (array, n), None)
array = _Multiprocess_SmoothLarge(iterable)
else:
pool, send = PoolFor(0, N, Nprocess), []
ns, a = pool.nsplit, []
for i in range(len(ns)):
m = n[ns[i, 0]:ns[i, 1]]
a = array[m[0, 0]:m[-1, 1]]
m -= m[0, 0]
send.append([a, m])
pool = PoolFor()
array = pool.map_async(_Multiprocess_SmoothLarge, send)
array = np.concatenate(array, 0)
#----------------------------------------
else:
if (Nprocess <= 1):
iterable = (None, array.T, N)
array = _Multiprocess_SmoothSmall(iterable)
else:
pool = PoolFor(0, array.shape[1], Nprocess)
array = pool.map_async(_Multiprocess_SmoothSmall, array.T, N)
array = np.concatenate(array, 1)
array = array.reshape((len(array), ) + shape0[1:])
#--------------------------------------------------
#--------------------------------------------------
# per is even, b[i] = a[i+1-per/2:i+1+per/2]
# per is odd, b[i] = a[i -per/2:i+1+per/2]
# per is even, left end + [:per/2-1], right end + [-per/2:]
# per is odd, left end + [:per/2 ], right end + [-per/2:]
#--------------------------------------------------
# nlr:
# If reduceshape==False, large times will make the left and right edge worse and worse
# (1) ==False/None: append first/last element
# (2) ==True: set nlr=[len(array)/100, len(array)/100]
# (3) ==[int, int]: set nlr=[int, int]
# (4) isnum (float or int): as the outside value
# (5) =='periodic': append right end to the left head, left head to right end
elif (case in [1, 2, 3, 6]):
# Smooth(array, axis, per, times)
# Smooth(array, axis, std=std)
# Smooth(array, axis, filt=filt)
# Smooth(array, axis, per, 'stock')
if (case == 1): weight = GaussianFilter(per, times)[0]
elif (case == 2): weight = GaussianFilter(None, None, std, shape[0])[0]
elif (case == 3): weight = filt
elif (case == 6): weight = GaussianFilter(per, 1)[0] #@!
# Normalized
weight /= weight.sum()
# Cut the filter in order to faster
weight = weight[weight > 3e-5 * weight.max()] #@#@#@
Nw = len(weight)
#--------------------------------------------------
if (case == 6): nla, nra, nlr = Nw - 1, 0, None #@!
else:
nla = int(Nw / 2) if (Nw % 2 == 1) else int(Nw / 2) - 1
nra = int(Nw / 2)
if (nlr is None): casenlr = 1
elif (nlr is True):
n = int(len(array) / 100)
if (n < 5): n = 5
nlr, casenlr = [n, n], 2
elif (IsType.isnum(nlr)):
casenlr = 4
elif (
IsType.isstr(nlr) and
(str(nlr).lower() == 'periodic' or str(nlr).lower()[:3] == 'per')):
casenlr = 5
elif (IsType.isstr(nlr) and str(nlr).lower() == 'mirror'):
casenlr = 7
elif (nlr is False):
nlr, casenlr = np.nan, 6
else:
nlr, casenlr = nlr[:2], 3
#--------------------------------------------------
# nla, nra = nla+1, nra+1
bcast = [weight, nla, nra, sigma, nlr, casenlr, fft]
if (Nprocess == 1):
iterable = ((None, None), array.T, bcast)
array = _Multiprocess_Smooth(iterable)
else:
pool = PoolFor(0, array.shape[1], Nprocess)
array = pool.map_async(_Multiprocess_Smooth, array.T, bcast)
array = np.concatenate(array, 1)
if (sigma is True):
arrstd = array[shape[0]:].reshape(shape0)
array = array[:shape[0]]
array = array.reshape(shape0)
#--------------------------------------------------
#--------------------------------------------------
elif (case == 4): # Smooth(array, axis, per, reduceshape=True)
shape = array.shape
if (Nprocess > shape[0]): Nprocess = shape[0]
bcast = (per, sigma)
#--------------------------------------------------
if (Nprocess <= 1):
iterable = (None, array, bcast)
array = _Multiprocess_SmoothReduce(iterable)
#--------------------------------------------------
else:
# n2 = PoolFor(0, len(n1), Nprocess).nsplit
# send = []
# for i in range(len(n2)) :
# j, k = n2[i]
# n3 = n1[j:k]
# send.append([n3, array[n3.min():n3.max()]])
n1 = np.append(np.arange(0, shape[0], per), [shape[0]])
n1 = np.array([n1[:-1], n1[1:]]).T
n2 = np.linspace(0, len(n1), Nprocess + 1).round().astype(int)
nsplit, send = [], []
for i in range(len(n2) - 1):
n = n1[n2[i]:n2[i + 1]]
if (n.size > 0): nsplit.append(n)
for i in range(len(nsplit)):
n1, n2 = nsplit[i].min(), nsplit[i].max()
send.append(array[n1:n2])
pool = PoolFor()
array = pool.map_async(_Multiprocess_SmoothReduce, send, bcast)
array = np.concatenate(array, 0)
if (sigma):
array, arrstd = array[:, 0], array[:, 1]
arrstd = arrstd.reshape((len(array), ) + shape0[1:])
array = array.reshape((len(array), ) + shape0[1:])
#--------------------------------------------------
#--------------------------------------------------
array = ArrayAxis(array, 0, axis, 'move')
array = array.astype(dtype)
if (sigma):
arrstd = ArrayAxis(arrstd, 0, axis, 'move')
arrstd = arrstd.astype(dtype)
return [array, arrstd]
return array
def Medfilt(array, axis, kernel_size, Nprocess=1):
'''
array:
Any shape
dtype:
int/float/complex
NOT MaskedArray
axis:
None | int number
(1) ==None:
use scipy.signal.medfilt(array, kernel_size)
do 1D, 2D, 3D, ..., N-D medfilt
(2) ==int number:
mediam filter along this axis
do 1D medfilt for all axes
'''
if (kernel_size is None or kernel_size < 3): return array
import scipy.signal as spsn
import numpy as np
from jizhipy.Array import Asarray, ArrayAxis
from jizhipy.Process import PoolFor, NprocessCPU
Nprocess = NprocessCPU(Nprocess)[0]
array, kernel_size = Asarray(array), Asarray(kernel_size).flatten()
shape, dtype = array.shape, array.dtype
array = 1. * array
tf = (kernel_size % 2 == 0)
kernel_size[tf] += 1
kernel_size = kernel_size[:len(shape)]
kernel_size = np.append(
kernel_size,
[kernel_size[0] for i in range(len(shape) - len(kernel_size))])
kernel_size = kernel_size.astype(int)
tf = False if (dtype.name[:7] == 'complex') else True
if (axis is None):
if (len(shape) == 2):
if (tf): array = spsn.medfilt2d(array, kernel_size)
else:
array = spsn.medfilt2d(
array.real,
kernel_size) + 1j * spsn.medfilt2d(array.imag, kernel_size)
else:
if (tf): array = spsn.medfilt(array, kernel_size)
else:
array = spsn.medfilt(
array.real,
kernel_size) + 1j * spsn.medfilt(array.imag, kernel_size)
else:
axis = int(round(axis))
if (axis < 0): axis = len(shape) + axis
if (axis >= len(shape)):
Raise(
Exception, 'axis=' + str(axis) + ' out of ' + str(len(shape)) +
'D array.shape=' + str(shape))
kernel_size = kernel_size[axis]
if (shape[axis] == 1): return array
if (len(shape) == 1):
if (tf): array = spsn.medfilt(array, kernel_size)
else:
array = spsn.medfilt(
array.real,
kernel_size) + 1j * spsn.medfilt(array.imag, kernel_size)
else:
array = ArrayAxis(array, axis, -1)
shape = array.shape
array = array.reshape(np.prod(shape[:-1]), shape[-1])
sent = array
bcast = [kernel_size, tf]
if (Nprocess == 1):
array = _Multiprocess_medfilt([None, sent, bcast])
else:
pool = PoolFor(0, len(array), Nprocess)
array = pool.map_async(_Multiprocess_medfilt, sent, bcast)
array = np.concatenate(array, 0)
array = array.reshape(shape)
array = ArrayAxis(array, -1, axis)
array = array.astype(dtype)
return array
def _xyzRotation(self, xyz, **kwargs):
'''
(x,y,z) coordinates rotation with ax,ay,az=
xyz:
xyz can be any shape, but must:
x, y, z = xyz
x = sin(theta) * cos(phi)
y = sin(theta) * sin(phi)
z = cos(theta)
**kwargs:
See self.xyzRotationMatrix()
ax=None: means NOT rotate this axis
(1) xyzRotation(xyz, ay=1, ax=2, az=3)
(2)
ang = {'ay':1, 'ax':2, 'az':3, 'order':['ay','ax','az']}
xyzRotation(xyz, **ang)
return:
Same shape and type as input xyz
xyz_new.shape = xyz.shape+(3,) # (3,) for x,y,z
'''
from jizhipy.Basic import OrderKwargs, Raise
from jizhipy.Array import ArrayAxis, Asarray
xyz = Asarray(xyz, True, float)
shapexyz = xyz.shape
if (shapexyz[0] != 3):
Raise(Exception, 'xyz.shape=' + str(shapexyz) + ', shape[0] != 3')
if (xyz.shape == (3, )): islistx = False
else: islistx = True
x, y, z = xyz
#--------------------------------------------------
try:
kwargs['order']
except:
kwargs['order'] = OrderKwargs()
if ('which' in kwargs.keys()):
which = kwargs['which']
else:
which = 'system'
R = self.xyzRotationMatrix(which, **kwargs)
shapeR = R.shape
R = ArrayAxis(R, -2, 0, 'move') #@#@
if (R.shape == (3, 3)): islistr = False
else: islistr = True
Rx, Ry, Rz = R.T
Rx, Ry, Rz = Rx.T, Ry.T, Rz.T
#--------------------------------------------------
if (Rx.shape == (3, )):
sR = Rx.shape + len(x.shape) * (1, )
Rx, Ry, Rz = Rx.reshape(sR), Ry.reshape(sR), Rz.reshape(sR)
try:
xyz = x * Rx + y * Ry + z * Rz
except:
Raise(
Exception, 'x.shape=' + str(x.shape) + ', Rx.shape=' +
str(Rx.shape) + ', can NOT broadcast')
return xyz