def destroy_with_variance_2pol(Data, sigma_thres=6, bad_freq_list=[], submean=True):
'''Mask frequencies with high variance.
This is the same as last function, but for Parkes 2 pol data.
'''
# Get the normalized variance array for each polarization.
#Data.data[Data.data>3] = ma.masked
#Data.data[Data.data<3] = ma.masked
Data.data[np.isnan(Data.data)] = ma.masked
Data.data[Data.data <= 0.] = ma.masked
if submean:
a = ma.var(Data.data[:,0,0,:],0)/(ma.mean(Data.data[:,0,0,:],0)**2)#XX
b = ma.var(Data.data[:,1,0,:],0)/(ma.mean(Data.data[:,1,0,:],0)**2)#YY
else:
a = ma.var(Data.data[:,0,0,:],0)
b = ma.var(Data.data[:,1,0,:],0)
# Get the mean and standard deviation [sigma].
means = sp.array([ma.mean(a), ma.mean(b)])
sig = sp.array([ma.std(a), ma.std(b)])
# Get the max accepted value [sigma_thres*sigma, sigma_thres=6 works really well].
max_sig = sigma_thres*sig
max_accepted = means + max_sig
min_accepted = means - max_sig
amount_masked = 0
for freq in range(0, len(a)):
if ((a[freq] > max_accepted[0]) or (b[freq] > max_accepted[1]) or
(a[freq] < min_accepted[0]) or (b[freq] < min_accepted[1])):
# mask
amount_masked += 1
bad_freq_list.append(freq)
Data.data[:,:,:,freq].mask = True
return amount_masked
def attest_ind(a, b, dim=None):
""" Return the t-test statistics on arrays a and b over the dim axis.
Returns both the t statistic as well as the p-value
"""
# dim = a.ndim - 1 if dim is None else dim
x1, x2 = ma.mean(a, dim), ma.mean(b, dim)
v1, v2 = ma.var(a, dim), ma.var(b, dim)
n1, n2 = (a.shape[dim], b.shape[dim]) if dim is not None else (a.size, b.size)
df = float(n1+n2-2)
svar = ((n1-1)*v1+(n2-1)*v2) / df
t = (x1-x2)/ma.sqrt(svar*(1.0/n1 + 1.0/n2))
if t.ndim == 0:
return (t, scipy.stats.betai(0.5*df,0.5,df/(df+t**2)) if t is not ma.masked and df/(df+t**2) <= 1.0 else ma.masked)
else:
prob = [scipy.stats.betai(0.5*df,0.5,df/(df+tsq)) if tsq is not ma.masked and df/(df+tsq) <= 1.0 else ma.masked for tsq in t*t]
return t, prob
def nothing(noth):
# If requested, remove the time gradient from all channels.
if remove_slope:
un_mask = sp.logical_not(ma.getmaskarray(NoiseData.data))
NoiseData.calc_time()
time = NoiseData.time
n_time = len(time)
# Test if the mask is the same for all slices. If it is, that greatly
# reduces the work as we only have to generate one set of polynomials.
all_masks_same = True
for jj in range(n_time):
if sp.all(un_mask[jj,...] == un_mask[jj,0,0,0]):
continue
else:
all_masks_same = False
break
if all_masks_same:
polys = misc.ortho_poly(time, 2, un_mask[:,0,0,0], 0)
polys.shape = (2, len(time), 1, 1, 1)
else:
polys = misc.ortho_poly(time[:,None,None,None], 2, un_mask, 0)
# Subtract the slope mode (1th mode) out of the NoiseData.
slope_amps = sp.sum(polys[1,...] * un_mask * NoiseData.data.filled(0),
0)
NoiseData.data -= polys[1,...] * slope_amps
# Iteratively flag on sliding scale to get closer and closer to desired
# threshold.
n_time = Data.data.shape[0]
max_thres = sp.sqrt(n_time)/2.
n_iter = 3
thresholds = (max_thres ** (n_iter - 1 - sp.arange(n_iter))
* thres ** sp.arange(n_iter)) ** (1./(n_iter - 1))
for threshold in thresholds:
# Get the deviation from the mean.
residuals = ma.anom(NoiseData.data, 0).filled(0)
# Get indices above the threshold.
mask = abs(residuals) > threshold * ma.std(NoiseData.data, 0)
# Mask the data.
Data.data[mask] = ma.masked
NoiseData.data[mask] = ma.masked
# Now flag for very noisey channels.
if max_noise_factor > 0:
vars = ma.var(NoiseData.data, 0)
mean_vars = ma.mean(vars, -1).filled(0)
bad_chans = vars.filled(0) > max_noise_factor * mean_vars[:,:,None]
Data.data[:,bad_chans] = ma.masked
NoiseData.data[:,bad_chans] = ma.masked
def _attvalues(attribute, stacked):
"""Attribute values computed in numpy.ma stack."""
if attribute == "max":
attvalues = ma.max(stacked, axis=2)
elif attribute == "min":
attvalues = ma.min(stacked, axis=2)
elif attribute == "rms":
attvalues = np.sqrt(ma.mean(np.square(stacked), axis=2))
elif attribute == "var":
attvalues = ma.var(stacked, axis=2)
elif attribute == "mean":
attvalues = ma.mean(stacked, axis=2)
elif attribute == "maxpos":
stacked = ma.masked_less(stacked, 0.0, copy=True)
attvalues = ma.max(stacked, axis=2)
elif attribute == "maxneg": # ~ minimum of negative values?
stacked = ma.masked_greater_equal(stacked, 0.0, copy=True)
attvalues = ma.min(stacked, axis=2)
elif attribute == "maxabs":
attvalues = ma.max(abs(stacked), axis=2)
elif attribute == "sumpos":
stacked = ma.masked_less(stacked, 0.0, copy=True)
attvalues = ma.sum(stacked, axis=2)
elif attribute == "sumneg":
stacked = ma.masked_greater_equal(stacked, 0.0, copy=True)
attvalues = ma.sum(stacked, axis=2)
elif attribute == "sumabs":
attvalues = ma.sum(abs(stacked), axis=2)
elif attribute == "meanabs":
attvalues = ma.mean(abs(stacked), axis=2)
elif attribute == "meanpos":
stacked = ma.masked_less(stacked, 0.0, copy=True)
attvalues = ma.mean(stacked, axis=2)
elif attribute == "meanneg":
stacked = ma.masked_greater_equal(stacked, 0.0, copy=True)
attvalues = ma.mean(stacked, axis=2)
else:
etxt = "Invalid attribute applied: {}".format(attribute)
raise ValueError(etxt)
if not attvalues.flags["C_CONTIGUOUS"]:
mask = ma.getmaskarray(attvalues)
mask = np.asanyarray(mask, order="C")
attvalues = np.asanyarray(attvalues, order="C")
attvalues = ma.array(attvalues, mask=mask, order="C")
return attvalues
def destroy_with_variance(Data, sigma_thres=6, bad_freq_list=[]):
'''Mask spikes in Data using variance. Polarizations must be in
XX,XY,YX,YY format.
sigma_thres represents how sensitive the flagger is (smaller = more masking).
The flagged frequencies are appended to bad_freq_list.'''
XX_YY_0 = ma.mean(Data.data[:, 0, 0, :], 0) * ma.mean(Data.data[:, 3, 0, :], 0)
XX_YY_1 = ma.mean(Data.data[:, 0, 1, :], 0) * ma.mean(Data.data[:, 3, 1, :], 0)
# Get the normalized variance array for each polarization.
a = ma.var(Data.data[:, 0, 0, :], 0) / (ma.mean(Data.data[:, 0, 0, :], 0)**2) # XX
b = ma.var(Data.data[:, 1, 0, :], 0) / XX_YY_0 # XY
c = ma.var(Data.data[:, 2, 0, :], 0) / XX_YY_0 # YX
d = ma.var(Data.data[:, 3, 0, :], 0) / (ma.mean(Data.data[:, 3, 0, :], 0)**2) # YY
# And for cal off.
e = ma.var(Data.data[:, 0, 1, :], 0) / (ma.mean(Data.data[:, 0, 1, :], 0)**2) # XX
f = ma.var(Data.data[:, 1, 1, :], 0) / XX_YY_1 # XY
g = ma.var(Data.data[:, 2, 1, :], 0) / XX_YY_1 # YX
h = ma.var(Data.data[:, 3, 1, :], 0) / (ma.mean(Data.data[:, 3, 1, :], 0)**2) # YY
# Get the mean and standard deviation [sigma].
means = sp.array([ma.mean(a), ma.mean(b), ma.mean(c), ma.mean(d),
ma.mean(e), ma.mean(f), ma.mean(g), ma.mean(h)])
sig = sp.array([ma.std(a), ma.std(b), ma.std(c), ma.std(d),
ma.std(e), ma.std(f), ma.std(g), ma.std(h)])
# Get the max accepted value [sigma_thres*sigma, sigma_thres=6 works really well].
max_sig = sigma_thres*sig
max_accepted = means + max_sig
amount_masked = 0
for freq in range(0, len(a)):
if ((a[freq] > max_accepted[0]) or
(b[freq] > max_accepted[1]) or
(c[freq] > max_accepted[2]) or
(d[freq] > max_accepted[3]) or
(e[freq] > max_accepted[4]) or
(f[freq] > max_accepted[5]) or
(g[freq] > max_accepted[6]) or
(h[freq] > max_accepted[7])):
# mask
amount_masked += 1
bad_freq_list.append(freq)
Data.data[:,:,:,freq].mask = True
return amount_masked
def aF_oneway(*args, **kwargs):
dim = kwargs.get("dim", None)
arrays = args
means = [ma.mean(a, dim) for a in arrays]
vars = [ma.var(a, dim) for a in arrays]
lens = [ma.sum(ma.array(ma.ones(a.shape), mask=ma.asarray(a).mask), dim) for a in arrays]
alldata = ma.concatenate(arrays, dim if dim is not None else 0)
bign = ma.sum(ma.array(ma.ones(alldata.shape), mask=alldata.mask), dim)
sstot = ma.sum(alldata ** 2, dim) - (ma.sum(alldata, dim) ** 2) / bign
ssbn = ma.sum([(ma.sum(a, dim) ** 2) / L for a, L in zip(arrays, lens)], dim)
# print ma.sum(alldata, dim) ** 2 / bign, ssbn
ssbn -= ma.sum(alldata, dim) ** 2 / bign
sswn = sstot - ssbn
dfbn = dfnum = float(len(args) - 1.0)
dfwn = bign - len(args) # + 1.0
F = (ssbn / dfbn) / (sswn / dfwn)
if F.ndim == 0 and dfwn.ndim == 0:
return (F,scipy.stats.betai(0.5 * dfwn, 0.5 * dfnum, dfwn/float(dfwn+dfnum*F)) if F is not ma.masked and dfwn/float(dfwn+dfnum*F) <= 1.0 \
and dfwn/float(dfwn+dfnum*F) >= 0.0 else ma.masked)
else:
prob = [scipy.stats.betai(0.5 * dfden, 0.5 * dfnum, dfden/float(dfden+dfnum*f)) if f is not ma.masked and dfden/float(dfden+dfnum*f) <= 1.0 \
and dfden/float(dfden+dfnum*f) >= 0.0 else ma.masked for dfden, f in zip (dfwn, F)]
return F, prob
def get_phidp_offset(ncfile):
# ----------------
# Open NetCDF file
# ----------------
print('Opening NetCDF file ' + ncfile)
dataset = nc4.Dataset(ncfile,'r+',format='NETCDF3_CLASSIC')
nray = len(dataset.dimensions['time']);
ngate = len(dataset.dimensions['range']);
phidp = dataset.variables['PDP'][:];
phi0arr = ma.mean(phidp,axis=1)
for iray in np.arange(nray):
pdpsum = 0
nphipixel = 0
phidp_use = phidp[iray,:].copy()
for igate in np.arange(6,ngate):
tmp_mean = ma.mean(phidp_use[igate-4:igate])
tmp_var = ma.var(phidp_use[igate-4:igate])
if tmp_var<9:
pdpsum = pdpsum+tmp_mean;
nphipixel = nphipixel+1
break
if nphipixel>0:
phidp_floor = pdpsum/nphipixel
phi0arr[iray] = phidp_floor
phi0 = ma.median(phi0arr)
dataset.close()
return np.round(phi0,2)
def testVariance(self):
self.assertAlmostEqual(ma.var(self.masked_data, ddof=1), 25669.716886585673)
def testVariance(self):
self.assertAlmostEqual(ma.var(self.masked_data, ddof=1), 206.5207728)
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster!
v = y.view(numpy.ndarray)
xm = ma.masked_outside(xt, x0, x1).T
ym = ma.array(v, mask=ma.getmask(xm))
if mode == 'mean':
r1 = ma.mean(ym)
r2 = ma.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return (0, 0)
slope = numpy.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = numpy.array(self.dat[i][j, thisaxis, tb])
ppars = numpy.polyfit(
x[tb], ym[tb],
1) # do a linear fit - smooths the slope measures
slope = numpy.append(slope, ppars[0]) # keep track of max slope
r1 = numpy.amax(slope)
r2 = numpy.argmax(slope)
return (r1, r2)
def destroy_with_variance(Data, sigma_thres=6, bad_freq_list=[]):
'''Mask frequencies with high variance.
Since the signal we are looking for is much weaker than what is in `Data`,
any frequency that is 'too spiky' is not signal and is RFI instead. Using
variance as a test really makes this 'spikyness' stand out.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT. Bad
frequencies will be flagged in all polarizations and cal states.
sigma_thres : int or float
Any frequency with variance > `sigma_thres` sigmas will be
flagged (recursively).
bad_freq_list : list of int
A list of bad frequencies. Since this method is called over and over,
this list keeps track of what has been flagged. Bad frequencies that
are found will be appended to this list.
Returns
-------
amount_masked : int
The amount of frequencies masked.
Notes
-----
Polarizations must be in XX,XY,YX,YY format.
'''
XX_YY_0 = ma.mean(Data.data[:, 0, 0, :], 0) * ma.mean(
Data.data[:, 3, 0, :], 0)
XX_YY_1 = ma.mean(Data.data[:, 0, 1, :], 0) * ma.mean(
Data.data[:, 3, 1, :], 0)
# Get the normalized variance array for each polarization.
a = ma.var(Data.data[:, 0, 0, :], 0) / (ma.mean(Data.data[:, 0, 0, :], 0)**
2) # XX
b = ma.var(Data.data[:, 1, 0, :], 0) / XX_YY_0 # XY
c = ma.var(Data.data[:, 2, 0, :], 0) / XX_YY_0 # YX
d = ma.var(Data.data[:, 3, 0, :], 0) / (ma.mean(Data.data[:, 3, 0, :], 0)**
2) # YY
# And for cal off.
e = ma.var(Data.data[:, 0, 1, :], 0) / (ma.mean(Data.data[:, 0, 1, :], 0)**
2) # XX
f = ma.var(Data.data[:, 1, 1, :], 0) / XX_YY_1 # XY
g = ma.var(Data.data[:, 2, 1, :], 0) / XX_YY_1 # YX
h = ma.var(Data.data[:, 3, 1, :], 0) / (ma.mean(Data.data[:, 3, 1, :], 0)**
2) # YY
# Get the mean and standard deviation [sigma].
means = sp.array([
ma.mean(a),
ma.mean(b),
ma.mean(c),
ma.mean(d),
ma.mean(e),
ma.mean(f),
ma.mean(g),
ma.mean(h)
])
sig = sp.array([
ma.std(a),
ma.std(b),
ma.std(c),
ma.std(d),
ma.std(e),
ma.std(f),
ma.std(g),
ma.std(h)
])
# Get the max accepted value [sigma_thres*sigma, sigma_thres=6 works really well].
max_sig = sigma_thres * sig
max_accepted = means + max_sig
amount_masked = 0
for freq in range(0, len(a)):
if ((a[freq] > max_accepted[0]) or (b[freq] > max_accepted[1])
or (c[freq] > max_accepted[2]) or (d[freq] > max_accepted[3])
or (e[freq] > max_accepted[4]) or (f[freq] > max_accepted[5])
or (g[freq] > max_accepted[6]) or (h[freq] > max_accepted[7])):
# mask
amount_masked += 1
bad_freq_list.append(freq)
Data.data[:, :, :, freq].mask = True
return amount_masked
def __call__(self, target):
ind1, ind2 = self.test_indices(target)
a1, a2 = self.array[ind1, :], self.array[ind2, :]
stn = (ma.mean(a1, self.dim) - ma.mean(a2, self.dim)) / (
ma.sqrt(ma.var(a1, self.dim)) + ma.sqrt(ma.var(a2, self.dim)))
return list(zip(self.keys, stn))
s = sum(a)
return s * 1.0 / len(a)
def vara(self, a, n):
mean = self.aver(a)
b = [(x - mean)**n for x in a]
return sum(b) * 1.0 / len(a)
def skew(self, a):
var = self.vara(a, 2)
var3 = self.vara(a, 3)
return var3 / var**1.5
def kurto(self, a):
var = self.vara(a, 2)
var4 = self.vara(a, 4)
return var4 / var**2 - 3
def res(self, a):
return [round(x, 6) for x in a]
if __name__ == '__main__':
s = Solution()
# norm1 = norm(loc=0, scale=4)
# l = norm1.rvs(size=1000)
l = [1, 2, 3]
print(mean(l), var(l), skew(l), kurtosis(l))
print(s.describe(l))
# print(norm1.stats(moments='mvsk'))
def destroy_with_variance(Data, sigma_thres=6, bad_freq_list=[]):
'''Mask frequencies with high variance.
Since the signal we are looking for is much weaker than what is in `Data`,
any frequency that is 'too spiky' is not signal and is RFI instead. Using
variance as a test really makes this 'spikyness' stand out.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT. Bad
frequencies will be flagged in all polarizations and cal states.
sigma_thres : int or float
Any frequency with variance > `sigma_thres` sigmas will be
flagged (recursively).
bad_freq_list : list of int
A list of bad frequencies. Since this method is called over and over,
this list keeps track of what has been flagged. Bad frequencies that
are found will be appended to this list.
Returns
-------
amount_masked : int
The amount of frequencies masked.
Notes
-----
Polarizations must be in XX,XY,YX,YY format.
'''
XX_YY_0 = ma.mean(Data.data[:, 0, 0, :], 0) * ma.mean(Data.data[:, 3, 0, :], 0)
XX_YY_1 = ma.mean(Data.data[:, 0, 1, :], 0) * ma.mean(Data.data[:, 3, 1, :], 0)
# Get the normalized variance array for each polarization.
a = ma.var(Data.data[:, 0, 0, :], 0) / (ma.mean(Data.data[:, 0, 0, :], 0)**2) # XX
b = ma.var(Data.data[:, 1, 0, :], 0) / XX_YY_0 # XY
c = ma.var(Data.data[:, 2, 0, :], 0) / XX_YY_0 # YX
d = ma.var(Data.data[:, 3, 0, :], 0) / (ma.mean(Data.data[:, 3, 0, :], 0)**2) # YY
# And for cal off.
e = ma.var(Data.data[:, 0, 1, :], 0) / (ma.mean(Data.data[:, 0, 1, :], 0)**2) # XX
f = ma.var(Data.data[:, 1, 1, :], 0) / XX_YY_1 # XY
g = ma.var(Data.data[:, 2, 1, :], 0) / XX_YY_1 # YX
h = ma.var(Data.data[:, 3, 1, :], 0) / (ma.mean(Data.data[:, 3, 1, :], 0)**2) # YY
# Get the mean and standard deviation [sigma].
means = sp.array([ma.mean(a), ma.mean(b), ma.mean(c), ma.mean(d),
ma.mean(e), ma.mean(f), ma.mean(g), ma.mean(h)])
sig = sp.array([ma.std(a), ma.std(b), ma.std(c), ma.std(d),
ma.std(e), ma.std(f), ma.std(g), ma.std(h)])
# Get the max accepted value [sigma_thres*sigma, sigma_thres=6 works really well].
max_sig = sigma_thres*sig
max_accepted = means + max_sig
amount_masked = 0
for freq in range(0, len(a)):
if ((a[freq] > max_accepted[0]) or
(b[freq] > max_accepted[1]) or
(c[freq] > max_accepted[2]) or
(d[freq] > max_accepted[3]) or
(e[freq] > max_accepted[4]) or
(f[freq] > max_accepted[5]) or
(g[freq] > max_accepted[6]) or
(h[freq] > max_accepted[7])):
# mask
amount_masked += 1
bad_freq_list.append(freq)
Data.data[:,:,:,freq].mask = True
return amount_masked
┃ 神兽保佑 ┣┓
┃ 永无BUG！ ┏┛
┗┓┓┏━┳┓┏┛
┃┫┫ ┃┫┫
┗┻┛ ┗┻┛
"""
from random import uniform
from numpy.ma import mean, arctan, sin, var
N = 10000
f = lambda x: arctan(x) / (x**2 + x * sin(x)) # 要求积分的函数
a, b = 0, 1 # 积分区间
xs = [uniform(a, b) for _ in range(N)] # 从均匀分布uniform(a,answers)生成N个样本
mean = mean([f(x) for x in xs]) # 代入积分函数，用均值去近似期望，因为函数不收敛，所以这个值也不确定
print(mean)
print(var([f(x) for x in xs])) # 由于函数不收敛，方差巨大
def para():
import numpy as np
import scipy as sp
N = 10000000
f = lambda x: arctan(x) / (x**2 + x * sin(x)) # 要求积分的函数
f = sp.vectorize(f)
xs = np.array([random() for _ in range(N)]) # 生成N个积分区间（0，1）的数据
fs = f(xs)
mean = fs.mean()
print(mean)
var = fs.var()
print(var)
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1) # .compressed()
ym = ma.array(y, mask=ma.getmask(xm)) # .compressed()
if mode == 'mean':
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'minormax':
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == '1090': #measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1 * r1
y90 = 0.9 * r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return (0, 0)
slope = np.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = np.array(self.dat[i][j, thisaxis, tb])
ppars = np.polyfit(
x[tb], ym[tb],
1) # do a linear fit - smooths the slope measures
slope = np.append(slope, ppars[0]) # keep track of max slope
r1 = np.amax(slope)
r2 = np.argmax(slope)
return (r1, r2)
s = sum(a)
return s * 1.0 / len(a)
def vara(self, a, n):
mean = self.aver(a)
b = [(x - mean) ** n for x in a]
return sum(b) * 1.0 / len(a)
def skew(self, a):
var = self.vara(a, 2)
var3 = self.vara(a, 3)
return var3 / var ** 1.5
def kurto(self, a):
var = self.vara(a, 2)
var4 = self.vara(a, 4)
return var4 / var ** 2 - 3
def res(self, a):
return [round(x, 6) for x in a]
if __name__ == '__main__':
s = Solution()
# norm1 = norm(loc=0, scale=4)
# l = norm1.rvs(size=1000)
l = [1, 2, 3]
print(mean(l), var(l), skew(l), kurtosis(l))
print(s.describe(l))
# print(norm1.stats(moments='mvsk'))
from datetime import datetime
from data_rain import RainData
import re
response = requests.get(
'https://or.water.usgs.gov/non-usgs/bes/hayden_island.rain')
text = response.text
data = re.findall(r'(\d+-\w+-\d+)\s+(\d+)', text)
days_of_rain = []
total_tips = 0
for day in data:
total_tips += int(day[1])
date = datetime.strptime(day[0], '%d-%b-%Y')
days_of_rain.append(RainData(date, int(day[1])))
most_rain = days_of_rain[0]
for day in days_of_rain:
if day.tips > most_rain.tips:
most_rain = day
variance = ma.var([day.tips for day in days_of_rain])
mean = total_tips / len(days_of_rain)
print(
f"Start: {days_of_rain[0].date} End: {days_of_rain[-1].date} Variance: {variance} Mean: {mean} Day with most rain: {most_rain.date} with {most_rain.inches} inches"
)
def measure(mode, x, y, x0, x1, thresh = 0):
""" return the a measure of y in the window x0 to x1
"""
xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster!
v = y.view(numpy.ndarray)
xm = ma.masked_outside(xt, x0, x1).T
ym = ma.array(v, mask = ma.getmask(xm))
if mode == 'mean':
r1 = ma.mean(ym)
r2 = ma.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
return(0,0)
slope = numpy.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win)/20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k-st, k+st) # get tb array
newa = numpy.array(self.dat[i][j, thisaxis, tb])
ppars = numpy.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope = numpy.append(slope, ppars[0]) # keep track of max slope
r1 = numpy.amax(slope)
r2 = numpy.argmax(slope)
return(r1, r2)
def measure(mode, x, y, x0, x1, thresh=0, slopewin=1.0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1)# .compressed()
ym = ma.array(y, mask = ma.getmask(xm))# .compressed()
if mode == 'mean':
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == 'max' or mode == 'maximum':
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == 'min' or mode == 'minimum':
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == 'minormax':
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == 'median':
r1 = ma.median(ym)
r2 = 0
if mode == 'p2p': # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == 'std': # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == 'var': # variance
r1 = ma.var(ym)
r2 = 0
if mode == 'cumsum': # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == 'anom': # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == 'sum':
r1 = ma.sum(ym)
r2 = 0
if mode == 'area' or mode == 'charge':
r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm))
r2 = 0
if mode == 'latency': # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == '1090': #measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1*r1
y90 = 0.9*r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == 'count':
r1 = ma.count(ym)
r2 = 0
if mode == 'maxslope':
slope = []
win = ma.flatnotmasked_contiguous(ym)
dt = x[1]-x[0]
st = int(slopewin/dt) # use slopewin duration window for fit.
print('st: ', st)
for k, w in enumerate(win): # move through the slope measurementwindow
tb = range(k-st, k+st) # get tb array
ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope.append(ppars[0]) # keep track of max slope
r1 = np.max(slope)
r2 = np.argmax(slope)
return(r1, r2)
def _calc_var_from_top_gates(data: np.ndarray) -> np.ndarray:
fraction = 0.1
n_gates = round(data.shape[1] * fraction)
return ma.var(data[:, -n_gates:], axis=1)
def measure(mode, x, y, x0, x1, thresh=0):
""" return the a measure of y in the window x0 to x1
"""
xm = ma.masked_outside(x, x0, x1) # .compressed()
ym = ma.array(y, mask=ma.getmask(xm)) # .compressed()
if mode == "mean":
r1 = np.mean(ym)
r2 = np.std(ym)
if mode == "max" or mode == "maximum":
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
if mode == "min" or mode == "minimum":
r1 = ma.min(ym)
r2 = xm[ma.argmin(ym)]
if mode == "minormax":
r1p = ma.max(ym)
r1n = ma.min(ym)
if ma.abs(r1p) > ma.abs(r1n):
r1 = r1p
r2 = xm[ma.argmax(ym)]
else:
r1 = r1n
r2 = xm[ma.argmin(ym)]
if mode == "median":
r1 = ma.median(ym)
r2 = 0
if mode == "p2p": # peak to peak
r1 = ma.ptp(ym)
r2 = 0
if mode == "std": # standard deviation
r1 = ma.std(ym)
r2 = 0
if mode == "var": # variance
r1 = ma.var(ym)
r2 = 0
if mode == "cumsum": # cumulative sum
r1 = ma.cumsum(ym) # Note: returns an array
r2 = 0
if mode == "anom": # anomalies = difference from averge
r1 = ma.anom(ym) # returns an array
r2 = 0
if mode == "sum":
r1 = ma.sum(ym)
r2 = 0
if mode == "area" or mode == "charge":
r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm))
r2 = 0
if mode == "latency": # return first point that is > threshold
sm = ma.nonzero(ym > thresh)
r1 = -1 # use this to indicate no event detected
r2 = 0
if ma.count(sm) > 0:
r1 = sm[0][0]
r2 = len(sm[0])
if mode == "1090": # measure 10-90% time, also returns max
r1 = ma.max(ym)
r2 = xm[ma.argmax(ym)]
y10 = 0.1 * r1
y90 = 0.9 * r1
sm1 = ma.nonzero(ym >= y10)
sm9 = ma.nonzero(ym >= y90)
r1 = xm[sm9] - xm[sm1]
if mode == "count":
r1 = ma.count(ym)
r2 = 0
if mode == "maxslope":
return (0, 0)
slope = np.array([])
win = ma.flatnotmasked_contiguous(ym)
st = int(len(win) / 20) # look over small ranges
for k in win: # move through the slope measurementwindow
tb = range(k - st, k + st) # get tb array
newa = np.array(self.dat[i][j, thisaxis, tb])
ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures
slope = np.append(slope, ppars[0]) # keep track of max slope
r1 = np.amax(slope)
r2 = np.argmax(slope)
return (r1, r2)
def testVariance(self):
self.assertAlmostEqual(ma.var(self.masked_data, ddof=1),
25669.716886585673)
def __call__(self, target):
ind1, ind2 = self.test_indices(target)
a1, a2 = self.array[ind1, :], self.array[ind2, :]
stn = (ma.mean(a1, self.dim) - ma.mean(a2, self.dim)) / (ma.sqrt(ma.var(a1, self.dim)) + ma.sqrt(ma.var(a2, self.dim)))
return list(zip(self.keys, stn))