def probablistic_signal_template_extraction(beacon_signal, non_beacon_signal):
"""
Characterize the noise floor, and use its probability distribution to decide
which samples of the beacon signal most likely are non-pauses.
Parameters
----------
beacon_signal : complex
Beacon signal (as obtained from an FFT bin)
non_beacon_signal : complex
Non-beacon signal (as obtained from an FFT bin sufficiently far away in
frequency from the beacon signal, but not so far that the noise floor
no longer has the same distribution)
Returns
-------
high_values : boolean
Signal template, with 1 corresponding to high values, 0 to low values
"""
#run test for normality on segments of the noise floor
window_length = 5000
window_pos = np.arange(window_length, len(non_beacon_signal) - window_length)
normaltest_pvalues = np.zeros(len(window_pos))
for i, start_samp in enumerate(window_pos):
subset = non_beacon_signal[start_samp-window_length/2:start_samp+window_length/2]
_, normaltest_pvalues[i] = normaltest(np.real(subset))
#select samples that within a stretch of normal distributed samples, with critical value of 0.3, use this to characterize the noise floor distribution
window_pos_normal = window_pos[normaltest > 0.3]
normal_samples = non_beacon_signal[window_pos_normal]
mean_noisefloor = np.mean(np.real(normal_samples))
std_noisefloor = np.std(np.real(normal_samples))
#get folded normal distribution (distribution of signal magnitudes of the noise)
c = 0
loc = 0
S = std_noisefloor
distrib = foldnorm(c=0, scale=S, loc=loc)
#calculate probability that the noise floor probability density produces a
#more extreme value than each observed sample
noise_floor_prob = 1-distrib.cdf(np.abs(beacon_signal))
#signal threshold (implicit threshold: probabilities lower than the
#granularity of the datatype). Might have to be adjusted for more low-level
#signals.
high_values = noise_floor_prob == 0.0
high_values = binary_erosion(high_values)
return high_values
def __init__(self):
self.dist_equivalents = [
#transf, stats.lognorm(1))
(lognormalg, stats.lognorm(1)),
#transf2
(squarenormalg, stats.chi2(1)),
(absnormalg, stats.halfnorm),
(absnormalg, stats.foldnorm(1e-5)), #try frozen
#(negsquarenormalg, 1-stats.chi2), # won't work as distribution
(squaretg(10), stats.f(1, 10))] #try both frozen
l,s = 0.0, 1.0
self.ppfq = [0.1,0.5,0.9]
self.xx = [0.95,1.0,1.1]
self.nxx = [-0.95,-1.0,-1.1]
def __init__(self):
self.dist_equivalents = [
#transf, stats.lognorm(1))
(lognormalg, stats.lognorm(1)),
#transf2
(squarenormalg, stats.chi2(1)),
(absnormalg, stats.halfnorm),
(absnormalg, stats.foldnorm(1e-5)), #try frozen
#(negsquarenormalg, 1-stats.chi2), # won't work as distribution
(squaretg(10), stats.f(1, 10))] #try both frozen
l,s = 0.0, 1.0
self.ppfq = [0.1,0.5,0.9]
self.xx = [0.95,1.0,1.1]
self.nxx = [-0.95,-1.0,-1.1]
def setup_class(cls):
cls.dist_equivalents = [
#transf, stats.lognorm(1))
#The below fails on the SPARC box with scipy 10.1
#(lognormalg, stats.lognorm(1)),
#transf2
(squarenormalg, stats.chi2(1)),
(absnormalg, stats.halfnorm),
(absnormalg, stats.foldnorm(1e-5)), #try frozen
#(negsquarenormalg, 1-stats.chi2), # won't work as distribution
(squaretg(10), stats.f(1, 10))
] #try both frozen
l, s = 0.0, 1.0
cls.ppfq = [0.1, 0.5, 0.9]
cls.xx = [0.95, 1.0, 1.1]
cls.nxx = [-0.95, -1.0, -1.1]
def setup_class(cls):
cls.dist_equivalents = [
#transf, stats.lognorm(1))
#The below fails on the SPARC box with scipy 10.1
#(lognormalg, stats.lognorm(1)),
#transf2
(squarenormalg, stats.chi2(1)),
(absnormalg, stats.halfnorm),
(absnormalg, stats.foldnorm(1e-5)), #try frozen
#(negsquarenormalg, 1-stats.chi2), # won't work as distribution
(squaretg(10), stats.f(1, 10))
] #try both frozen
l,s = 0.0, 1.0
cls.ppfq = [0.1,0.5,0.9]
cls.xx = [0.95,1.0,1.1]
cls.nxx = [-0.95,-1.0,-1.1]
def __init__(self):
self.dist_equivalents = [
#transf, stats.lognorm(1))
#The below fails on the SPARC box with scipy 10.1
#(lognormalg, stats.lognorm(1)),
#transf2
(squarenormalg, stats.chi2(1)),
(absnormalg, stats.halfnorm),
(absnormalg, stats.foldnorm(1e-5)), #try frozen
#(negsquarenormalg, 1-stats.chi2), # won't work as distribution
#(squaretg(10), stats.f(1, 10)) # disable temporarily see #1864
] #try both frozen
l,s = 0.0, 1.0
self.ppfq = [0.1,0.5,0.9]
self.xx = [0.95,1.0,1.1]
self.nxx = [-0.95,-1.0,-1.1]
xx, nxx, ppfq = self.xx, self.nxx, self.ppfq
d1,d2 = (negsquarenormalg, stats.chi2(1))
#print d1.name
assert_almost_equal(d1.cdf(nxx), 1-d2.cdf(xx), err_msg='cdf'+d1.name)
assert_almost_equal(d1.pdf(nxx), d2.pdf(xx))
assert_almost_equal(d1.sf(nxx), 1-d2.sf(xx))
assert_almost_equal(d1.ppf(ppfq), -d2.ppf(ppfq)[::-1])
assert_almost_equal(d1.isf(ppfq), -d2.isf(ppfq)[::-1])
assert_almost_equal(d1.moment(3), -d2.moment(3))
ch2oddneg = [v*(-1)**(i+1) for i,v in
enumerate(d2.stats(moments='mvsk'))]
assert_almost_equal(d1.stats(moments='mvsk'), ch2oddneg,
err_msg='stats '+d1.name+d2.name)
if __name__ == '__main__':
tt = Test_Transf2()
tt.test_equivalent()
tt.test_equivalent_negsq()
debug = 0
if debug:
print negsquarenormalg.ppf([0.1,0.5,0.9])
print stats.chi2.ppf([0.1,0.5,0.9],1)
print negsquarenormalg.a
print negsquarenormalg.b
print absnormalg.stats( moments='mvsk')
print stats.foldnorm(1e-10).stats( moments='mvsk')
print stats.halfnorm.stats( moments='mvsk')
1 - d2.cdf(xx),
err_msg='cdf' + d1.name)
assert_almost_equal(d1.pdf(nxx), d2.pdf(xx))
assert_almost_equal(d1.sf(nxx), 1 - d2.sf(xx))
assert_almost_equal(d1.ppf(ppfq), -d2.ppf(ppfq)[::-1])
assert_almost_equal(d1.isf(ppfq), -d2.isf(ppfq)[::-1])
assert_almost_equal(d1.moment(3), -d2.moment(3))
ch2oddneg = [
v * (-1)**(i + 1) for i, v in enumerate(d2.stats(moments='mvsk'))
]
assert_almost_equal(d1.stats(moments='mvsk'),
ch2oddneg,
err_msg='stats ' + d1.name + d2.name)
if __name__ == '__main__':
tt = Test_Transf2()
tt.test_equivalent()
tt.test_equivalent_negsq()
debug = 0
if debug:
print(negsquarenormalg.ppf([0.1, 0.5, 0.9]))
print(stats.chi2.ppf([0.1, 0.5, 0.9], 1))
print(negsquarenormalg.a)
print(negsquarenormalg.b)
print(absnormalg.stats(moments='mvsk'))
print(stats.foldnorm(1e-10).stats(moments='mvsk'))
print(stats.halfnorm.stats(moments='mvsk'))
def test_foldnorm_zero():
# Parameter value c=0 was not enabled, see gh-2399.
rv = stats.foldnorm(0, scale=1)
assert_equal(rv.cdf(0), 0) # rv.cdf(0) previously resulted in: nan
def test_foldnorm_zero():
# Parameter value c=0 was not enabled, see gh-2399.
rv = stats.foldnorm(0, scale=1)
assert_equal(rv.cdf(0), 0) # rv.cdf(0) previously resulted in: nan
def all_dists():
# dists param were taken from scipy.stats official
# documentaion examples
# Total - 89
return {
"alpha":
stats.alpha(a=3.57, loc=0.0, scale=1.0),
"anglit":
stats.anglit(loc=0.0, scale=1.0),
"arcsine":
stats.arcsine(loc=0.0, scale=1.0),
"beta":
stats.beta(a=2.31, b=0.627, loc=0.0, scale=1.0),
"betaprime":
stats.betaprime(a=5, b=6, loc=0.0, scale=1.0),
"bradford":
stats.bradford(c=0.299, loc=0.0, scale=1.0),
"burr":
stats.burr(c=10.5, d=4.3, loc=0.0, scale=1.0),
"cauchy":
stats.cauchy(loc=0.0, scale=1.0),
"chi":
stats.chi(df=78, loc=0.0, scale=1.0),
"chi2":
stats.chi2(df=55, loc=0.0, scale=1.0),
"cosine":
stats.cosine(loc=0.0, scale=1.0),
"dgamma":
stats.dgamma(a=1.1, loc=0.0, scale=1.0),
"dweibull":
stats.dweibull(c=2.07, loc=0.0, scale=1.0),
"erlang":
stats.erlang(a=2, loc=0.0, scale=1.0),
"expon":
stats.expon(loc=0.0, scale=1.0),
"exponnorm":
stats.exponnorm(K=1.5, loc=0.0, scale=1.0),
"exponweib":
stats.exponweib(a=2.89, c=1.95, loc=0.0, scale=1.0),
"exponpow":
stats.exponpow(b=2.7, loc=0.0, scale=1.0),
"f":
stats.f(dfn=29, dfd=18, loc=0.0, scale=1.0),
"fatiguelife":
stats.fatiguelife(c=29, loc=0.0, scale=1.0),
"fisk":
stats.fisk(c=3.09, loc=0.0, scale=1.0),
"foldcauchy":
stats.foldcauchy(c=4.72, loc=0.0, scale=1.0),
"foldnorm":
stats.foldnorm(c=1.95, loc=0.0, scale=1.0),
# "frechet_r": stats.frechet_r(c=1.89, loc=0.0, scale=1.0),
# "frechet_l": stats.frechet_l(c=3.63, loc=0.0, scale=1.0),
"genlogistic":
stats.genlogistic(c=0.412, loc=0.0, scale=1.0),
"genpareto":
stats.genpareto(c=0.1, loc=0.0, scale=1.0),
"gennorm":
stats.gennorm(beta=1.3, loc=0.0, scale=1.0),
"genexpon":
stats.genexpon(a=9.13, b=16.2, c=3.28, loc=0.0, scale=1.0),
"genextreme":
stats.genextreme(c=-0.1, loc=0.0, scale=1.0),
"gausshyper":
stats.gausshyper(a=13.8, b=3.12, c=2.51, z=5.18, loc=0.0, scale=1.0),
"gamma":
stats.gamma(a=1.99, loc=0.0, scale=1.0),
"gengamma":
stats.gengamma(a=4.42, c=-3.12, loc=0.0, scale=1.0),
"genhalflogistic":
stats.genhalflogistic(c=0.773, loc=0.0, scale=1.0),
"gilbrat":
stats.gilbrat(loc=0.0, scale=1.0),
"gompertz":
stats.gompertz(c=0.947, loc=0.0, scale=1.0),
"gumbel_r":
stats.gumbel_r(loc=0.0, scale=1.0),
"gumbel_l":
stats.gumbel_l(loc=0.0, scale=1.0),
"halfcauchy":
stats.halfcauchy(loc=0.0, scale=1.0),
"halflogistic":
stats.halflogistic(loc=0.0, scale=1.0),
"halfnorm":
stats.halfnorm(loc=0.0, scale=1.0),
"halfgennorm":
stats.halfgennorm(beta=0.675, loc=0.0, scale=1.0),
"hypsecant":
stats.hypsecant(loc=0.0, scale=1.0),
"invgamma":
stats.invgamma(a=4.07, loc=0.0, scale=1.0),
"invgauss":
stats.invgauss(mu=0.145, loc=0.0, scale=1.0),
"invweibull":
stats.invweibull(c=10.6, loc=0.0, scale=1.0),
"johnsonsb":
stats.johnsonsb(a=4.32, b=3.18, loc=0.0, scale=1.0),
"johnsonsu":
stats.johnsonsu(a=2.55, b=2.25, loc=0.0, scale=1.0),
"ksone":
stats.ksone(n=1e03, loc=0.0, scale=1.0),
"kstwobign":
stats.kstwobign(loc=0.0, scale=1.0),
"laplace":
stats.laplace(loc=0.0, scale=1.0),
"levy":
stats.levy(loc=0.0, scale=1.0),
"levy_l":
stats.levy_l(loc=0.0, scale=1.0),
"levy_stable":
stats.levy_stable(alpha=0.357, beta=-0.675, loc=0.0, scale=1.0),
"logistic":
stats.logistic(loc=0.0, scale=1.0),
"loggamma":
stats.loggamma(c=0.414, loc=0.0, scale=1.0),
"loglaplace":
stats.loglaplace(c=3.25, loc=0.0, scale=1.0),
"lognorm":
stats.lognorm(s=0.954, loc=0.0, scale=1.0),
"lomax":
stats.lomax(c=1.88, loc=0.0, scale=1.0),
"maxwell":
stats.maxwell(loc=0.0, scale=1.0),
"mielke":
stats.mielke(k=10.4, s=3.6, loc=0.0, scale=1.0),
"nakagami":
stats.nakagami(nu=4.97, loc=0.0, scale=1.0),
"ncx2":
stats.ncx2(df=21, nc=1.06, loc=0.0, scale=1.0),
"ncf":
stats.ncf(dfn=27, dfd=27, nc=0.416, loc=0.0, scale=1.0),
"nct":
stats.nct(df=14, nc=0.24, loc=0.0, scale=1.0),
"norm":
stats.norm(loc=0.0, scale=1.0),
"pareto":
stats.pareto(b=2.62, loc=0.0, scale=1.0),
"pearson3":
stats.pearson3(skew=0.1, loc=0.0, scale=1.0),
"powerlaw":
stats.powerlaw(a=1.66, loc=0.0, scale=1.0),
"powerlognorm":
stats.powerlognorm(c=2.14, s=0.446, loc=0.0, scale=1.0),
"powernorm":
stats.powernorm(c=4.45, loc=0.0, scale=1.0),
"rdist":
stats.rdist(c=0.9, loc=0.0, scale=1.0),
"reciprocal":
stats.reciprocal(a=0.00623, b=1.01, loc=0.0, scale=1.0),
"rayleigh":
stats.rayleigh(loc=0.0, scale=1.0),
"rice":
stats.rice(b=0.775, loc=0.0, scale=1.0),
"recipinvgauss":
stats.recipinvgauss(mu=0.63, loc=0.0, scale=1.0),
"semicircular":
stats.semicircular(loc=0.0, scale=1.0),
"t":
stats.t(df=2.74, loc=0.0, scale=1.0),
"triang":
stats.triang(c=0.158, loc=0.0, scale=1.0),
"truncexpon":
stats.truncexpon(b=4.69, loc=0.0, scale=1.0),
"truncnorm":
stats.truncnorm(a=0.1, b=2, loc=0.0, scale=1.0),
"tukeylambda":
stats.tukeylambda(lam=3.13, loc=0.0, scale=1.0),
"uniform":
stats.uniform(loc=0.0, scale=1.0),
"vonmises":
stats.vonmises(kappa=3.99, loc=0.0, scale=1.0),
"vonmises_line":
stats.vonmises_line(kappa=3.99, loc=0.0, scale=1.0),
"wald":
stats.wald(loc=0.0, scale=1.0),
"weibull_min":
stats.weibull_min(c=1.79, loc=0.0, scale=1.0),
"weibull_max":
stats.weibull_max(c=2.87, loc=0.0, scale=1.0),
"wrapcauchy":
stats.wrapcauchy(c=0.0311, loc=0.0, scale=1.0),
}
mean, var, skew, kurt = foldnorm.stats(c, moments='mvsk')
# Display the probability density function (``pdf``):
x = np.linspace(foldnorm.ppf(0.01, c),
foldnorm.ppf(0.99, c), 100)
ax.plot(x, foldnorm.pdf(x, c),
'r-', lw=5, alpha=0.6, label='foldnorm pdf')
# Alternatively, the distribution object can be called (as a function)
# to fix the shape, location and scale parameters. This returns a "frozen"
# RV object holding the given parameters fixed.
# Freeze the distribution and display the frozen ``pdf``:
rv = foldnorm(c)
ax.plot(x, rv.pdf(x), 'k-', lw=2, label='frozen pdf')
# Check accuracy of ``cdf`` and ``ppf``:
vals = foldnorm.ppf([0.001, 0.5, 0.999], c)
np.allclose([0.001, 0.5, 0.999], foldnorm.cdf(vals, c))
# True
# Generate random numbers:
r = foldnorm.rvs(c, size=1000)
# And compare the histogram:
ax.hist(r, normed=True, histtype='stepfilled', alpha=0.2)
