def test_sg_coeffs_exact():
polyorder = 4
window_length = 9
halflen = window_length // 2
x = np.linspace(0, 21, 43)
delta = x[1] - x[0]
# The data is a cubic polynomial. We'll use an order 4
# SG filter, so the filtered values should equal the input data
# (except within half window_length of the edges).
y = 0.5 * x**3 - x
h = savgol_coeffs(window_length, polyorder)
y0 = convolve1d(y, h)
assert_allclose(y0[halflen:-halflen], y[halflen:-halflen])
# Check the same input, but use deriv=1. dy is the exact result.
dy = 1.5 * x**2 - 1
h = savgol_coeffs(window_length, polyorder, deriv=1, delta=delta)
y1 = convolve1d(y, h)
assert_allclose(y1[halflen:-halflen], dy[halflen:-halflen])
# Check the same input, but use deriv=2. d2y is the exact result.
d2y = 3.0 * x
h = savgol_coeffs(window_length, polyorder, deriv=2, delta=delta)
y2 = convolve1d(y, h)
assert_allclose(y2[halflen:-halflen], d2y[halflen:-halflen])
def test_sg_coeffs_exact():
polyorder = 4
window_length = 9
halflen = window_length // 2
x = np.linspace(0, 21, 43)
delta = x[1] - x[0]
# The data is a cubic polynomial. We'll use an order 4
# SG filter, so the filtered values should equal the input data
# (except within half window_length of the edges).
y = 0.5 * x ** 3 - x
h = savgol_coeffs(window_length, polyorder)
y0 = convolve1d(y, h)
assert_allclose(y0[halflen:-halflen], y[halflen:-halflen])
# Check the same input, but use deriv=1. dy is the exact result.
dy = 1.5 * x ** 2 - 1
h = savgol_coeffs(window_length, polyorder, deriv=1, delta=delta)
y1 = convolve1d(y, h)
assert_allclose(y1[halflen:-halflen], dy[halflen:-halflen])
# Check the same input, but use deriv=2. d2y is the exact result.
d2y = 3.0 * x
h = savgol_coeffs(window_length, polyorder, deriv=2, delta=delta)
y2 = convolve1d(y, h)
assert_allclose(y2[halflen:-halflen], d2y[halflen:-halflen])
def test_sg_coeffs_large():
# Test that for large values of window_length and polyorder the array of
# coefficients returned is symmetric. The aim is to ensure that
# no potential numeric overflow occurs.
coeffs0 = savgol_coeffs(31, 9)
assert_array_almost_equal(coeffs0, coeffs0[::-1])
coeffs1 = savgol_coeffs(31, 9, deriv=1)
assert_array_almost_equal(coeffs1, -coeffs1[::-1])
def test_sg_coeffs_large():
# Test that for large values of window_length and polyorder the array of
# coefficients returned is symmetric. The aim is to ensure that
# no potential numeric overflow occurs.
coeffs0 = savgol_coeffs(31, 9)
assert_array_almost_equal(coeffs0, coeffs0[::-1])
coeffs1 = savgol_coeffs(31, 9, deriv=1)
assert_array_almost_equal(coeffs1, -coeffs1[::-1])
def test_sg_coeffs_deriv_gt_polyorder():
"""
If deriv > polyorder, the coefficients should be all 0.
This is a regression test for a bug where, e.g.,
savgol_coeffs(5, polyorder=1, deriv=2)
raised an error.
"""
coeffs = savgol_coeffs(5, polyorder=1, deriv=2)
assert_array_equal(coeffs, np.zeros(5))
coeffs = savgol_coeffs(7, polyorder=4, deriv=6)
assert_array_equal(coeffs, np.zeros(7))
def test_sg_coeffs_deriv_gt_polyorder():
"""
If deriv > polyorder, the coefficients should be all 0.
This is a regression test for a bug where, e.g.,
savgol_coeffs(5, polyorder=1, deriv=2)
raised an error.
"""
coeffs = savgol_coeffs(5, polyorder=1, deriv=2)
assert_array_equal(coeffs, np.zeros(5))
coeffs = savgol_coeffs(7, polyorder=4, deriv=6)
assert_array_equal(coeffs, np.zeros(7))
def test_sg_coeffs_deriv():
# The data in `x` is a sampled parabola, so using savgol_coeffs with an
# order 2 or higher polynomial should give exact results.
i = np.array([-2.0, 0.0, 2.0, 4.0, 6.0])
x = i ** 2 / 4
dx = i / 2
d2x = 0.5 * np.ones_like(i)
for pos in range(x.size):
coeffs0 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot')
assert_allclose(coeffs0.dot(x), x[pos], atol=1e-10)
coeffs1 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot', deriv=1)
assert_allclose(coeffs1.dot(x), dx[pos], atol=1e-10)
coeffs2 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot', deriv=2)
assert_allclose(coeffs2.dot(x), d2x[pos], atol=1e-10)
def test_sg_coeffs_deriv():
# The data in `x` is a sampled parabola, so using savgol_coeffs with an
# order 2 or higher polynomial should give exact results.
i = np.array([-2.0, 0.0, 2.0, 4.0, 6.0])
x = i**2 / 4
dx = i / 2
d2x = np.full_like(i, 0.5)
for pos in range(x.size):
coeffs0 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot')
assert_allclose(coeffs0.dot(x), x[pos], atol=1e-10)
coeffs1 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot', deriv=1)
assert_allclose(coeffs1.dot(x), dx[pos], atol=1e-10)
coeffs2 = savgol_coeffs(5, 3, pos=pos, delta=2.0, use='dot', deriv=2)
assert_allclose(coeffs2.dot(x), d2x[pos], atol=1e-10)
def test_sg_coeffs_trivial():
# Test a trivial case of savgol_coeffs: polyorder = window_length - 1
h = savgol_coeffs(1, 0)
assert_allclose(h, [1])
h = savgol_coeffs(3, 2)
assert_allclose(h, [0, 1, 0], atol=1e-10)
h = savgol_coeffs(5, 4)
assert_allclose(h, [0, 0, 1, 0, 0], atol=1e-10)
h = savgol_coeffs(5, 4, pos=1)
assert_allclose(h, [0, 0, 0, 1, 0], atol=1e-10)
h = savgol_coeffs(5, 4, pos=1, use='dot')
assert_allclose(h, [0, 1, 0, 0, 0], atol=1e-10)
def sgolay(order: int, flamelen: int) -> Tuple:
"""
Parameters
----------
order : int
The order of the polynomial used to fit the samples. polyorder must be
less than flamelen.
flamelen : int
The length of the filter window (i.e. the number of coefficients).
framelen must be an odd positive integer.
Returns
-------
system :a tuple of array_like describing the system.
The following gives the number of elements in the tuple and
the interpretation:
* (num, den)
"""
num = signal.savgol_coeffs(flamelen, order)
den = 1
return num, den
def savgol(T, ord=3, Fs=1.0, smoothing=True):
"""Return a Savitzky-Golay FIR filter for smoothing or residuals.
Parameters
----------
T : float
Window length (in seconds if sampling rate is given).
ord : int
Order of local polynomial.
Fs : float
Sampling rate
smoothing : bool
Filter coefficients yield local polynomial fits by default.
Alternatively, the local polynomial can be subtracted from
the signal if smoothing==False
Returns
-------
b : array
FIR filter
a : constant (1)
denominator of filter polynomial
"""
N = int(T * Fs)
N += 1 - N % 2
b = signal.savgol_coeffs(N, ord)
if not smoothing:
b *= -1
# midpoint add one?
b[N // 2] += 1
#b[0] += 1
return b, 1
def test_sg_coeffs_trivial():
# Test a trivial case of savgol_coeffs: polyorder = window_length - 1
h = savgol_coeffs(1, 0)
assert_allclose(h, [1])
h = savgol_coeffs(3, 2)
assert_allclose(h, [0, 1, 0], atol=1e-10)
h = savgol_coeffs(5, 4)
assert_allclose(h, [0, 0, 1, 0, 0], atol=1e-10)
h = savgol_coeffs(5, 4, pos=1)
assert_allclose(h, [0, 0, 0, 1, 0], atol=1e-10)
h = savgol_coeffs(5, 4, pos=1, use='dot')
assert_allclose(h, [0, 1, 0, 0, 0], atol=1e-10)
def savitzky_golay_filter(data, window, polyorder, pos_back=1, order=0, axis=-1, mode='nearest'):
"""
Výpočet Savitzky-Golay filtru - aproximace klouzavého okna (hodnoty uvnitř)
pomocí konvoluce s polynomem
Input: data .. vektor dat (np.array() "1D")
window .. časový úsek, na kterém je počítán SG filtr " (int)
polyorder .. řád polynomu, který je využit při vyhlazování dat v okně
(int)
pos_back .. je pozice od konce okna, ve níž probíhá aproximace,
posunem pozice ze středu okna přicházíme o robustnost
(int)
Output: output .. data vyhlazená pomocí S-G filtru (np.array() "1D")
"""
if pos_back > window:
raise ValueError("pozice není uvnitř okna")
# okraje mám default pomocí nearest => nakopíruje krajní body
if mode not in ["mirror", "nearest", "wrap"]:
raise ValueError("mode must be 'mirror', 'nearest' or 'wrap'")
data = np.asarray(data)
# Nastavili jsem, aby se koeficienty počítaly v posledním bodě -> pos = window_lenght-1
coeffs = savgol_coeffs(window, polyorder, pos=window - pos_back, deriv=order)
# dále používám stejnou konvoluci jako je v originále
output = convolve1d(data, coeffs, axis=axis, mode=mode, cval=0.0)
return output
def __init__(self, width=5, order=1):
super().__init__()
self.width = width
self.order = order
kernel = savgol_coeffs(width, order, deriv=order, delta=1.0).astype(np.float32)
self.kernel = nn.Parameter(
torch.from_numpy((-1)**(order % 2)*kernel), requires_grad=False
)
def savgol(x,
total_width=None,
weights=None,
window_width=7,
order=3,
n_iter=1):
"""Savitzky-Golay smoothing.
Fitted polynomial order is typically much less than half the window width.
`total_width` overrides `n_iter`.
"""
if len(x) < 2:
return x
# If the effective (total) window width is not specified explicitly, compute it.
if total_width is None:
total_width = n_iter * window_width
# Pad the signal.
if weights is None:
x, total_wing, signal = check_inputs(x, total_width, False)
else:
x, total_wing, signal, weights = check_inputs(x, total_width, False,
weights)
# If the signal is short, the effective window length originally requested may not be possible. Because of this, we
# recalculate it given the actual wing length obtained.
total_width = 2 * total_wing + 1
# In case the signal is *very* short, the smoothing parameters will have to be adjusted as well.
window_width = min(window_width, total_width)
order = min(order, window_width // 2)
# Given the adjusted window widths (one-iteration and total), calculate the number of iterations we can do.
n_iter = max(1, min(1000, total_width // window_width))
# Apply signal smoothing.
logging.debug(
'Smoothing in {} iterations with window width {} and order {} for effective bandwidth {}'
.format(n_iter, window_width, order, total_width))
if weights is None:
y = signal
for _i in range(n_iter):
y = savgol_filter(y, window_width, order, mode='interp')
# y = convolve_unweighted(window, signal, wing)
else:
# TODO fit edges here, too
window = savgol_coeffs(window_width, order)
y, w = convolve_weighted(window, signal, weights, n_iter)
# Safety
bad_idx = (y > x.max()) | (y < x.min())
if bad_idx.any():
logging.warning("Smoothing overshot at {} / {} indices: "
"({}, {}) vs. original ({}, {})".format(
bad_idx.sum(), len(bad_idx), y.min(), y.max(),
x.min(), x.max()))
return y[total_wing:-total_wing]
def apply_sg_scan(y, window_size, order, deriv=0):
out = np.zeros_like(y)
c = sig.savgol_coeffs(window_size, order, deriv=0)
# for s in range(y.shape[-1]):
# for i in range(y.shape[1]):
# print c.shape
out = nd.convolve1d(y, c, 0, mode='nearest')
#out, s] = c.dot(y[:, i, 1, s])
return out
def apply_sg_scan(y, window_size, order, deriv=0):
out = np.zeros_like(y)
c = sig.savgol_coeffs(window_size, order, deriv=0)
# for s in range(y.shape[-1]):
# for i in range(y.shape[1]):
# print c.shape
out = nd.convolve1d(y, c, 0, mode='nearest')
#out, s] = c.dot(y[:, i, 1, s])
return out
def compare_coeffs_to_alt(window_length, order):
# For the given window_length and order, compare the results
# of savgol_coeffs and alt_sg_coeffs for pos from 0 to window_length - 1.
# Also include pos=None.
for pos in [None] + list(range(window_length)):
h1 = savgol_coeffs(window_length, order, pos=pos, use='dot')
h2 = alt_sg_coeffs(window_length, order, pos=pos)
assert_allclose(h1, h2, atol=1e-10,
err_msg=("window_length = %d, order = %d, pos = %s" %
(window_length, order, pos)))
def compare_coeffs_to_alt(window_length, order):
# For the given window_length and order, compare the results
# of savgol_coeffs and alt_sg_coeffs for pos from 0 to window_length - 1.
# Also include pos=None.
for pos in [None] + list(range(window_length)):
h1 = savgol_coeffs(window_length, order, pos=pos, use='dot')
h2 = alt_sg_coeffs(window_length, order, pos=pos)
assert_allclose(h1, h2, atol=1e-10,
err_msg=("window_length = %d, order = %d, pos = %s" %
(window_length, order, pos)))
def errorWeightedSmoothing(myData, myError, myWidth, myOrder):
myWeights = savgol_coeffs(myWidth, myOrder, 0)
mySignal = convolve(myData / myError**2, myWeights, mode='same')
mySignal = mySignal / convolve(1.0 / myError**2, myWeights, mode='same')
myErrors = (1.0 / convolve(1.0 / myError**2, myWeights,
mode='same'))**0.5 / sum(myWeights)**0.5
return mySignal, myErrors
def __init__(self, resolutionX, resolutionY, screenWidth, screenHeight, samplerate, window, threshold): self.resolutionX = resolutionX self.resolutionY = resolutionY self.centerx = self.resolutionX / 2.0 self.centery = self.resolutionY / 2.0 self.screenWidth = screenWidth self.screenHeight = screenHeight self.threshold = threshold self.nFixations = 0 self.fixating = False self.samplerate = samplerate self.order = 2 self.window = window self.halfwindow = int(self.window/2) self.coeff = [savgol_coeffs(self.window, self.order, 0), savgol_coeffs(self.window, self.order, 1), savgol_coeffs(self.window, self.order, 2)] self.time = np.zeros(self.window, dtype = float) self.winax = np.zeros(self.window, dtype = float) self.winay = np.zeros(self.window, dtype = float) self.winx = np.zeros(self.window, dtype = float) self.winy = np.zeros(self.window, dtype = float)
def __init__(self, resolutionX, resolutionY, screenWidth, screenHeight, samplerate, window, threshold): self.resolutionX = resolutionX self.resolutionY = resolutionY self.centerx = self.resolutionX / 2.0 self.centery = self.resolutionY / 2.0 self.screenWidth = screenWidth self.screenHeight = screenHeight self.threshold = threshold self.nFixations = 0 self.nSamples = 0 self.fixating = False self.samplerate = samplerate self.order = 2 self.window = window self.halfwindow = int(self.window/2) self.coeff = [savgol_coeffs(self.window, self.order, 0), savgol_coeffs(self.window, self.order, 1), savgol_coeffs(self.window, self.order, 2)] self.time = np.zeros(self.window, dtype = float) self.winax = np.zeros(self.window, dtype = float) self.winay = np.zeros(self.window, dtype = float) self.winx = np.zeros(self.window, dtype = float) self.winy = np.zeros(self.window, dtype = float)
def __call__(self, lst):
"filter a list. the last having the more weight"
l = len(lst)
if l % 2 == 0:
lst = numpy.array(lst[1:])
l -= 1
else:
lst = numpy.array(lst)
if len(lst) < self.order:
return lst[-1]
if l not in self.cache:
self.cache[l] = savgol_coeffs(l, self.order, pos=0)
return numpy.dot(lst, self.cache[l])
def __init__(self):
# parameters in the direct extraction
self.skypars = {'width': 10, 'order': 2}
self.coeffs = signal.savgol_coeffs(33, 4, deriv=0, delta=1.0)
self.fluxscale = 1e-17 # just a number to avoid rounding errors
# set up the models to fit
self.models = {}
#self.models['source']=Model([1.],'tabulated',table=[])
self.models['source'] = Model(np.array([0, 0, 0.7]), 'gaussian')
self.models['sky'] = Model(np.zeros(self.skypars['order'] + 1),
'polynomial')
def golay(data, diff, order, win):
import numpy as np
from scipy.signal import savgol_coeffs
from scipy.sparse import spdiags
import numpy.matlib
n = int((win - 1) / 2)
sgcoeff = savgol_coeffs(win, order, deriv=diff)[:, None]
sgcoeff = np.matlib.repmat(sgcoeff, 1, data['r'].shape[1])
diags = np.arange(-n, n + 1)
D = spdiags(sgcoeff, diags, data['r'].shape[1],
data['r'].shape[1]).toarray()
D[:, 0:n] = 0
D[:, data['r'].shape[1] - 5:data['r'].shape[1]] = 0
data['r'] = np.dot(data['r'], D)
return data
def savgol(x,
total_width=None,
weights=None,
window_width=5,
order=3,
n_iter=1):
"""Savitzky-Golay smoothing.
Fitted polynomial order is typically much less than half the window width.
`total_width` overrides `n_iter`.
"""
if len(x) < 2:
return x
if total_width:
n_iter = max(1, min(1000, total_width // window_width))
else:
total_width = n_iter * window_width
logging.debug("Smoothing in %d iterations for effective bandwidth %d",
n_iter, total_width)
# Apply signal smoothing
if weights is None:
x, total_wing, signal = check_inputs(x, total_width, False)
y = signal
for _i in range(n_iter):
y = savgol_filter(y, window_width, order, mode='interp')
# y = convolve_unweighted(window, signal, wing)
else:
# TODO fit edges here, too
x, total_wing, signal, weights = check_inputs(x, total_width, False,
weights)
window = savgol_coeffs(window_width, order)
y, w = convolve_weighted(window, signal, weights, n_iter)
# Safety
bad_idx = (y > x.max()) | (y < x.min())
if bad_idx.any():
logging.warning("Smoothing overshot at {} / {} indices: "
"({}, {}) vs. original ({}, {})".format(
bad_idx.sum(), len(bad_idx), y.min(), y.max(),
x.min(), x.max()))
return y[total_wing:-total_wing]
def savgol(x, width=None, weights=None, order=None):
"""Savitzky-Golay smoothing."""
if len(x) < 2:
return x
if width is None:
width = guess_window_size(x, weights)
x, wing, signal = check_inputs(x, width, False)
# Fitted polynomial order is typically much less than half width
if order is None:
order = int(round(np.log2(2 * wing + 1)))
# Apply signal smoothing
window = savgol_coeffs(2 * wing + 1, order)
if weights is None:
y = savgol_filter(x, 2 * wing + 1, order, mode='interp')
# y = convolve_unweighted(window, signal, wing)
else:
# TODO fit edges here, too
y, _w = convolve_weighted(window, signal, weights)
return y
def savgol(x, total_width=None, weights=None,
window_width=5, order=3, n_iter=1):
"""Savitzky-Golay smoothing.
Fitted polynomial order is typically much less than half the window width.
`total_width` overrides `n_iter`.
"""
if len(x) < 2:
return x
if total_width:
n_iter = max(1, min(1000, total_width // window_width))
else:
total_width = n_iter * window_width
logging.debug("Smoothing in %d iterations for effective bandwidth %d",
n_iter, total_width)
# Apply signal smoothing
if weights is None:
x, total_wing, signal = check_inputs(x, total_width, False)
y = signal
for _i in range(n_iter):
y = savgol_filter(y, window_width, order, mode='interp')
# y = convolve_unweighted(window, signal, wing)
else:
# TODO fit edges here, too
x, total_wing, signal, weights = check_inputs(x, total_width, False, weights)
window = savgol_coeffs(window_width, order)
y, w = convolve_weighted(window, signal, weights, n_iter)
# Safety
bad_idx = (y > x.max()) | (y < x.min())
if bad_idx.any():
logging.warning("Smoothing overshot at {} / {} indices: "
"({}, {}) vs. original ({}, {})"
.format(bad_idx.sum(), len(bad_idx),
y.min(), y.max(),
x.min(), x.max()))
return y[total_wing:-total_wing]
def noise_cube(data,
mask=None,
nThresh=30,
iterations=1,
do_map=True,
do_spec=True,
box=None,
spec_box=None,
bandpass_smooth_window=None,
bandpass_smooth_order=3,
oversample_boundary=False):
"""
Makes an empirical estimate of the noise in a cube assuming that
it is normally distributed about zero. Treats the spatial and
spectral dimensions as separable.
Parameters:
-----------
data : np.array
Array of data (floats)
Keywords:
---------
mask : np.bool
Boolean array with False indicating where data can be
used in the noise estimate. (i.e., True is signal)
do_map : np.bool
Estimate spatial variations in the noise. Default is True. If
set to False, all locations in a plane have the same noise
estimate.
do_spec : np.bool
Estimate spectral variations in the noise. Default is True. If
set to False, all channels in a spectrum have the same noise
estimate.
box : int
Spatial size of the box over which noise is calculated in
pixels. Default: no box, every pixel gets its own noise
estimte.
spec_box : int
Spectral size of the box overwhich the noise is calculated.
Default: no box, each channel gets its own noise estimate.
nThresh : int
Minimum number of data to be used in an individual noise estimate.
iterations : int
Number of times to iterate the noise solution to force Gaussian
statistics. Default: no iterations.
bandpass_smooth_window : int
Number of channels used in bandpass smoothing kernel. Defaults to
nChan / 4 where nChan number of channels. Set to zero to suppress
smoothing. Uses Savitzky-Golay smoothing
bandpass_smooth_order : int
Polynomial order used in smoothing kernel. Defaults to 3.
"""
# TBD: add error checking
# Create a mask that identifies voxels to be fitting the noise
noisemask = np.isfinite(data)
if mask is not None:
noisemask[mask] = False
# Default the spatial step size to individual pixels
step = 1
halfbox = step // 2
# If the user has supplied a spatial box size, recast this into a
# step size that critically samples the box and a halfbox size
# used for convenience.
if box is not None:
step = np.floor(box / 2.5).astype(np.int)
halfbox = int(box // 2)
# Include all pixels adjacent to the spatial
# boundary of the data as set by NaNs
boundary = np.all(np.isnan(data), axis=0)
if oversample_boundary:
struct = nd.generate_binary_structure(2, 1)
struct = nd.iterate_structure(struct, halfbox)
rind = np.logical_xor(nd.binary_dilation(boundary, struct), boundary)
extray, extrax = np.where(rind)
else:
extray, extrax = None, None
# If the user has supplied a spectral box size, use this to
# calculate a spectral step size.
if spec_box is not None:
spec_step = np.floor(spec_box / 2).astype(np.int)
boxv = int(spec_box // 2)
else:
boxv = 0
# Default the bandpass smoothing window
if bandpass_smooth_window is None:
bandpass_smooth_window = 2 * (data.shape[0] // 8) + 1
# Initialize output to be used in the case of iterative
# estimation.
noise_cube_out = np.ones_like(data)
# Iterate
for ii in np.arange(iterations):
if not do_map:
# If spatial variations are turned off then estimate a
# single value and fill the noise map with this value.
noise_value = mad_zero_centered(data, mask=noisemask)
noise_map = np.zeros(data.shape[1:]) + noise_value
else:
# Initialize map to be full of not-a-numbers
noise_map = np.zeros(data.shape[1:]) + np.nan
# Make a noise map
xx = np.arange(data.shape[2])
yy = np.arange(data.shape[1])
# Sample starting at halfbox and stepping by step. In the
# individual pixel limit this just samples every spectrum.
xsamps = xx[halfbox::step]
ysamps = yy[halfbox::step]
xsampsf = (xsamps[np.newaxis, :] *
(np.ones_like(ysamps))[:, np.newaxis]).flatten()
ysampsf = (ysamps[:, np.newaxis] * np.ones_like(xsamps)).flatten()
for x, y in zip(xsampsf, ysampsf):
# Extract a minicube and associated mask from the cube
minicube = data[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
minicube_mask = noisemask[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
# If we have enough data, fit a noise value for this entry
if np.sum(minicube_mask) > nThresh:
noise_map[y, x] = mad_zero_centered(minicube,
mask=minicube_mask)
if extrax is not None and extray is not None:
for x, y in zip(extrax, extray):
minicube = data[:, (y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
minicube_mask = noisemask[:,
(y - halfbox):(y + halfbox + 1),
(x - halfbox):(x + halfbox + 1)]
if np.sum(minicube_mask) > nThresh:
noise_map[y, x] = mad_zero_centered(minicube,
mask=minicube_mask)
noise_map[boundary] = np.nan
# If we are using a box size greater than an individual pixel
# interpolate to fill in the noise map.
if halfbox > 0:
# Note the location of data, this is the location
# where we want to fill in noise values.
data_footprint = np.any(np.isfinite(data), axis=0)
# Generate a smoothing kernel based on the box size.
kernel = Gaussian2DKernel(box / np.sqrt(8 * np.log(2)))
# Make a weight map to be used in the convolution, in
# this weight map locations with measured values have
# unity weight. This without measured values have zero
# weight.
# wt_map = np.isfinite(noise_map).astype(np.float)
# wt_map[boundary] = np.nan
# Take an average weighted by the kernel at each
# location.
# noise_map[np.isnan(noise_map)] = 0.0
# y, x = np.where(np.isfinite(noise_map))
# import scipy.interpolate as interp
# func = interp.interp2d(x, y, noise_map[y, x], kind='cubic')
noise_map = convolve(noise_map, kernel, boundary='extend')
# yy, xx = np.indices(noise_map.shape)
# noise_map_beta = interp.griddata((y, x), noise_map[y,x],
# (yy, xx), method='cubic')
# noise_map_beta = func(yy, xx)
# noise_map_beta[boundary] = np.nan
# noise_map = noise_map_beta
# wt_map = convolve(wt_map, kernel, boundary='extend')
# noise_map /= wt_map
# Set the noise map to not-a-number outside the data
# footprint.
noise_map[~data_footprint] = np.nan
# Initialize spectrum
noise_spec = np.zeros(data.shape[0]) + np.nan
if not do_spec:
# If spectral variations are turned off then assume that
# the noise_map describes all channels of the cube.
pass
else:
# Loop over channels
zz = np.arange(data.shape[0])
for z in zz:
# Idententify the range of channels to be considered
# in this estimate.
lowz = np.clip(z - boxv, 0, data.shape[0])
hiz = np.clip(z + boxv + 1, 0, data.shape[0])
# Extract a slab from the cube and normalize it by the
# noise map. Now any measured noise variations are
# relative to those in the noise map.
slab = data[lowz:hiz, :, :] / noise_map[np.newaxis, :, :]
slab_mask = noisemask[lowz:hiz, :, :]
noise_spec[z] = mad_zero_centered(slab, mask=slab_mask)
# Smooth the spectral variations in the noise.
if bandpass_smooth_window > 0:
# Initialize a Savitzky-Golay filter then run it over
# the noise spectrum.
kernel = savgol_coeffs(int(bandpass_smooth_window),
int(bandpass_smooth_order))
baddata = np.isnan(noise_spec)
noise_spec = convolve(noise_spec,
kernel,
nan_treatment='interpolate',
boundary='extend')
noise_spec[baddata] = np.nan
# Make sure that the noise spectrum is normalized by
# setting the median to one.
noise_spec /= np.nanmedian(noise_spec)
# Combine the spatial and spectral variations into a
# three-dimensional noise estimate.
noise_cube = np.ones_like(data)
noise_cube *= (noise_map[np.newaxis, :] *
noise_spec[:, np.newaxis, np.newaxis])
if iterations == 1:
return (noise_cube)
else:
# If iterating, normalize the data by the current noise
# estimate and scale the current noise cube by the new
# estimate.
data = data / noise_cube
noise_cube_out *= noise_cube
# If iterating return the iterated noise cube.
return (noise_cube_out)
def noise_cube(data,
mask=None,
box=None,
spec_box=None,
nThresh=30,
iterations=1,
bandpass_smooth_window=None,
bandpass_smooth_order=3):
"""
# Taken from PHANGS pipeline
Makes an empirical estimate of the noise in a cube assuming that it
is normally distributed.
Parameters:
-----------
data : np.array
Array of data (floats)
Keywords:
---------
mask : np.bool
Boolean array with False indicating where data can be
used in the noise estimate. (i.e., True is Signal)
box : int
Spatial size of the box over which noise is calculated (correlation
scale). Default: no box
spec_box : int
Spectral size of the box overwhich the noise is calculated. Default:
no box
nThresh : int
Minimum number of data to be used in a noise estimate.
iterations : int
Number of times to iterate the noise solution to force Gaussian
statistics. Default: no iterations.
bandpass_smooth_window : int
Number of channels used in bandpass smoothing kernel. Defaults to
nChan / 4 where nChan number of channels. Set to zero to suppress
smoothing. Uses Savitzky-Golay smoothing
bandpass_smooth_order : int
Polynomial order used in smoothing kernel. Defaults to 3.
"""
from astropy.convolution import convolve, Gaussian2DKernel
from scipy.signal import savgol_coeffs
noisemask = np.isfinite(data)
if mask is not None:
noisemask[mask] = False
step = 1
boxr = step // 2 # floor division
if box is not None:
step = np.floor(box / 2.5).astype(np.int)
boxr = int(box // 2)
if spec_box is not None:
spec_step = np.floor(spec_box / 2).astype(np.int)
boxv = int(spec_box // 2)
else:
boxv = 0
noise_cube_out = np.ones_like(data)
if bandpass_smooth_window is None:
bandpass_smooth_window = 2 * (data.shape[0] // 8) + 1
for i in np.arange(iterations):
noise_map = np.zeros(data.shape[1:]) + np.nan
noise_spec = np.zeros(data.shape[0]) + np.nan
xx = np.arange(data.shape[2])
yy = np.arange(data.shape[1])
zz = np.arange(data.shape[0])
for x in xx[boxr::step]:
for y in yy[boxr::step]:
spec = data[:, (y - boxr):(y + boxr + 1),
(x - boxr):(x + boxr + 1)]
spec_mask = noisemask[:, (y - boxr):(y + boxr + 1),
(x - boxr):(x + boxr + 1)]
if np.sum(spec_mask) > nThresh:
noise_map[y, x] = mad_zero_centered(spec, mask=spec_mask)
if boxr > 0:
data_footprint = np.any(np.isfinite(data), axis=0)
kernel = Gaussian2DKernel(box / np.sqrt(8 * np.log(2)))
wt_map = np.isfinite(noise_map).astype(np.float)
noise_map[np.isnan(noise_map)] = 0.0
noise_map = convolve(noise_map, kernel)
wt_map = convolve(wt_map, kernel)
noise_map /= wt_map
noise_map[~data_footprint] = np.nan
for z in zz:
lowz = np.clip(z - boxv, 0, data.shape[0])
hiz = np.clip(z + boxv + 1, 0, data.shape[0])
plane = data[lowz:hiz, :, :] / noise_map[np.newaxis, :, :]
plane_mask = noisemask[lowz:hiz, :, :]
noise_spec[z] = mad_zero_centered(plane, mask=plane_mask)
# Smooth spectral shape
if bandpass_smooth_window > 0:
kernel = savgol_coeffs(int(bandpass_smooth_window),
int(bandpass_smooth_order))
noise_spec = convolve(noise_spec, kernel, boundary='extend')
noise_spec /= np.nanmedian(noise_spec)
noise_cube = np.ones_like(data)
noise_cube *= (noise_map[np.newaxis, :] *
noise_spec[:, np.newaxis, np.newaxis])
if iterations == 1:
return (noise_cube)
else:
data = data / noise_cube
noise_cube_out *= noise_cube
return (noise_cube_out)
def __init__(self, n_samples, sg_order):
self.savgol_weights = savgol_coeffs(n_samples,
sg_order,
pos=n_samples - 1)
self.b, self.a = (self.savgol_weights, [1.])
self.zi = np.zeros((n_samples - 1, ))
from pylab import *
from scipy.signal import savgol_coeffs, convolve
import numpy as np
data = np.loadtxt("samples.log")
coef = savgol_coeffs(11, 2)
ret = convolve(data, coef, 'same')
plot(ret)
show()
plt.show()
###Median
k = 11
m = 0
median_smooth = np.zeros(n_1)
for i in range(k + 1, n_1 - k - 1):
m += 1
median_smooth[i] = np.median(sig_input_sensor[i - k:m + k])
plt.title("Filtro Median")
plt.plot(median_smooth, color='m', label="data smoothing")
plt.plot(sig_input_sensor, color='red', label="original sample")
plt.show()
#### Savitzky Golay
savis_smooth = signal.savgol_filter(sig_input_sensor, 9,
3) ##Señal de entrada, tamaño, exponente
signal.savgol_coeffs(11, 3)
plt.title("Filtro Savitzky-Golay")
plt.plot(savis_smooth, color='red', label="data smoothing")
plt.plot(sig_input_sensor, color='yellow', label="original sample")
plt.show()
####### Gaussian smoothing
gaus_smooth = gaussian_filter(sig_input_sensor, 2)
plt.title("Filtro Gaussian smoothing")
plt.plot(gaus_smooth, color='red', label="data smoothing")
plt.plot(sig_input_sensor, color='g', label="original sample")
plt.show()
######### Finding optiminzed parameters for S-G filtering #############
#######################################################################
SG_param = np.loadtxt(file_path_SG)
n_SG_param = len(SG_param)
SG_param_add = np.zeros(n_SG_param, dtype = float)
k1 = SG_param[:,0]
f1 = SG_param[:,2]
# 1st derivative
noise_dIdV = np.gradient(noise_IV[:,1] / dV_new)
bb = 0
for k,f in zip(k1, f1):
k = int(k)
f = int(f)
g0 = signal.savgol_coeffs(f, k, deriv=0)
g1 = signal.savgol_coeffs(f, k, deriv=1) * -1
Half_win1 = ((f + 1) / 2) - 1
Half_win1 = int(Half_win1)
sg0 = np.zeros(int(n - Half_win1 - 1), dtype = float)
sg1 = np.zeros(int(n - Half_win1 - 1), dtype = float)
for ii in range(int((f + 1) / 2), int(n - (f + 1) / 2 + 1)):
sg0[ii-1] = np.dot(g0, noise_dIdV[ii-1-Half_win1 : ii-1+Half_win1+1])
sg1[ii-1] = np.dot(g1, noise_dIdV[ii-1-Half_win1 : ii-1+Half_win1+1])
sg1 = sg1 / dV_new
sg0_new = signal.savgol_filter(noise_dIdV, f, k, deriv = 0)
sg1_new = signal.savgol_filter(noise_dIdV, f, k, deriv = 1)
sg2_new = signal.savgol_filter(noise_dIdV, f, k, deriv = 2)
sg1_new = sg1_new / dV_new
def apply_sg(y, window_size, order, deriv=0):
out = np.zeros_like(y)
coeffs = sig.savgol_coeffs(window_size, order, deriv=0, use='dot')
for i in range(y.shape[1]):
out[:, i] = coeffs.dot(y[:, i])
return out
def savgol_error(x,
sigma,
window_length,
polyorder,
deriv=0,
delta=1.0,
axis=-1,
mode='interp',
cval=0.0):
""" Apply a Savitzky-Golay filter to an array.
This is a 1-d filter. If `x` has dimension greater than 1, `axis`
determines the axis along which the filter is applied.
Parameters
----------
x : array_like
The data to be filtered. If `x` is not a single or double precision
floating point array, it will be converted to type ``numpy.float64``
before filtering.
window_length : int
The length of the filter window (i.e. the number of coefficients).
`window_length` must be a positive odd integer. If `mode` is 'interp',
`window_length` must be less than or equal to the size of `x`.
polyorder : int
The order of the polynomial used to fit the samples.
`polyorder` must be less than `window_length`.
deriv : int, optional
The order of the derivative to compute. This must be a
nonnegative integer. The default is 0, which means to filter
the data without differentiating.
delta : float, optional
The spacing of the samples to which the filter will be applied.
This is only used if deriv > 0. Default is 1.0.
axis : int, optional
The axis of the array `x` along which the filter is to be applied.
Default is -1.
mode : str, optional
Must be 'mirror', 'constant', 'nearest', 'wrap' or 'interp'. This
determines the type of extension to use for the padded signal to
which the filter is applied. When `mode` is 'constant', the padding
value is given by `cval`. See the Notes for more details on 'mirror',
'constant', 'wrap', and 'nearest'.
When the 'interp' mode is selected (the default), no extension
is used. Instead, a degree `polyorder` polynomial is fit to the
last `window_length` values of the edges, and this polynomial is
used to evaluate the last `window_length // 2` output values.
cval : scalar, optional
Value to fill past the edges of the input if `mode` is 'constant'.
Default is 0.0.
Returns
-------
y : ndarray, same shape as `x`
The filtered data.
See Also
--------
savgol_coeffs
Notes
-----
Details on the `mode` options:
'mirror':
Repeats the values at the edges in reverse order. The value
closest to the edge is not included.
'nearest':
The extension contains the nearest input value.
'constant':
The extension contains the value given by the `cval` argument.
'wrap':
The extension contains the values from the other end of the array.
For example, if the input is [1, 2, 3, 4, 5, 6, 7, 8], and
`window_length` is 7, the following shows the extended data for
the various `mode` options (assuming `cval` is 0)::
mode | Ext | Input | Ext
-----------+---------+------------------------+---------
'mirror' | 4 3 2 | 1 2 3 4 5 6 7 8 | 7 6 5
'nearest' | 1 1 1 | 1 2 3 4 5 6 7 8 | 8 8 8
'constant' | 0 0 0 | 1 2 3 4 5 6 7 8 | 0 0 0
'wrap' | 6 7 8 | 1 2 3 4 5 6 7 8 | 1 2 3
.. versionadded:: 0.14.0
Examples
--------
>>> from scipy.signal import savgol_filter
>>> np.set_printoptions(precision=2) # For compact display.
>>> x = np.array([2, 2, 5, 2, 1, 0, 1, 4, 9])
Filter with a window length of 5 and a degree 2 polynomial. Use
the defaults for all other parameters.
>>> savgol_filter(x, 5, 2)
array([1.66, 3.17, 3.54, 2.86, 0.66, 0.17, 1. , 4. , 9. ])
Note that the last five values in x are samples of a parabola, so
when mode='interp' (the default) is used with polyorder=2, the last
three values are unchanged. Compare that to, for example,
`mode='nearest'`:
>>> savgol_filter(x, 5, 2, mode='nearest')
array([1.74, 3.03, 3.54, 2.86, 0.66, 0.17, 1. , 4.6 , 7.97])
"""
if mode not in ["mirror", "constant", "nearest", "interp", "wrap"]:
raise ValueError("mode must be 'mirror', 'constant', 'nearest' "
"'wrap' or 'interp'.")
x = np.asarray(x)
# Ensure that x is either single or double precision floating point.
if x.dtype != np.float64 and x.dtype != np.float32:
x = x.astype(np.float64)
coeffs = savgol_coeffs(window_length, polyorder, deriv=deriv, delta=delta)
if mode == "interp":
if window_length > x.size:
raise ValueError("If mode is 'interp', window_length must be less "
"than or equal to the size of x.")
# Do not pad. Instead, for the elements within `window_length // 2`
# of the ends of the sequence, use the polynomial that is fitted to
# the last `window_length` elements.
y = np.sqrt(convolve1d(sigma**2, coeffs**2, axis=axis,
mode="constant"))
_fit_edges_polyfit(sigma, window_length, polyorder, deriv, delta, axis,
y)
else:
# Any mode other than 'interp' is passed on to ndimage.convolve1d.
y = convolve1d(sigma**2, coeffs**2, axis=axis, mode=mode, cval=cval)
return y
def apply_sg(y, window_size, order, deriv=0):
out = np.zeros_like(y)
coeffs = sig.savgol_coeffs(window_size, order, deriv=0, use='dot')
for i in range(y.shape[1]):
out[:, i] = coeffs.dot(y[:, i])
return out
plt.figure(figsize=(5, 4))
f = np.fft.rfftfreq(fft_length, d=1/win_length)
plt.plot(f, X)
plt.grid(True)
plt.xlabel('Frequency(Hz)')
plt.tight_layout()
plt.gcf().savefig('figs\\fourier_ex_freq.png', dpi=dpi)
## slide 15
winl = 101
pos = np.round((winl-1)/2)
t = np.arange(winl)/winl
for polyorder in [1, 3, 5, 7]:
h = sig.savgol_coeffs(window_length=winl, polyorder=polyorder, pos=pos)
plt.plot(h, label='$h, m=$' + str(polyorder))
plt.xlabel('$n$')
plt.grid(True)
plt.legend()
plt.gcf().savefig('figs\\sg_impulse', dpi=dpi)
## slide 16
fs = 160
win_length = fs*2
fft_length = win_length*2
t = np.arange(win_length)/fs
omega1 = 2*np.pi*2
omega2 = 2*np.pi*20
def __init__(self, n_samples, sg_order):
self.savgol_weights = savgol_coeffs(n_samples, sg_order, pos=n_samples - 1)
self.b, self.a = (self.savgol_weights, [1.])
self.zi = np.zeros((n_samples - 1, ))
def compare_with_savgol(fs, b, a):
from scipy.signal import savgol_coeffs
b_sg = savgol_coeffs(81, 2)
mfreqz(fs, b_sg, 1, False)
mfreqz(fs, b, a)
xMatrix = vstack([xMatrix, x])
x = 0 + xMatrix
yEstimatorMatrix = dot(x.transpose(), dot(inv(dot(x, x.transpose())), x))
return yEstimatorMatrix[int((window_length - 1) / 2) - shift]
#plot(yEstimatorMatrix[int((window_length-1)/2)],'bo')
subplot(221)
plot(
mySavgol_coeffs(polyorder=polyorder, window_length=window_length, deriv=0),
'bo')
plot(savgol_coeffs(polyorder=polyorder, window_length=window_length, deriv=0),
'g-')
subplot(222)
#plot(bandwidth_limited(window_length=window_length,bandwidth=bandwidth,nPeaks=2,shift=0))
#plot(bandwidth_limited(window_length=window_length,bandwidth=bandwidth,nPeaks=4,shift=0))
for i in arange(2, 8, 2):
y = abs(
fft(
bandwidth_limited(window_length=window_length,
bandwidth=bandwidth,
nPeaks=i,
shift=0)))[:int(window_length / 2)]
y /= y[1]
loglog(y)
