def shift(acq, data_shift=None, kern_shift=None):
"""Apply translation to acquisition data"""
return mutate(acq,
data_fn=None
if not data_shift else lambda d: scipy_shift(d, data_shift),
kern_fn=None
if not kern_shift else lambda k: scipy_shift(k, kern_shift))
def test_random_shift_forward_backward(seed, inshape, shifts, border_mode,
constant_value, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_shift(i, shifts, border_mode, constant_value, 0, seed)
result_shifts = (0, 0, 0)
max_correl = 0
for shift_amount in itertools.product(*map(
tuple,
map(lambda x: range(*x), [(-2, 3) for _ in range(len(inshape))]))):
r = scipy_shift(inputs[0],
shift_amount,
mode=border_mode,
cval=constant_value)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
result_shifts = shift_amount
max_correl = correl_and_p[0]
ref = scipy_shift(inputs[0],
result_shifts,
mode=border_mode,
cval=constant_value)
if shifts is None:
shifts = (0, ) * len(inputs[0].shape)
for result, shift_range in zip(result_shifts, shifts):
assert abs(result) <= shift_range
assert_allclose(o.d, ref)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert_allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
assert np.all(i.g == 0)
def test_random_shift_forward_backward(seed, inshape, shifts, border_mode, ctx, func_name):
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
inputs = [rng.randn(*inshape).astype(np.float32)]
i = nn.Variable(inputs[0].shape, need_grad=True)
i.d = inputs[0]
# NNabla forward
with nn.context_scope(ctx), nn.auto_forward():
o = F.random_shift(i, shifts, border_mode, 0, seed)
result_shifts = (0, 0, 0)
max_correl = 0
for shift_amount in itertools.product(*map(tuple, map(lambda x: range(*x), [(-2, 3) for _ in range(len(inshape))]))):
r = scipy_shift(inputs[0], shift_amount, mode=border_mode)
correl_and_p = pearsonr(o.d.flatten(), r.flatten())
if correl_and_p[0] > max_correl:
result_shifts = shift_amount
max_correl = correl_and_p[0]
ref = scipy_shift(inputs[0], result_shifts, mode=border_mode)
if shifts is None:
shifts = (0,) * len(inputs[0].shape)
for result, shift_range in zip(result_shifts, shifts):
assert abs(result) <= shift_range
assert np.allclose(o.d, ref)
assert o.parent.name == func_name
# Skipping Backward check
g = np.random.randn(*i.shape)
i.g = g
o_grad = np.random.randn(*o.shape)
o.g = o_grad
o.parent.backward([i], [o])
ref_grad = i.g.copy() - g
# Check accum=False with NaN gradient
i.g = np.float32('nan')
o.parent.backward([i], [o], [False])
assert not np.any(np.isnan(i.g))
# Check if accum option works
i.g[...] = 1
o.g = o_grad
o.parent.backward([i], [o], [False])
assert np.allclose(i.g, ref_grad, atol=1e-6)
# Check if need_grad works
i.g[...] = 0
i.need_grad = False
o_grad = rng.randn(*i.shape).astype(i.data.dtype)
o.backward(o_grad)
assert np.all(i.g == 0)
def test_shift_with_noise(self):
""" Test measuring the time-shift between traces shifted from one another, with added 10% gaussian noise """
trace1 = np.sin(2 * np.pi * np.linspace(0, 10, 64))
trace2 = scipy_shift(trace1, 5)
trace1 = trace1[6:-6]
trace2 = trace2[6:-6]
trace1 += 0.05 * np.random.random(size=trace1.shape)
trace2 += 0.05 * np.random.random(size=trace2.shape)
shift = register_time_shift(trace1, trace2)
self.assertEqual(shift, -5)
def _convolve_psf_shift(self, image, dcr, weights, psf=None, useInverse=False, n_substep=20):
if psf is None:
psf = self.psf
interp_x = np.linspace(0, len(weights), num=n_substep)
weights_interp = np.interp(interp_x, np.arange(len(weights)), weights)
psf_image = psf.computeKernelImage().getArray()
kernel = np.zeros_like(psf_image)
for n in range(n_substep):
x_loc = (dcr.dx.start*(n_substep - (n + 0.5)) + dcr.dx.end*(n + 0.5))/n_substep
y_loc = (dcr.dy.start*(n_substep - (n + 0.5)) + dcr.dy.end*(n + 0.5))/n_substep
shift = (y_loc, x_loc)
kernel += weights_interp[n]*scipy_shift(psf_image, shift)
return self._kernel_convolution(image, kernel, useInverse=useInverse)
def build_matched_psf(self, el, rotation_angle, weather=lsst_weather, stretch_threshold=None):
"""Sub-routine to calculate the PSF as elongated by DCR for a given exposure.
Once the matched templates incorporate variable seeing, this function should also match the seeing.
Parameters
----------
el : lsst.afw.geom.Angle
Elevation angle of the observation. If not set, it is read from the exposure.
rotation_angle : lsst.afw.geom.Angle
Sky rotation angle of the observation. If not set it is calculated from the exposure metadata.
weather : lsst.afw.coord Weather
Class containing the measured temperature, pressure, and humidity
at the observatory during an observation
Returns
-------
lsst.meas.algorithms KernelPsf object
Designed to be passed to a lsst.afw.image ExposureF through the method setPsf()
"""
dcr_gen = self._dcr_generator(self.bandpass, pixel_scale=self.pixel_scale,
observatory=self.observatory, weather=weather,
elevation=el, rotation_angle=rotation_angle, use_midpoint=False)
dcr_list = [dcr for dcr in dcr_gen]
sub_weights = self._subband_weights()
psf_vals = self.psf.computeKernelImage().getArray()
psf_vals_out = np.zeros((self.psf_size, self.psf_size))
if stretch_threshold is None:
stretch_test = [False for dcr in dcr_list]
else:
stretch_test = [(abs(dcr.dy.start - dcr.dy.end) > stretch_threshold) or
(abs(dcr.dx.start - dcr.dx.end) > stretch_threshold) for dcr in dcr_list]
for f, dcr in enumerate(dcr_list):
if stretch_test[f]:
psf_vals_out += fft_shift_convolve(psf_vals, dcr, weights=sub_weights[f])
else:
shift = ((dcr.dy.start + dcr.dy.end)/2., (dcr.dx.start + dcr.dx.end)/2.)
psf_vals_out += scipy_shift(psf_vals, shift)
psf_image = afwImage.ImageD(self.psf_size, self.psf_size)
psf_image.getArray()[:, :] = psf_vals_out
psfK = afwMath.FixedKernel(psf_image)
psf = measAlg.KernelPsf(psfK)
return psf
def ref_shift(x, shifts, border_mode):
if shifts is None:
shifts = (0,) * len(x.shape)
return scipy_shift(x, shifts, mode=border_mode)
def shifted_zero(self,down,right):
index_vertical = np.arange(self.height)
index_horizontal = np.arange(self.width)
temp1 = self.arr[list(scipy_shift(index_vertical, down, output = int, mode = 'constant')),:]
temp2 = temp1[:,list(scipy_shift(index_horizontal, right, output = int, mode = 'constant'))]
return temp2
def ref_shift(x, shifts, border_mode):
if shifts is None:
shifts = (0,) * len(x.shape)
return scipy_shift(x, shifts, mode=border_mode, order=1)
def build_matched_template(self, exposure=None, model=None, el=None, rotation_angle=None,
return_weights=True, weather=None, stretch_threshold=None):
"""Sub-routine to calculate the sum of the model images shifted by DCR for a given exposure.
Parameters
----------
exposure : lsst.afw.image.ExposureF object, optional if all metadata is supplied directly.
Single exposure to create a DCR-matched template for from the model.
model : List of numpy ndarrays, optional
The DCR model. If not set, then self.model is used.
el : lsst.afw.geom.Angle, optional
Elevation angle of the observation. If not set, it is read from the exposure.
rotation_angle : lsst.afw.geom.Angle, optional
Sky rotation angle of the observation. If not set it is calculated from the exposure metadata.
return_weights : bool, optional
Set to True to return the variance plane, as well as the image.
weather : lsst.afw.coord Weather, optional
Class containing the measured temperature, pressure, and humidity
at the observatory during an observation
Weather data is read from the exposure metadata if not supplied.
stretch_threshold : float, optional
Set to simulate the effect of DCR across each sub-band by stretching the model.
If any sub-band has DCR greater than this amount for an exposure the finite bandwidth
FFT-based shift will be used for that instance rather than a simple shift.
Returns
-------
np.ndarrary or (np.ndarray, np.ndarray)
Returns a numpy ndarray of the image values for the template.
If ``return_weights`` is set, then it returns a tuple of the image and variance arrays.
"""
if el is None:
el = exposure.getInfo().getVisitInfo().getBoresightAzAlt().getLatitude()
if rotation_angle is None:
rotation_angle = calculate_rotation_angle(exposure)
if weather is None:
try:
weather = exposure.getInfo().getVisitInfo().getWeather()
except:
weather = lsst_weather
dcr_gen = self._dcr_generator(self.bandpass, pixel_scale=self.pixel_scale,
observatory=self.observatory, weather=weather,
elevation=el, rotation_angle=rotation_angle, use_midpoint=False)
dcr_list = [dcr for dcr in dcr_gen]
if stretch_threshold is None:
stretch_test = [False for dcr in dcr_list]
else:
stretch_test = [(abs(dcr.dy.start - dcr.dy.end) > stretch_threshold) or
(abs(dcr.dx.start - dcr.dx.end) > stretch_threshold) for dcr in dcr_list]
template = np.zeros((self.y_size, self.x_size))
if return_weights:
weights = np.zeros((self.y_size, self.x_size))
if model is None:
model_use = self.model
else:
model_use = model
subband_weights = self._subband_weights()
for f, dcr in enumerate(dcr_list):
if stretch_test[f]:
template += fft_shift_convolve(model_use[f], dcr, weights=subband_weights[f])
if return_weights:
weights += fft_shift_convolve(self.weights, dcr, weights=subband_weights[f])
else:
shift = ((dcr.dy.start + dcr.dy.end)/2., (dcr.dx.start + dcr.dx.end)/2.)
template += scipy_shift(model_use[f], shift)
if return_weights:
weights += scipy_shift(self.weights, shift)
if return_weights:
weights /= self.n_step
variance = np.zeros((self.y_size, self.x_size))
variance[weights > 0] = 1./weights[weights > 0]
return (template, variance)
else:
return template
def getModeMatrix(self, npola=1, shift=None, angle=None):
'''
Returns the matrix containing the mode profiles.
Note that while you can set two polarizations, the modes profiles are obtained under a scalar apporoximation.
Parameters
----------
npola : int (1 or 2)
number of polarizations considered. For npola = 2, the mode matrix will be a block diagonal matrix.
shift : list or None
(slow) value of a coordinate offset, allows to return the mode matrix for a fiber with the center shifted with regard
to the center of the observation window.
defaults to None
rotation: float or None
(slow) angle in radians, allows to rotate the mode matrix with an arbitrary angle.
Note that the rotation is applied BEFORE the transverse shift.
defaults to None
Returns
-------
M : numpy array
the matrix representing the basis of the propagating modes.
'''
assert (self.profiles)
if shift is not None:
assert (len(shift) == 2)
N = self.profiles[0].shape[0]
M = np.zeros((N * npola, npola * self.number), dtype=np.complex128)
angle = angle / np.pi * 180. if (angle is not None) else None
for pol in range(npola):
for ind, modeProfile in enumerate(self.profiles):
if (shift is None and angle is None):
M[pol * N:(pol + 1) * N, pol * self.number +
ind] = modeProfile #.reshape(1,self._npoints**2)
else:
mode2D = modeProfile.reshape([self.indexProfile.npoints] *
2)
if angle is not None:
mode2D = \
scipy_rotate(mode2D.real,angle,reshape=False) + \
complex(0,1)*scipy_rotate(mode2D.imag,angle,reshape=False)
if shift is not None:
mode2D = \
scipy_shift(input=mode2D.real,shift=shift) \
+ complex(0,1)*scipy_shift(input=mode2D.imag,shift=shift)
M[pol * N:(pol + 1) * N,
pol * self.number + ind] = mode2D.flatten()
self.modeMatrix = M
return M
def _gen_corrected_im(x, y, data):
for i, (xs, ys, frame) in enumerate(zip(x, y, data)):
corrected = scipy_shift(frame, (xs, ys))
printProgress(i, len(data))
yield corrected
def _calculate_new_model(self, last_solution, exp_cut=None, use_variance=True,
airmass_weight=False, stretch_threshold=None, useNonnegative=False):
"""Sub-routine to calculate a new model from the residuals of forward-modeling the previous solution.
Parameters
----------
last_solution : list of np.ndarrays
One np.ndarray for each model sub-band, from the previous iteration.
exp_cut : List of bools
Exposures that failed to converge in the previous iteration are flagged,
and not included in the current iteration solution.
use_variance : bool, optional
Set to weight pixels by their inverse variance when combining images.
airmass_weight : bool, optional
Set to weight each observation by its airmass.
stretch_threshold : float, optional
Set to simulate the effect of DCR across each sub-band by stretching the model.
If any sub-band has DCR greater than this amount for an exposure the finite bandwidth
FFT-based shift will be used for that instance rather than a simple shift.
Returns
-------
Tuple of two lists of np.ndarrays
One np.ndarray for each model sub-band, and the associated inverse variance array.
"""
residual_arr = [np.zeros((self.y_size, self.x_size)) for f in range(self.n_step)]
inverse_var_arr = [np.zeros((self.y_size, self.x_size)) for f in range(self.n_step)]
sub_weights = self._subband_weights()
if exp_cut is None:
exp_cut = [False for exp_i in range(self.n_images)]
for exp_i, exp in enumerate(self.exposures):
if exp_cut[exp_i]:
continue
img, inverse_var = self._extract_image(exp, use_variance=use_variance)
visitInfo = exp.getInfo().getVisitInfo()
el = visitInfo.getBoresightAzAlt().getLatitude()
rotation_angle = calculate_rotation_angle(exp)
weather = visitInfo.getWeather()
dcr_gen = self._dcr_generator(self.bandpass, pixel_scale=self.pixel_scale,
observatory=self.observatory, weather=weather,
elevation=el, rotation_angle=rotation_angle,
use_midpoint=False)
if airmass_weight:
inverse_var *= visitInfo.getBoresightAirmass()
if use_variance:
inverse_var = np.sqrt(inverse_var)
template = self.build_matched_template(exp, model=last_solution, return_weights=False,
stretch_threshold=stretch_threshold)
residual = img - template
for f, dcr in enumerate(dcr_gen):
# NOTE: We do NOT use the finite bandwidth shift here,
# since that would require a deconvolution.
inv_shift = (-(dcr.dy.start + dcr.dy.end)/2., -(dcr.dx.start + dcr.dx.end)/2.)
inv_var_shift = scipy_shift(inverse_var, inv_shift)
residual_shift = scipy_shift(residual, inv_shift)
inv_var_shift[inv_var_shift < 0] = 0.
residual_arr[f] += residual_shift*inv_var_shift # *weights_shift
inverse_var_arr[f] += inv_var_shift
new_solution = copy.deepcopy(last_solution)
for f in range(self.n_step):
inds_use = inverse_var_arr[f] > 0
new_solution[f][inds_use] += residual_arr[f][inds_use]/inverse_var_arr[f][inds_use]
if useNonnegative:
neg_inds = new_solution[f] < 0
new_solution[f][neg_inds] = 0.
return (new_solution, inverse_var_arr)