def test_sky_crop_order():
# Test that order of cropping and subtraction doesn't affect result if inside cropped image
img_dim = 80
img = np.random.random((img_dim, img_dim))
xmax, ymax = np.random.randint(20, high=img_dim - 20, size=2)
img[ymax, xmax] = img.max() * 3 + 1 # Add max pixel at pre-determined location
isz = 2 * np.min([xmax, img.shape[1] - xmax - 1, ymax, img.shape[0] - ymax - 1]) + 1
dr = 2
r1 = isz // 2 - dr
# Correct then crop
correct = sky_correction(img, r1=r1, dr=dr, center=(xmax, ymax))[0]
correct_crop = crop_max(correct, isz, filtmed=False)[0]
# Crop then correct
cropped, center = crop_max(img, isz, filtmed=False)
crop_correct = sky_correction(cropped, r1=r1, dr=dr)[0]
# Make sure find expected center
assert center == (xmax, ymax)
# Use 10 * machine precision to make sure passes
assert np.all(np.abs(correct_crop - crop_correct) < 10 * np.finfo(float).eps)
def test_crop_max():
img_size = 80 # Same size as NIRISS images
img = np.random.random((img_size, img_size))
xmax, ymax = np.random.randint(0, high=img_size, size=2)
img[ymax,
xmax] = img.max() * 3 + 1 # Add max pixel at pre-determined location
# Pre-calculate expected max size
isz_max = (2 * np.min(
[xmax, img.shape[1] - xmax - 1, ymax, img.shape[0] - ymax - 1]) + 1)
isz_too_big = isz_max + 1
# Using full message because we also check the suggested size
size_msg = (
f"The specified cropped image size, {isz_too_big}, is greater than the distance"
" to the PSF center in at least one dimension. The max size for this image is"
f" {isz_max}")
with pytest.raises(ValueError, match=size_msg):
# Above max size should raise the error
tools.crop_max(img, isz_too_big, filtmed=False)
# Setting filtmed=False because the simple image has only one pixe > 1
img_cropped, center_pos = tools.crop_max(img, isz_max, filtmed=False)
assert center_pos == (xmax, ymax)
assert img_cropped.shape[0] == isz_max
assert img_cropped.shape[0] == img_cropped.shape[1]
def test_clean_crop_order():
# Test that clean_data performs both subtraction and cropping as expected
img_dim = 80
n_im = 5
data = np.random.random((n_im, img_dim, img_dim))
xmax, ymax = np.random.randint(20, high=img_dim - 20, size=2)
data[:, ymax, xmax] = data.max() * 3 + 1 # Add max pixel at pre-determined location
isz = (
2 * np.min([xmax, data.shape[2] - xmax - 1, ymax, data.shape[1] - ymax - 1]) + 1
)
dr = 2
r1 = isz // 2 - dr
# Clean cube all at once
cube = data.copy()
cube_clean = clean_data(cube, isz=isz, r1=r1, dr=dr, apod=False, f_kernel=None)
img_cube_clean = cube_clean[0]
# Correct then crop
img = data.copy()[0]
correct = sky_correction(img, r1=r1, dr=dr, center=(xmax, ymax))[0]
correct[correct < 0] = 0
img_correct_crop = crop_max(correct, isz, filtmed=False)[0]
assert np.all(np.abs(img_correct_crop - img_cube_clean) < 10 * np.finfo(float).eps)
def clean_data(data, isz=None, r1=None, dr=None, edge=0,
bad_map=None, add_bad=[], apod=True,
offx=0, offy=0, sky=True, window=None,
darkfile=None, f_kernel=3, verbose=False):
""" Clean data.
Parameters:
-----------
`data` {np.array} -- datacube containing the NRM data\n
`isz` {int} -- Size of the cropped image (default: {None})\n
`r1` {int} -- Radius of the rings to compute background sky (default: {None})\n
`dr` {int} -- Outer radius to compute sky (default: {None})\n
`edge` {int} -- Patch the edges of the image (VLT/SPHERE artifact, default: {200}),\n
`checkrad` {bool} -- If True, check the resizing and sky substraction parameters (default: {False})\n
Returns:
--------
`cube` {np.array} -- Cleaned datacube.
"""
n_im = data.shape[0]
cube_cleaned = [] # np.zeros([n_im, isz, isz])
l_bad_frame = []
for i in tqdm(range(n_im), ncols=100, desc='Cleaning', leave=False):
img0 = data[i]
img0 = _apply_edge_correction(img0, edge=edge)
if bad_map is not None:
img1 = fix_bad_pixels(img0, bad_map, add_bad=add_bad)
else:
img1 = img0.copy()
img1 = _remove_dark(img1, darkfile=darkfile, verbose=verbose)
im_rec_max = crop_max(img1, isz, offx=offx, offy=offy, f=f_kernel)[0]
if sky:
img_biased = sky_correction(im_rec_max, r1=r1, dr=dr,
verbose=verbose)[0]
else:
img_biased = im_rec_max.copy()
img_biased[img_biased < 0] = 0 # Remove negative pixels
if (img_biased.shape[0] != img_biased.shape[1]) or (img_biased.shape[0] != isz):
l_bad_frame.append(i)
else:
if apod:
img = apply_windowing(img_biased, window=window)
else:
img = img_biased.copy()
cube_cleaned.append(img)
if verbose:
print('Bad centering frame number:', l_bad_frame)
cube_cleaned = np.array(cube_cleaned)
return cube_cleaned
def check_data_params(filename, isz, r1, dr, bad_map=None, add_bad=[],
edge=0, remove_bad=True, nframe=0, ihdu=0, f_kernel=3,
offx=0, offy=0, apod=False, window=None):
""" Check the input parameters for the cleaning.
Parameters:
-----------
`filename` {str}: filename containing the datacube,\n
`isz` {int}: Size of the cropped image (default: 256)\n
`r1` {int}: Radius of the rings to compute background sky (default: 100)\n
`dr` {int}: Outer radius to compute sky (default: 10)\n
`bad_map` {array}: Bad pixel map with 0 and 1 where 1 set for a bad pixel (default: None),\n
`add_bad` {list}: List of 2d coordinates of bad pixels/cosmic rays (default: []),\n
`edge` {int}: Number of pixel to be removed on the edge of the image (SPHERE),\n
`remove_bad` {bool}: If True, the bad pixels are removed using a gaussian interpolation,\n
`nframe` {int}: Frame number to be shown (default: 0),\n
`ihdu` {int}: Hdu number of the fits file. Normally 1 for NIRISS and 0 for SPHERE (default: 0).
"""
data = fits.open(filename)[ihdu].data
img0 = data[nframe]
if (bad_map is None) and (len(add_bad) != 0):
bad_map = np.zeros(img0.shape)
if edge != 0:
img0[:, 0:edge] = 0
img0[:, -edge:-1] = 0
img0[0:edge, :] = 0
img0[-edge:-1, :] = 0
if (bad_map is not None) & (remove_bad):
img1 = fix_bad_pixels(img0, bad_map, add_bad=add_bad)
else:
img1 = img0.copy()
cropped_infos = crop_max(img1, isz, offx=offx, offy=offy, f=f_kernel)
pos = cropped_infos[1]
noBadPixel = False
bad_pix_x, bad_pix_y = [], []
if (bad_map is not None) & (len(add_bad) != 0):
for j in range(len(add_bad)):
bad_map[add_bad[j][1], add_bad[j][0]] = 1
bad_pix = np.where(bad_map == 1)
bad_pix_x = bad_pix[0]
bad_pix_y = bad_pix[1]
else:
noBadPixel = True
r2 = r1 + dr
theta = np.linspace(0, 2*np.pi, 100)
x0 = pos[0]
y0 = pos[1]
x1 = r1 * np.cos(theta) + x0
y1 = r1 * np.sin(theta) + y0
x2 = r2 * np.cos(theta) + x0
y2 = r2 * np.sin(theta) + y0
if window is not None:
r3 = window
x3 = r3 * np.cos(theta) + x0
y3 = r3 * np.sin(theta) + y0
xs1, ys1 = x0 + isz//2, y0 + isz//2
xs2, ys2 = x0 - isz//2, y0 + isz//2
xs3, ys3 = x0 - isz//2, y0 - isz//2
xs4, ys4 = x0 + isz//2, y0 - isz//2
max_val = img1[y0, x0]
fig = plt.figure(figsize=(5, 5))
plt.title("--- CLEANING PARAMETERS ---")
plt.imshow(img1, norm=PowerNorm(.5), cmap='afmhot', vmin=0, vmax=max_val)
plt.plot(x1, y1, label='Inner radius for sky subtraction')
plt.plot(x2, y2, label='Outer radius for sky subtraction')
if apod:
if window is not None:
plt.plot(x3, y3, '--', label='Super-gaussian windowing')
plt.plot(x0, y0, '+', color='c', ms=10, label='Centering position')
plt.plot([xs1, xs2, xs3, xs4, xs1], [ys1, ys2, ys3, ys4, ys1], 'w--',
label='Resized image')
if not noBadPixel:
if remove_bad:
label = 'Fixed hot/bad pixels'
else:
label = 'Hot/bad pixels'
plt.scatter(bad_pix_y, bad_pix_x, color='', marker='s',
edgecolors='r', s=20, label=label)
plt.xlabel('X [pix]')
plt.ylabel('Y [pix]')
plt.legend(fontsize=8, loc=1)
plt.tight_layout()
return fig
def clean_data(data,
isz=None,
r1=None,
dr=None,
edge=0,
r2=None,
bad_map=None,
add_bad=[],
apod=True,
offx=0,
offy=0,
sky=True,
window=None,
f_kernel=3,
verbose=False):
""" Clean data.
Parameters:
-----------
`data` {np.array} -- datacube containing the NRM data\n
`isz` {int} -- Size of the cropped image (default: {None})\n
`r1` {int} -- Radius of the rings to compute background sky (default: {None})\n
`dr` {int} -- Outer radius to compute sky (default: {None})\n
`edge` {int} -- Patch the edges of the image (VLT/SPHERE artifact, default: {200}),\n
`checkrad` {bool} -- If True, check the resizing and sky substraction parameters (default: {False})\n
Returns:
--------
`cube` {np.array} -- Cleaned datacube.
"""
# print(data.shape[1])
# if data.shape[1] % 2 == 1:
# data = np.array([im[:-1, :-1] for im in data])
n_im = data.shape[0]
cube_cleaned = np.zeros([n_im, isz, isz])
for i in tqdm(range(n_im), ncols=100, desc='Cleaning', leave=False):
img0 = data[i]
if edge != 0:
img0[:, 0:edge] = 0
img0[:, -edge:-1] = 0
img0[0:edge, :] = 0
img0[-edge:-1, :] = 0
if bad_map is not None:
img1 = fix_bad_pixels(img0, bad_map, add_bad=add_bad)
else:
img1 = img0.copy()
im_rec_max = crop_max(img1, isz, offx=offx, offy=offy, f=f_kernel)[0]
if sky:
img_biased = sky_correction(im_rec_max,
r1=r1,
dr=dr,
verbose=verbose)[0]
else:
img_biased = im_rec_max.copy()
img_biased[img_biased < 0] = 0 # Remove negative pixels
if img_biased.shape[0] != img_biased.shape[1]:
cprint(
'\nCropped image do not have same X, Y dimensions -> check isz',
'red')
return None
if apod:
if r2 is None:
r2 = isz // 3
img = apply_windowing(img_biased, window=window)
else:
img = img_biased.copy()
cube_cleaned[i] = img
return cube_cleaned
def clean_data(
data,
isz=None,
r1=None,
dr=None,
edge=0,
bad_map=None,
add_bad=None,
apod=True,
offx=0,
offy=0,
sky=True,
window=None,
darkfile=None,
f_kernel=3,
verbose=False,
*,
mask=None,
):
"""Clean data.
Parameters:
-----------
`data` {np.array} -- datacube containing the NRM data\n
`isz` {int} -- Size of the cropped image (default: {None})\n
`r1` {int} -- Radius of the rings to compute background sky (default: {None})\n
`dr` {int} -- Outer radius to compute sky (default: {None})\n
`edge` {int} -- Patch the edges of the image (VLT/SPHERE artifact, default: {200}),\n
`checkrad` {bool} -- If True, check the resizing and sky substraction parameters (default: {False})\n
Returns:
--------
`cube` {np.array} -- Cleaned datacube.
"""
n_im = data.shape[0]
cube_cleaned = [] # np.zeros([n_im, isz, isz])
l_bad_frame = []
bad_map, add_bad = _get_3d_bad_pixels(bad_map, add_bad, data)
for i in tqdm(range(n_im), ncols=100, desc="Cleaning", leave=False):
img0 = data[i]
img0 = _apply_edge_correction(img0, edge=edge)
if bad_map is not None:
img1 = fix_bad_pixels(img0, bad_map[i], add_bad=add_bad[i])
else:
img1 = img0.copy()
img1 = _remove_dark(img1, darkfile=darkfile, verbose=verbose)
if isz is not None:
# Get expected center for sky correction
filtmed = f_kernel is not None
center = find_max(img1, filtmed=filtmed, f=f_kernel)
else:
center = None
if sky and (r1 is not None or mask is not None):
img_biased = sky_correction(
img1, r1=r1, dr=dr, verbose=verbose, center=center, mask=mask
)[0]
elif sky:
warnings.warn(
"sky is set to True, but r1 and mask are set to None. Skipping sky correction",
RuntimeWarning,
)
img_biased = img1.copy()
else:
img_biased = img1.copy()
img_biased[img_biased < 0] = 0 # Remove negative pixels
if isz is not None:
# Get expected center for sky correction
filtmed = f_kernel is not None
im_rec_max = crop_max(
img_biased, isz, offx=offx, offy=offy, filtmed=filtmed, f=f_kernel
)[0]
else:
im_rec_max = img_biased.copy()
if (
(im_rec_max.shape[0] != im_rec_max.shape[1])
or (isz is not None and im_rec_max.shape[0] != isz)
or (isz is None and im_rec_max.shape[0] != img0.shape[0])
):
l_bad_frame.append(i)
else:
if apod and window is not None:
img = apply_windowing(im_rec_max, window=window)
elif apod:
warnings.warn(
"apod is set to True, but window is None. Skipping apodisation",
RuntimeWarning,
)
img = im_rec_max.copy()
else:
img = im_rec_max.copy()
cube_cleaned.append(img)
if verbose:
print("Bad centering frame number:", l_bad_frame)
cube_cleaned = np.array(cube_cleaned)
return cube_cleaned
def show_clean_params(
filename,
isz,
r1=None,
dr=None,
bad_map=None,
add_bad=None,
edge=0,
remove_bad=True,
nframe=0,
ihdu=0,
f_kernel=3,
offx=0,
offy=0,
apod=False,
window=None,
*,
mask=None,
):
"""Display the input parameters for the cleaning.
Parameters:
-----------
`filename` {str}: filename containing the datacube,\n
`isz` {int}: Size of the cropped image (default: 256)\n
`r1` {int}: Radius of the rings to compute background sky (default: 100)\n
`dr` {int}: Outer radius to compute sky (default: 10)\n
`bad_map` {array}: Bad pixel map with 0 and 1 where 1 set for a bad pixel (default: None),\n
`add_bad` {list}: List of 2d coordinates of bad pixels/cosmic rays (default: []),\n
`edge` {int}: Number of pixel to be removed on the edge of the image (SPHERE),\n
`remove_bad` {bool}: If True, the bad pixels are removed using a gaussian interpolation,\n
`nframe` {int}: Frame number to be shown (default: 0),\n
`ihdu` {int}: Hdu number of the fits file. Normally 1 for NIRISS and 0 for SPHERE (default: 0).
"""
with fits.open(filename) as fd:
data = fd[ihdu].data
img0 = data[nframe]
dims = img0.shape
if isz is None:
print(
"Warning: isz not found (None by default). isz is set to the original image size (%i)"
% (dims[0]),
file=sys.stderr,
)
isz = dims[0]
bad_map, add_bad = _get_3d_bad_pixels(bad_map, add_bad, data)
bmap0 = bad_map[nframe]
ab0 = add_bad[nframe]
if edge != 0:
img0[:, 0:edge] = 0
img0[:, -edge:-1] = 0
img0[0:edge, :] = 0
img0[-edge:-1, :] = 0
if (bad_map is not None) & (remove_bad):
img1 = fix_bad_pixels(img0, bmap0, add_bad=ab0)
else:
img1 = img0.copy()
cropped_infos = crop_max(img1, isz, offx=offx, offy=offy, f=f_kernel)
pos = cropped_infos[1]
noBadPixel = False
bad_pix_x, bad_pix_y = [], []
if np.any(bmap0):
if len(ab0) != 0:
for j in range(len(ab0)):
bmap0[ab0[j][1], ab0[j][0]] = 1
bad_pix = np.where(bmap0 == 1)
bad_pix_x = bad_pix[0]
bad_pix_y = bad_pix[1]
else:
noBadPixel = True
theta = np.linspace(0, 2 * np.pi, 100)
x0 = pos[0]
y0 = pos[1]
if r1 is not None:
x1 = r1 * np.cos(theta) + x0
y1 = r1 * np.sin(theta) + y0
if dr is not None:
r2 = r1 + dr
x2 = r2 * np.cos(theta) + x0
y2 = r2 * np.sin(theta) + y0
sky_method = "ring"
elif mask is not None:
bg_coords = np.where(mask == 1)
bg_x = bg_coords[0]
bg_y = bg_coords[1]
sky_method = "mask"
if window is not None:
r3 = window
x3 = r3 * np.cos(theta) + x0
y3 = r3 * np.sin(theta) + y0
xs1, ys1 = x0 + isz // 2, y0 + isz // 2
xs2, ys2 = x0 - isz // 2, y0 + isz // 2
xs3, ys3 = x0 - isz // 2, y0 - isz // 2
xs4, ys4 = x0 + isz // 2, y0 - isz // 2
max_val = img1[y0, x0]
fig = plt.figure(figsize=(5, 5))
plt.title("--- CLEANING PARAMETERS ---")
plt.imshow(img1, norm=PowerNorm(0.5, vmin=0, vmax=max_val), cmap="afmhot")
if sky_method == "ring":
if dr is not None:
plt.plot(x1, y1, label="Inner radius for sky subtraction")
plt.plot(x2, y2, label="Outer radius for sky subtraction")
else:
plt.plot(x1, y1, label="Boundary for sky subtraction")
elif sky_method == "mask":
plt.scatter(
bg_y,
bg_x,
color="None",
marker="s",
edgecolors="C0",
s=20,
label="Pixels used for sky subtraction",
)
if apod:
if window is not None:
plt.plot(x3, y3, "--", label="Super-gaussian windowing")
plt.plot(x0, y0, "+", color="c", ms=10, label="Centering position")
plt.plot(
[xs1, xs2, xs3, xs4, xs1],
[ys1, ys2, ys3, ys4, ys1],
"w--",
label="Resized image",
)
plt.xlim((0, dims[0] - 1))
plt.ylim((0, dims[1] - 1))
if not noBadPixel:
if remove_bad:
label = "Fixed hot/bad pixels"
else:
label = "Hot/bad pixels"
plt.scatter(
bad_pix_y,
bad_pix_x,
color="None",
marker="s",
edgecolors="r",
facecolors="None",
s=20,
label=label,
)
plt.xlabel("X [pix]")
plt.ylabel("Y [pix]")
plt.legend(fontsize=8, loc=1)
plt.tight_layout()
return fig