def flaglos(cube, threshold, dilation):
rms0 = GetRMS(cube,
rmsMode="mad",
fluxRange="negative",
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0,
twoPass=True)
# COMPACT ...
#los_rms = np.nanstd(cube, axis=0)
#los_rms = 1.4826 * np.nanmedian(abs(cube), axis=0)
# ... OR NOT in order to use the TwoPass option of GetRMS
los_rms = np.zeros((cube.shape[-2], cube.shape[-1]))
for xx in range(cube.shape[-1]):
for yy in range(cube.shape[-2]):
los_rms[yy, xx] = GetRMS(cube[:, yy:yy + 1, xx:xx + 1],
rmsMode="mad",
fluxRange="all",
twoPass=True)
# Mask all LOS whose RMS is > threshold*STD above rms0 (optionally extended to neighbouring LOS using binary dilation with a box structuring element)
los_rms_disp = np.nanstd(los_rms)
los_rms = (los_rms < rms0 + threshold * los_rms_disp)
los_rms = binary_dilation(~los_rms,
structure=np.ones((dilation, dilation)))
cube[:, los_rms] = np.nan
return cube
def filter(mask, cube, header, clipMethod, threshold, rmsMode, fluxRange, verbose):
err.message("Running threshold finder.")
# Sanity checks of user input
err.ensure(
clipMethod in {"absolute", "relative"},
"Threshold finder failed. Illegal clip method: '" + str(clipMethod) + "'.")
err.ensure(
rmsMode in {"std", "mad", "gauss", "negative"},
"Threshold finder failed. Illegal RMS mode: '" + str(rmsMode) + "'.")
err.ensure(
fluxRange in {"positive", "negative", "all"},
"Threshold finder failed. Illegal flux range: '" + str(fluxRange) + "'.")
# Scale threshold by RMS if requested
if clipMethod == "relative":
threshold *= GetRMS(cube, rmsMode=rmsMode, fluxRange=fluxRange, zoomx=1, zoomy=1, zoomz=1, verbose=verbose)
# Print some information and check sign of threshold
err.message(" Using threshold of " + str(threshold) + ".")
err.ensure(threshold >= 0.0, "Threshold finder failed. Threshold value is negative.")
# Run the threshold finder, setting bit 1 of the mask for |cube| >= |threshold|:
np.bitwise_or(mask, np.greater_equal(np.absolute(cube), threshold), out=mask)
return
def sigma_scale(cube,
scaleX=False,
scaleY=False,
scaleZ=True,
edgeX=0,
edgeY=0,
edgeZ=0,
statistic="mad",
fluxRange="all",
method="global",
windowSpatial=20,
windowSpectral=20,
gridSpatial=0,
gridSpectral=0,
interpolation="none"):
# Print some informational messages
err.message("Generating noise-scaled data cube:")
err.message(" Selecting " + str(method) + " noise measurement method.")
if statistic == "mad":
err.message(" Applying median absolute deviation to " +
str(fluxRange) + " pixels.")
if statistic == "std":
err.message(" Applying standard deviation to " + str(fluxRange) +
" pixels.")
if statistic == "gauss":
err.message(" Applying Gaussian fit to " + str(fluxRange) +
" pixels.")
if statistic == "negative":
err.message(" Applying Gaussian fit to negative pixels.")
# Check the dimensions of the cube (could be obtained from header information)
dimensions = np.shape(cube)
# LOCAL noise measurement within running window (slower and less memory-friendly)
if method == "local":
# Make window sizes integers >= 1
windowSpatial = max(int(windowSpatial), 1)
windowSpectral = max(int(windowSpectral), 1)
# Ensure that window sizes are odd
windowSpatial += (1 - windowSpatial % 2)
windowSpectral += (1 - windowSpectral % 2)
# Set grid sizes to half the window sizes if undefined
if not gridSpatial: gridSpatial = windowSpatial // 2
if not gridSpectral: gridSpectral = windowSpectral // 2
# Make grid sizes integers >= 1
gridSpatial = max(int(gridSpatial), 1)
gridSpectral = max(int(gridSpectral), 1)
# Ensure that grid sizes are odd
gridSpatial += (1 - gridSpatial % 2)
gridSpectral += (1 - gridSpectral % 2)
# Print grid and window sizes adopted
err.message(" Using grid size of [" + str(gridSpatial) + ", " +
str(gridSpectral) + "]")
err.message(" and window size of [" + str(windowSpatial) + ", " +
str(windowSpectral) + "].")
# Generate grid points to be used
gridPointsZ = np.arange(
(dimensions[0] - gridSpectral *
(int(math.ceil(float(dimensions[0]) / float(gridSpectral))) - 1))
// 2, dimensions[0], gridSpectral)
gridPointsY = np.arange(
(dimensions[1] - gridSpatial *
(int(math.ceil(float(dimensions[1]) / float(gridSpatial))) - 1))
// 2, dimensions[1], gridSpatial)
gridPointsX = np.arange(
(dimensions[2] - gridSpatial *
(int(math.ceil(float(dimensions[2]) / float(gridSpatial))) - 1))
// 2, dimensions[2], gridSpatial)
# Divide grid and window sizes by 2 to get radii
radiusGridSpatial = gridSpatial // 2
radiusGridSpectral = gridSpectral // 2
radiusWindowSpatial = windowSpatial // 2
radiusWindowSpectral = windowSpectral // 2
# Create empty cube (filled with NaN) to hold noise values
rms_cube = np.full(cube.shape, np.nan, dtype=cube.dtype)
# Determine RMS across window centred on grid cell
for z in gridPointsZ:
for y in gridPointsY:
for x in gridPointsX:
grid = (max(0, z - radiusGridSpectral),
min(dimensions[0], z + radiusGridSpectral + 1),
max(0, y - radiusGridSpatial),
min(dimensions[1], y + radiusGridSpatial + 1),
max(0, x - radiusGridSpatial),
min(dimensions[2], x + radiusGridSpatial + 1))
window = (max(0, z - radiusWindowSpectral),
min(dimensions[0], z + radiusWindowSpectral + 1),
max(0, y - radiusWindowSpatial),
min(dimensions[1], y + radiusWindowSpatial + 1),
max(0, x - radiusWindowSpatial),
min(dimensions[2], x + radiusWindowSpatial + 1))
if not np.all(
np.isnan(
cube[window[0]:window[1], window[2]:window[3],
window[4]:window[5]])):
if interpolation == "linear" or interpolation == "cubic":
# Write value into grid point for later interpolation
rms_cube[z, y,
x] = GetRMS(cube[window[0]:window[1],
window[2]:window[3],
window[4]:window[5]],
rmsMode=statistic,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0)
else:
# Fill entire grid cell
rms_cube[grid[0]:grid[1], grid[2]:grid[3],
grid[4]:grid[5]] = GetRMS(
cube[window[0]:window[1],
window[2]:window[3],
window[4]:window[5]],
rmsMode=statistic,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0)
del grid, window
# Carry out interpolation if requested, taking NaNs into account
if interpolation == "linear" or interpolation == "cubic":
err.message(" Interpolating in between grid points (" +
str(interpolation) + ").")
# First across each spatial plane
if gridSpatial > 1:
for z in gridPointsZ:
for y in gridPointsY:
data_values = rms_cube[z, y, gridPointsX]
not_nan = np.logical_not(np.isnan(data_values))
if any(not_nan):
interp_coords = np.arange(0, dimensions[2])
if interpolation == "cubic":
spline = InterpolatedUnivariateSpline(
gridPointsX[not_nan], data_values[not_nan])
rms_cube[z, y, 0:dimensions[2]] = spline(
interp_coords)
del spline
else:
interp_values = np.interp(
interp_coords, gridPointsX[not_nan],
data_values[not_nan])
rms_cube[z, y, 0:dimensions[2]] = interp_values
del interp_values
del interp_coords
del data_values, not_nan
for x in range(dimensions[2]):
data_values = rms_cube[z, gridPointsY, x]
not_nan = np.logical_not(np.isnan(data_values))
if any(not_nan):
interp_coords = np.arange(0, dimensions[1])
if interpolation == "cubic":
spline = InterpolatedUnivariateSpline(
gridPointsY[not_nan], data_values[not_nan])
rms_cube[z, 0:dimensions[1],
x] = spline(interp_coords)
del spline
else:
interp_values = np.interp(
interp_coords, gridPointsY[not_nan],
data_values[not_nan])
rms_cube[z, 0:dimensions[1], x] = interp_values
del interp_values
del interp_coords
del data_values, not_nan
# Alternative option: 2-D spatial interpolation using SciPy's interp2d
#from scipy.interpolate import interp2d
#xx, yy = np.meshgrid(gridPointsX, gridPointsY)
#data_values = rms_cube[z, yy, xx]
#f = interp2d(gridPointsX, gridPointsY, data_values, kind="cubic")
#interp_coords_x = np.arange(0, dimensions[2])
#interp_coords_y = np.arange(0, dimensions[1])
#rms_cube[z, :, :] = f(interp_coords_x, interp_coords_y)
# Then along the spectral axis
if gridSpectral > 1:
for y in range(dimensions[1]):
for x in range(dimensions[2]):
data_values = rms_cube[gridPointsZ, y, x]
not_nan = np.logical_not(np.isnan(data_values))
if any(not_nan):
interp_coords = np.arange(0, dimensions[0])
if interpolation == "cubic":
spline = InterpolatedUnivariateSpline(
gridPointsZ[not_nan], data_values[not_nan])
rms_cube[0:dimensions[0], y,
x] = spline(interp_coords)
del spline
else:
interp_values = np.interp(
interp_coords, gridPointsZ[not_nan],
data_values[not_nan])
rms_cube[0:dimensions[0], y, x] = interp_values
del interp_values
del interp_coords
del data_values, not_nan
# Replace any invalid RMS values with NaN
with np.errstate(invalid="ignore"):
rms_cube[rms_cube <= 0] = np.nan
# Divide data cube by RMS cube
cube /= rms_cube
# Delete the RMS cube again to release its memory
#del rms_cube
# GLOBAL noise measurement on entire 2D plane (faster and more memory-friendly)
else:
# Define the range over which statistics are calculated
z1 = int(edgeZ)
z2 = int(dimensions[0] - edgeZ)
y1 = int(edgeY)
y2 = int(dimensions[1] - edgeY)
x1 = int(edgeX)
x2 = int(dimensions[2] - edgeX)
# Make sure edges don't exceed cube size
err.ensure(z1 < z2 and y1 < y2 and x1 < x2,
"Edge size exceeds cube size for at least one axis.")
# Create empty cube (filled with 1) to hold noise values
rms_cube = np.ones(cube.shape, dtype=cube.dtype)
# Measure noise across 2D planes and scale cube accordingly
if scaleZ:
for i in range(dimensions[0]):
if not np.all(np.isnan(cube[i, y1:y2, x1:x2])):
rms = GetRMS(cube[i, y1:y2, x1:x2],
rmsMode=statistic,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0)
if rms > 0:
rms_cube[i, :, :] *= rms
cube[i, :, :] /= rms
if scaleY:
for i in range(dimensions[1]):
if not np.all(np.isnan(cube[z1:z2, i, x1:x2])):
rms = GetRMS(cube[z1:z2, i, x1:x2],
rmsMode=statistic,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0)
if rms > 0:
rms_cube[:, i, :] *= rms
cube[:, i, :] /= rms
if scaleX:
for i in range(dimensions[2]):
if not np.all(np.isnan(cube[z1:z2, y1:y2, i])):
rms = GetRMS(cube[z1:z2, y1:y2, i],
rmsMode=statistic,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=0)
if rms > 0:
rms_cube[:, :, i] *= rms
cube[:, :, i] /= rms
err.message("Noise-scaled data cube generated.\n")
return cube, rms_cube
def SCfinder_mem(cube,
mask,
header,
t0,
kernels=[
[0, 0, 0, "b"],
],
threshold=3.5,
sizeFilter=0,
maskScaleXY=2.0,
maskScaleZ=2.0,
kernelUnit="pixel",
edgeMode="constant",
rmsMode="negative",
fluxRange="all",
verbose=0,
perSCkernel=False,
scaleX=False,
scaleY=False,
scaleZ=True,
edgeX=0,
edgeY=0,
edgeZ=0,
method="1d2d",
windowSpatial=20,
windowSpectral=20,
gridSpatial=0,
gridSpectral=0,
interpolation="none"):
# Define a few constants
FWHM_CONST = 2.0 * math.sqrt(2.0 * math.log(
2.0)) # Conversion between sigma and FWHM of Gaussian function
MAX_PIX_CONST = 1.0e+6 # Maximum number of pixels for noise calculation; sampling is set accordingly
# Check for NaN in cube
found_nan = np.isnan(cube).any()
# Set sampling sampleRms for rms measurement
sampleRms = max(
1,
int((float(cube.size) / MAX_PIX_CONST)**(1.0 /
min(3, len(cube.shape)))))
# Measure noise in original cube with sampling "sampleRms"
rms = GetRMS(cube,
rmsMode=rmsMode,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=verbose,
sample=sampleRms)
# Loop over all kernels
for kernel in kernels:
[kx, ky, kz, kt] = kernel
if verbose:
err.linebreak()
err.print_progress_time(t0)
err.message(" Filter {0:} {1:} {2:} {3:} ...".format(
kx, ky, kz, kt))
if kernelUnit == "world" or kernelUnit == "w":
if verbose: err.message(" Converting filter size to pixels ...")
kx = abs(float(kx) / header["CDELT1"])
ky = abs(float(ky) / header["CDELT2"])
kz = abs(float(kz) / header["CDELT3"])
if kt == "b":
if kz != int(math.ceil(kz)) and verbose:
err.warning(
"Rounding width of boxcar z kernel to next integer.")
kz = int(math.ceil(kz))
# Create a copy of the original cube
cube_smooth = np.copy(cube)
# Replace all NaNs with zero
if found_nan:
cube_smooth[np.isnan(cube)] = 0.0
cube_smooth[(cube_smooth > 0) & (mask > 0)] = maskScaleXY * rms
cube_smooth[(cube_smooth < 0) & (mask > 0)] = -maskScaleXY * rms
# Spatial smoothing
if kx + ky:
cube_smooth = ndimage.filters.gaussian_filter(
cube_smooth, [0, ky / FWHM_CONST, kx / FWHM_CONST],
mode=edgeMode)
# Spectral smoothing
if kz:
if kt == "b":
cube_smooth = ndimage.filters.uniform_filter1d(cube_smooth,
kz,
axis=0,
mode=edgeMode)
elif kt == "g":
cube_smooth = ndimage.filters.gaussian_filter1d(cube_smooth,
kz /
FWHM_CONST,
axis=0,
mode=edgeMode)
# Re-insert the NaNs taken out earlier
if found_nan:
cube_smooth[np.isnan(cube)] = np.nan
# Per-kernel noise normalisation (Time consuming!)
if perSCkernel:
cube_smooth, noise_smooth = sigma_scale(
cube_smooth,
scaleX=scaleX,
scaleY=scaleY,
scaleZ=scaleZ,
edgeX=edgeX,
edgeY=edgeY,
edgeZ=edgeZ,
statistic=rmsMode,
fluxRange=fluxRange,
method=method,
windowSpatial=windowSpatial,
windowSpectral=windowSpectral,
gridSpatial=gridSpatial,
gridSpectral=gridSpectral,
interpolation=interpolation)
# Calculate the RMS of the smoothed (possibly normalised) cube
rms_smooth = GetRMS(cube_smooth,
rmsMode=rmsMode,
fluxRange=fluxRange,
zoomx=1,
zoomy=1,
zoomz=1,
verbose=verbose,
sample=sampleRms)
# Add pixels above threshold to mask by setting bit 1
err.message(" Applying +/- {0:} sigma detection threshold".format(
threshold))
with np.errstate(invalid="ignore"):
mask |= (np.absolute(cube_smooth) >= threshold * rms_smooth)
#mask = np.bitwise_or(mask, np.greater_equal(np.absolute(cube_smooth), threshold * rms_smooth))
# Delete smoothed cube again
del cube_smooth
return
def SCfinder_mem(cube, header, t0, kernels=[[0, 0, 0, "b"],], threshold=3.5, sizeFilter=0, maskScaleXY=2.0, maskScaleZ=2.0, kernelUnit="pixel", edgeMode="constant", rmsMode="negative", fluxRange="all", verbose=0):
# Define a few constants
FWHM_CONST = 2.0 * math.sqrt(2.0 * math.log(2.0)) # Conversion between sigma and FWHM of Gaussian function
MAX_PIX_CONST = 1e+6 # Maximum number of pixels for noise calculation; sampling is set accordingly
# Create binary mask array
msk = np.zeros(cube.shape, np.bool)
found_nan = np.isnan(cube).sum()
# Set sampling sampleRms for rms measurement
sampleRms = max(1, int((float(np.array(cube.shape).prod()) / MAX_PIX_CONST)**(1.0 / min(3, len(cube.shape)))))
# Measure noise in original cube with sampling "sampleRms"
rms = GetRMS(cube, rmsMode=rmsMode, fluxRange=fluxRange, zoomx=1, zoomy=1, zoomz=1, verbose=verbose, sample=sampleRms)
# Loop over all kernels
for kernel in kernels:
[kx, ky, kz, kt] = kernel
if verbose:
err.linebreak()
err.print_progress_time(t0)
err.message(" Filter %s %s %s %s ..." % (kx, ky, kz, kt))
if kernelUnit == "world" or kernelUnit == "w":
if verbose: err.message(" Converting filter size to pixels ...")
kx = abs(float(kx) / header["CDELT1"])
ky = abs(float(ky) / header["CDELT2"])
kz = abs(float(kz) / header["CDELT3"])
if kt == "b":
if kz != int(math.ceil(kz)) and verbose: err.warning("Rounding width of boxcar z kernel to next integer.")
kz = int(math.ceil(kz))
# Create a copy of the original cube
smoothedCube = np.copy(cube)
# Replace all NaNs with zero (and INFs with a finite number)
if found_nan: smoothedCube = np.nan_to_num(smoothedCube)
smoothedCube[(smoothedCube > 0) & (msk > 0)] = +maskScaleXY * rms
smoothedCube[(smoothedCube < 0) & (msk > 0)] = -maskScaleXY * rms
# Spatial smoothing
if kx + ky:
smoothedCube = ndimage.filters.gaussian_filter(smoothedCube, [0, ky / FWHM_CONST, kx / FWHM_CONST], mode=edgeMode)
# Spectral smoothing
if kz:
if kt == "b": smoothedCube = ndimage.filters.uniform_filter1d(smoothedCube, kz, axis=0, mode=edgeMode)
elif kt == "g": smoothedCube = ndimage.filters.gaussian_filter1d(smoothedCube, kz / FWHM_CONST, axis=0, mode=edgeMode)
# Re-insert the NaNs (but not the INFs) taken out earlier
if found_nan: smoothedCube[np.isnan(cube)] = np.nan
# Calculate the RMS of the smoothed cube:
smoothedrms = GetRMS(smoothedCube, rmsMode=rmsMode, fluxRange=fluxRange, zoomx=1, zoomy=1, zoomz=1, verbose=verbose, sample=sampleRms)
# Get rid of the NaNs a second time
#if found_nan: smoothedCube = np.nan_to_num(smoothedCube)
# NOTE: This should not be necessary because any comparison with NaN will always yield False.
# Hence, NaN pixels will never be included in the mask below.
# Add pixels above threshold to mask by setting bit 1
with np.errstate(invalid="ignore"):
msk = np.bitwise_or(msk, np.greater_equal(np.absolute(smoothedCube), threshold * smoothedrms))
# Delete smoothed cube again
del smoothedCube
return msk