def dem_slope(img, xres_m, yres_m):
"""
compute the slope of a DEM.
Parameters
----------
img: numpy.ndarray
the input DEM
xres_m: int or float
the x resolution of the DEM in same units as the height values
yres_m: int or float
the y resolution of the DEM in same units as the height values
Returns
-------
"""
boundary = 'extend'
kernel = CustomKernel(np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) / (8. * xres_m))
xchangerate_array = convolve(img, kernel, normalize_kernel=False, boundary=boundary,
nan_treatment='fill', fill_value=np.nan)
kernel = CustomKernel(np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) / (8. * yres_m))
ychangerate_array = convolve(img, kernel, normalize_kernel=False, boundary=boundary,
nan_treatment='fill', fill_value=np.nan)
slope_radians = np.arctan(np.sqrt(np.square(xchangerate_array) + np.square(ychangerate_array)))
slope_degrees = np.rad2deg(slope_radians)
return slope_degrees
def do_filtering(image, noisemap, matchedfilter, psf):
from astropy.convolution import CustomKernel
from astropy.convolution import convolve
## add artefact detector
grad = get_grad_logerror(noisemap)
mask_1 = grad > 0.25
mask_2 = np.log10(noisemap) < -2.25
ind_artefact = np.logical_and(mask_1, mask_2)
noisemap[ind_artefact] = np.nan
nanvalues = np.isnan(noisemap) | np.isnan(image)
# changed from Steve's code so that NaNs are dealt with properly using astropy convolve
image[nanvalues] = np.nan
noisemap[nanvalues] = np.nan
weightmap = 1. / noisemap ** 2.
norm = np.sum(psf * matchedfilter) / np.sum(matchedfilter ** 2.)
matchedfilter_kernel = CustomKernel(norm * matchedfilter)
matchedfilter_kernel_square = CustomKernel((norm * matchedfilter) ** 2)
print('max filt', np.max(psf))
conv_image = convolve(image * weightmap, matchedfilter_kernel)
conv_noise = convolve(weightmap, matchedfilter_kernel_square)
conv_image = conv_image / conv_noise
conv_noise = (1. / (conv_noise ** 0.5))
conv_image[nanvalues] = np.nan
conv_noise[nanvalues] = np.nan
conv_image -= np.nanmean(conv_image) # meansubtraction, ignore, NaN pixels:
return (conv_image, conv_noise, matchedfilter_kernel)
def jopa(image, scat_profile):
"""
Calculate scattering light image and substract it from given file.
Science spectrum and standard star spectrum must have the same Y axis
bining.
"""
from tqdm import tqdm
from scipy.interpolate import interp2d
img = fits.getdata(image)
hdr = fits.getheader(image)
prf = fits.getdata(scat_profile, 'PSF_SCATTERING')
prfs = prf['PSF_SCAT_Y']
wcenters = prf['PARS_WAVE_CENTER']
nbins = wcenters.size
_, waves = get_waves(hdr)
f = interp2d(np.arange(prfs.shape[1]), wcenters, prfs)
scat_image = np.zeros_like(img)
print("Scattering light for each column...")
for i in tqdm(range(img.shape[1])):
if waves[i] <= wcenters[0]:
kernel = CustomKernel(prfs[0, :])
elif waves[i] >= wcenters[-1]:
kernel = CustomKernel(prfs[-1, :])
else:
kernel = CustomKernel(f(f.x, waves[i]))
scat_image[:, i] = convolve(img[:, i],
kernel,
normalize_kernel=False,
boundary='wrap')
print("Smooth scattering light along the dispersion direction...")
kern_disp = CustomKernel(np.sum(prfs, axis=0) / nbins)
for j in tqdm(range(img.shape[0])):
scat_image[j, :] = convolve(scat_image[j, :],
kernel,
normalize_kernel=True)
# save file with scattering light
ofile = image.replace('.fits', '_scatlight.fits')
print("Write scattering light file: {}".format(ofile))
hdr['HISTORY'] = ("Create scattering light image based on the {} using "
"profile {}".format(image, scat_profile))
fits.PrimaryHDU(data=scat_image, header=hdr).writeto(ofile, overwrite=True)
# substract from image
oimfile = image.replace('.fits', '_scat_substr.fits')
print("Write image without with substracted scattering light: {}".format(
oimfile))
hdr['HISTORY'] = "Substract scattereing light."
fits.PrimaryHDU(data=img - scat_image, header=hdr).writeto(oimfile,
overwrite=True)
def smooth_days(x, y, kernel, ksize, fix):
"""Smooth data (x, y) given parameters kernel, ksize, fix"""
kernel = kernel.lower()
if kernel == 'none' or not ksize:
return x, y
if kernel == 'box':
from astropy.convolution import convolve, Box1DKernel
ysm = convolve(y, Box1DKernel(ksize), boundary='extend')
elif kernel == 'gaussian':
from astropy.convolution import convolve, Gaussian1DKernel
ysm = convolve(y, Gaussian1DKernel(ksize), boundary='extend')
elif kernel == 'trapezoid':
from astropy.convolution import convolve, Trapezoid1DKernel
ysm = convolve(y, Trapezoid1DKernel(ksize), boundary='extend')
elif kernel == 'triangle':
from astropy.convolution import convolve, CustomKernel
f = triangle(ksize)
ysm = convolve(y, CustomKernel(f), boundary='extend')
else:
raise ValueError('{} not supported.'.format(kernel))
redo = int(np.floor(ksize / 2))
xsm = x
if fix == 'cull':
_sfs = slice(0, len(ysm) - redo)
xsm = x[_sfs]
ysm = ysm[_sfs]
elif fix == 'redo':
for i in range(len(y) - redo, len(y)):
ave = 0.0
cnt = 0
for j in range(i, len(y)):
ave += y[j]
cnt += 1
ysm[i] = (ave / cnt + ysm[i]) / 2.0
return xsm, ysm
def dem_aspect(img):
"""
compute the aspect of a DEM.
Parameters
----------
img: numpy.ndarray
the DEM array
Returns
-------
numpy.ndarray
the computed aspect array
"""
boundary = 'extend'
kernel = CustomKernel(np.array([[-1, 0, 1], [-2, 0, 2], [-1, 0, 1]]) / 8.)
xchangerate_array = -convolve(img,
kernel,
normalize_kernel=False,
boundary=boundary,
nan_treatment='fill',
fill_value=np.nan)
kernel = CustomKernel(np.array([[-1, -2, -1], [0, 0, 0], [1, 2, 1]]) / 8.)
ychangerate_array = -convolve(img,
kernel,
normalize_kernel=False,
boundary=boundary,
nan_treatment='fill',
fill_value=np.nan)
aspect = np.rad2deg(np.arctan2(ychangerate_array, -xchangerate_array))
aspect_value = np.copy(aspect)
# numpy raises a warning if np.nan values are set to False in a mask
with np.errstate(invalid='ignore'):
mask = aspect < 0.0
aspect_value[mask] = 90.0 - aspect[mask]
mask = aspect > 90.0
aspect_value[mask] = 360.0 - aspect[mask] + 90.0
mask = (aspect >= 0.0) & (aspect < 90.0)
aspect_value[mask] = 90.0 - aspect[mask]
return aspect_value
def output_spectra(spectrum, opacity, filename, longitude, latitude, em_mean,
em_std, temp_bright, beam_area, sigma_tau, sigma_tau_smooth,
mean):
"""
Write the spectrum (velocity, flux and opacity) to a votable format file.
:param spectrum: The spectrum to be output.
:param opacity: The opacity to be output.
:param filename: The filename to be created
:param longitude: The galactic longitude of the target object
:param latitude: The galactic latitude of the target object
"""
table = Table(meta={'name': filename, 'id': 'opacity'})
table.add_column(Column(name='plane', data=spectrum.plane))
table.add_column(
Column(name='velocity',
data=spectrum.velocity,
unit='m/s',
description='velocity relative to LSRK'))
table.add_column(Column(name='opacity', data=opacity))
#following smooth should be using same parameter with line 950
hann_window = np.hanning(31)
hann_kernel = CustomKernel(hann_window)
smooth_y = convolve(opacity, hann_kernel, boundary='extend')
table.add_column(
Column(name='smooth_opacity',
data=smooth_y,
description='opacity smooth with hanning window 31 channels.'))
table.add_column(
Column(name='flux',
data=spectrum.flux,
unit='Jy',
description='Flux per beam'))
table.add_column(Column(name='temp_brightness', data=temp_bright,
unit='K'))
table.add_column(
Column(
name='sigma_tau',
data=sigma_tau,
description='Noise in the absorption profile, before smoothing'))
table.add_column(
Column(name='sigma_tau_smooth',
data=sigma_tau_smooth,
description='Noise in the absorption profile, after smoothing'))
if len(em_mean) > 0:
# The emission may not be available, so don't include it if not
table.add_column(Column(name='em_mean', data=em_mean, unit='K'))
table.add_column(Column(name='em_std', data=em_std, unit='K'))
votable = from_table(table)
votable.infos.append(Info('ra', 'longitude', longitude.value))
votable.infos.append(Info('dec', 'latitude', latitude.value))
votable.infos.append(Info('beam_area', 'beam_area', beam_area))
votable.infos.append(Info('cont', 'continuum', mean))
writeto(votable, filename)
def plot_em_abs(em_mean, em_std, opacity, sigma_tau_smooth, filename, CDELT3,
CRVAL3):
# read in the emission and absorption spectra and
# plot an emission-absortion diagram
if len(em_mean) == 0:
if os.path.exists(filename):
os.remove(filename)
return
#smooth the spectrum with hanning of 31 channels (1.6 km/s)
hann_window = np.hanning(31)
hann_kernel = CustomKernel(hann_window)
smooth_opacity = convolve(opacity, hann_kernel, boundary='extend')
smooth_abs = 1.0 - smooth_opacity
good = np.where(smooth_abs > (3.0 * sigma_tau_smooth))
super_good = np.where(smooth_abs > (5.0 * sigma_tau_smooth))
#print('sigma_tau_smooth=',sigma_tau_smooth)
#print('super_good',super_good)
sg = np.ravel(np.array(super_good))
#print('sg=',sg)
velo = CRVAL3 + (sg + 10) * CDELT3 #revised 20181023
velo = np.around(velo, decimals=1)
#print('velo',velo)
nsuper = len(sg)
fig = plt.figure()
plt.plot(smooth_abs[good],
em_mean[good],
color='lightsteelblue',
markerfacecolor='blue',
markeredgecolor='black',
marker='.')
plt.plot(smooth_abs[super_good],
em_mean[super_good],
markeredgecolor='deeppink',
markerfacecolor='black',
marker='.',
linestyle='')
ax = fig.add_subplot(111)
for i in np.arange(nsuper):
if (velo[i] % 1) == 0:
ax.annotate('%s' % velo[i],
xy=(smooth_abs[sg[i]], em_mean[sg[i]]),
textcoords='data')
plt.xlabel(r'1-$e^{(-\tau)}$')
plt.ylabel(r'$T_B$ (K)')
plt.grid(True)
plt.savefig(filename)
#plt.show()
plt.close()
return
def make_Gaussian_beam_kernel(header,oversampling):
bmaj = np.abs(header['BMAJ']/header['CDELT1'])/(2*np.sqrt(2*np.log(2)))
bmin = np.abs(header['BMIN']/header['CDELT2'])/(2*np.sqrt(2*np.log(2)))
bpa = header['BPA']+90.
size = int(oversampling*bmaj)
if size % 2 == 0:## to catch non odd kernels
size = size +1
gauss = makeGaussian_bpa(size=size,amplitude=1, std=[bmaj,bmin], bpa=bpa,center=None)
np.save('Gauss_model.npy',gauss)
return CustomKernel(gauss)
def plot_spectrum(x, y, filename, title, sigma_tau, sigma_tau_smooth):
"""
Output a plot of opacity vs LSR velocity to a specified file.
:param x: The velocity data
:param y: The opacity values for each velocity step
:param rms: array of channel RMS values
:param filename: The file the plot should be written to. Should be
an .eps or .pdf file.
:param title: The title for the plot
:param con_start_vel: The minimum velocity that the continuum was measured at.
:param con_end_vel: The maximum velocity that the continuum was measured at.
"""
fig = plt.figure()
plt.plot(x / 1000, y, color='gray', zorder=2)
#overplot a smoothed spectrum
#smooth the spectrum with hanning of 31 channels (1.6 km/s)
hann_window = np.hanning(31)
hann_kernel = CustomKernel(hann_window)
smooth_y = convolve(y, hann_kernel, boundary='extend')
plt.plot(x / 1000, smooth_y, color='blue', zorder=4)
if len(sigma_tau) > 0:
tau_max = 1 + sigma_tau
tau_min = 1 - sigma_tau
plt.fill_between(x / 1000,
tau_min,
tau_max,
facecolor='silver',
color='silver',
alpha=0.75,
zorder=1)
tau_max_smooth = 1 + sigma_tau_smooth
tau_min_smooth = 1 - sigma_tau_smooth
plt.fill_between(x / 1000,
tau_min_smooth,
tau_max_smooth,
facecolor='cornflowerblue',
color='cornflowerblue',
zorder=3,
alpha=0.5)
plt.axhline(1, color='black')
plt.xlabel(r'Velocity (km/s)')
plt.ylabel(r'$e^{(-\tau)}$')
plt.title(title)
plt.grid(True)
plt.savefig(filename)
#plt.show()
plt.close()
return
def fft_2d_hanning(mask, size=2):
assert np.min(mask.shape) > size * 2 + 1
assert size > 1
idx = np.linspace(-0.5, 0.5, size * 2 + 1, endpoint=True)
xx, yy = np.meshgrid(idx, idx)
n = np.sqrt(xx ** 2 + yy ** 2)
hann_kernel = (1 + np.cos(2 * np.pi * n)) / 2
hann_kernel[n > 0.5] = 0
hann_kernel = CustomKernel(hann_kernel)
hann_kernel.normalize("integral")
# Reduce mask size to apodize on the edge
apod = ~shrink_mask(mask, hann_kernel)
# Final convolution goes to 0 on the edge
apod = convolve_fft(apod, hann_kernel)
return apod
def optical_depth(self):
tau_array = -np.log(self.T_B / self.T_bg)
tau_noise = round(np.std(tau_array[self.vmin_ind:self.vmax_ind + 1]),
3)
self.tau_peak = max(tau_array)
self.tau = tau_array
self.tau_noise = tau_noise
self.tau_raw = tau_array
self.opacity = 1 - np.exp(-1 * self.tau_raw)
# Katie's awesome (hanning) smoothing, width is 5 velocity channels wide
hanning_sz = 5
hann_window = np.hanning(hanning_sz)
hann_kernel = CustomKernel(hann_window)
self.tau = convolve(self.tau, hann_kernel, boundary='extend')
def get_mean_continuum(spectrum):
"""
Calculate the mean of the continuum values. This is based on precalculated regions where there is no gas expected.
:param spectrum: The spectrum to be analysed, should be a numpy array of
plane, velocity and flux values.
:return: A single float which is the mean continuum flux.
"""
# velocities = spectrum.velocity
# velo1=-123.063627344+(start_channel+50)*0.103056351495
# velo2=-123.063627344+(start_channel+300)*0.103056351495
# velo3=-123.063627344+(start_channel+1700)*0.103056351495
# velo4=-123.063627344+(start_channel+1950)*0.103056351495
# continuum_range = np.where(((velocities>velo1*1000.)&(velocities<velo2*1000.))|((velocities>velo3*1000.)&(velocities<velo4*1000.)))
#
# # print ("...gave sample of", continuum_sample)
# mean_cont = np.nanmean(continuum_sample)
print("len(spectrum.flux[:])", len(spectrum.flux[:]))
channelno = np.arange(len(spectrum.flux[:]))
continuum_range = np.where((channelno < 260) | (channelno > 740))
x = np.ravel(continuum_range)
print("shape(x)", np.shape(x))
continuum_sample = spectrum.flux[continuum_range]
y = np.ravel(continuum_sample)
print("shape(y)", np.shape(y))
mean_cont = np.nanmean(continuum_sample)
print("mean_cont", mean_cont)
opacity_sample = continuum_sample / mean_cont
sd_cont = np.std(opacity_sample)
#same with line 950
hann_window = np.hanning(31)
hann_kernel = CustomKernel(hann_window)
smooth_opacity_sample = convolve(opacity_sample,
hann_kernel,
boundary='extend')
sd_cont_smooth = np.nanstd(smooth_opacity_sample)
print("Found standard deviation of the smoothed spectrum of {}".format(
sd_cont_smooth))
return mean_cont, sd_cont, sd_cont_smooth
def interpolate(img: CCDData):
"""
Takes a image with a mask for bad pixels and interpolates over the bad pixels
:param img: the image you want to interpolate bad pixels in
:return: interpolated image
"""
# TODO combiner does not care about this and marks it invalid still
from astropy.convolution import CustomKernel
from astropy.convolution import interpolate_replace_nans
# TODO this here doesn't really work all that well -> extended regions cause artifacts at border
kernel_array = np.array([[1, 1, 1], [1, 1, 1], [1, 1, 1]
]) / 9 # average of all surrounding pixels
# noinspection PyTypeChecker
kernel = CustomKernel(
kernel_array
) # TODO the original pipeline used fixpix, which says it uses linear interpolation
img.data[np.logical_not(img.mask)] = np.NaN
img.data = interpolate_replace_nans(img.data, kernel)
return img
def set_smoothing_kernel(self,
kernel=None,
kernel_width=None,
custom_array_kernel=None,
custom_kernel_function=None,
function_array_size=21):
#if custom_kernel_array is None and custom_kernel_function is None:
if custom_array_kernel is not None:
custom_kernel_array = np.array([i for i in custom_array_kernel])
self.kernel_func = CustomKernel(custom_kernel_array)
self.kernel_func_type = SmoothingKernels.CUSTOM
elif custom_kernel_function is not None:
if isinstance(custom_kernel_function, Fittable1DModel):
self.kernel_func = Model1DKernel(custom_kernel_function,
x_size=function_array_size)
else:
self.kernel_func = custom_kernel_function
self.kernel_func_type = SmoothingKernels.CUSTOM
elif kernel in default_smoothing_kernels:
width = int(kernel_width)
if kernel == SmoothingKernels.GAUSSIAN1D:
self.kernel_func = Gaussian1DKernel(width)
elif kernel == SmoothingKernels.Box1D:
self.kernel_func = Box1DKernel(width)
elif kernel == SmoothingKernels.MEDIAN:
self.kernel_func = ndimage.median_filter
else:
raise Exception("Unsupported smoothing kernel " + str(kernel))
self.kernel_func_type = kernel
self.kernel_width = width
else:
raise Exception("Problem while setting the smoothing kernel")
def run(self, maps, kernel, which="all", downsampling_factor=None):
"""
Run TS map estimation.
Requires `counts`, `exposure` and `background` map to run.
Parameters
----------
maps : dict
Input sky maps.
kernel : `astropy.convolution.Kernel2D` or 2D `~numpy.ndarray`
Source model kernel.
which : list of str or 'all'
Which maps to compute.
downsampling_factor : int
Sample down the input maps to speed up the computation. Only integer
values that are a multiple of 2 are allowed. Note that the kernel is
not sampled down, but must be provided with the downsampled bin size.
Returns
-------
maps : dict
Result maps.
"""
p = self.parameters
if (np.array(kernel.shape) > np.array(
maps["counts"].data.shape)).any():
raise ValueError(
"Kernel shape larger than map shape, please adjust"
" size of the kernel")
if downsampling_factor:
shape = maps["counts"].data.shape
pad_width = symmetric_crop_pad_width(shape, shape_2N(shape))[0]
for name in maps:
maps[name] = maps[name].pad(pad_width)
preserve_counts = name in ["counts", "background", "exclusion"]
maps[name] = maps[name].downsample(
downsampling_factor, preserve_counts=preserve_counts)
if not isinstance(kernel, Kernel2D):
kernel = CustomKernel(kernel)
if which == "all":
which = ["ts", "sqrt_ts", "flux", "flux_err", "flux_ul", "niter"]
result = {}
for name in which:
data = np.nan * np.ones_like(maps["counts"].data)
result[name] = maps["counts"].copy(data=data)
mask = self.mask_default(maps, kernel)
if "mask" in maps:
mask.data &= maps["mask"].data
if p["threshold"] or p["method"] == "root newton":
flux = self.flux_default(maps, kernel).data
else:
flux = None
# prepare dtype for cython methods
counts = maps["counts"].data.astype(float)
background = maps["background"].data.astype(float)
exposure = maps["exposure"].data.astype(float)
# Compute null statistics per pixel for the whole image
c_0 = _cash_cython(counts, background)
error_method = p["error_method"] if "flux_err" in which else "none"
ul_method = p["ul_method"] if "flux_ul" in which else "none"
wrap = partial(
_ts_value,
counts=counts,
exposure=exposure,
background=background,
c_0=c_0,
kernel=kernel,
flux=flux,
method=p["method"],
error_method=error_method,
threshold=p["threshold"],
error_sigma=p["error_sigma"],
ul_method=ul_method,
ul_sigma=p["ul_sigma"],
rtol=p["rtol"],
)
x, y = np.where(mask.data)
positions = list(zip(x, y))
with contextlib.closing(Pool(processes=p["n_jobs"])) as pool:
log.info("Using {} jobs to compute TS map.".format(p["n_jobs"]))
results = pool.map(wrap, positions)
# Set TS values at given positions
j, i = zip(*positions)
for name in ["ts", "flux", "niter"]:
result[name].data[j, i] = [_[name] for _ in results]
if "flux_err" in which:
result["flux_err"].data[j, i] = [_["flux_err"] for _ in results]
if "flux_ul" in which:
result["flux_ul"].data[j, i] = [_["flux_ul"] for _ in results]
# Compute sqrt(TS) values
if "sqrt_ts" in which:
result["sqrt_ts"] = self.sqrt_ts(result["ts"])
if downsampling_factor:
for name in which:
order = 0 if name == "niter" else 1
result[name] = result[name].upsample(
factor=downsampling_factor,
preserve_counts=False,
order=order)
result[name] = result[name].crop(crop_width=pad_width)
return result
def _estimate_significance(self, counts, background):
kernel = CustomKernel(self.kernel_src)
images = compute_lima_image(counts, background, kernel=kernel)
return images["significance"]
def run(self, dataset, kernel, which="all", downsampling_factor=None):
"""
Run TS map estimation.
Requires a MapDataset with counts, exposure and background_model
properly set to run.
Parameters
----------
kernel : `astropy.convolution.Kernel2D` or 2D `~numpy.ndarray`
Source model kernel.
which : list of str or 'all'
Which maps to compute.
downsampling_factor : int
Sample down the input maps to speed up the computation. Only integer
values that are a multiple of 2 are allowed. Note that the kernel is
not sampled down, but must be provided with the downsampled bin size.
Returns
-------
maps : dict
Result maps.
"""
p = self.parameters
if (np.array(kernel.shape) > np.array(dataset.counts.data.shape[1:])).any():
raise ValueError(
"Kernel shape larger than map shape, please adjust"
" size of the kernel"
)
# First create 2D map arrays
counts = dataset.counts.sum_over_axes(keepdims=False)
background = dataset.npred().sum_over_axes(keepdims=False)
exposure = dataset.exposure.sum_over_axes(keepdims=False)
if dataset.mask is not None:
mask = counts.copy(data=(dataset.mask.sum(axis=0) > 0).astype("int"))
else:
mask = counts.copy(data=np.ones_like(counts).astype("int"))
if downsampling_factor:
shape = counts.data.shape
pad_width = symmetric_crop_pad_width(shape, shape_2N(shape))[0]
counts = counts.pad(pad_width).downsample(
downsampling_factor, preserve_counts=True
)
background = background.pad(pad_width).downsample(
downsampling_factor, preserve_counts=True
)
exposure = exposure.pad(pad_width).downsample(
downsampling_factor, preserve_counts=False
)
mask = mask.pad(pad_width).downsample(
downsampling_factor, preserve_counts=False
)
mask.data = mask.data.astype("int")
mask.data &= self.mask_default(exposure, background, kernel).data
if not isinstance(kernel, Kernel2D):
kernel = CustomKernel(kernel)
if which == "all":
which = ["ts", "sqrt_ts", "flux", "flux_err", "flux_ul", "niter"]
result = {}
for name in which:
data = np.nan * np.ones_like(counts.data)
result[name] = counts.copy(data=data)
flux_map = self.flux_default(dataset, kernel)
if p["threshold"] or p["method"] == "root newton":
flux = flux_map.data
else:
flux = None
# prepare dtype for cython methods
counts_array = counts.data.astype(float)
background_array = background.data.astype(float)
exposure_array = exposure.data.astype(float)
# Compute null statistics per pixel for the whole image
c_0 = cash(counts_array, background_array)
error_method = p["error_method"] if "flux_err" in which else "none"
ul_method = p["ul_method"] if "flux_ul" in which else "none"
wrap = functools.partial(
_ts_value,
counts=counts_array,
exposure=exposure_array,
background=background_array,
c_0=c_0,
kernel=kernel,
flux=flux,
method=p["method"],
error_method=error_method,
threshold=p["threshold"],
error_sigma=p["error_sigma"],
ul_method=ul_method,
ul_sigma=p["ul_sigma"],
rtol=p["rtol"],
)
x, y = np.where(np.squeeze(mask.data))
positions = list(zip(x, y))
results = list(map(wrap, positions))
# Set TS values at given positions
j, i = zip(*positions)
for name in ["ts", "flux", "niter"]:
result[name].data[j, i] = [_[name] for _ in results]
if "flux_err" in which:
result["flux_err"].data[j, i] = [_["flux_err"] for _ in results]
if "flux_ul" in which:
result["flux_ul"].data[j, i] = [_["flux_ul"] for _ in results]
# Compute sqrt(TS) values
if "sqrt_ts" in which:
result["sqrt_ts"] = self.sqrt_ts(result["ts"])
if downsampling_factor:
for name in which:
order = 0 if name == "niter" else 1
result[name] = result[name].upsample(
factor=downsampling_factor, preserve_counts=False, order=order
)
result[name] = result[name].crop(crop_width=pad_width)
# Set correct units
if "flux" in which:
result["flux"].unit = flux_map.unit
if "flux_err" in which:
result["flux_err"].unit = flux_map.unit
if "flux_ul" in which:
result["flux_ul"].unit = flux_map.unit
return result
def compute_ts_image(counts,
background,
exposure,
kernel,
mask=None,
flux=None,
method='root brentq',
parallel=True,
threshold=None):
"""
Compute TS image using different optimization methods.
Parameters
----------
counts : `~gammapy.image.SkyImage`
Counts image.
background : `~gammapy.image.SkyImage`
Background image
exposure : `~gammapy.image.SkyImage`
Exposure image
kernel : `astropy.convolution.Kernel2D` or 2D `~numpy.ndarray`
Source model kernel.
flux : float (None)
Flux image used as a starting value for the amplitude fit.
method : str ('root')
The following options are available:
* ``'root brentq'`` (default)
Fit amplitude finding roots of the the derivative of
the fit statistics. Described in Appendix A in Stewart (2009).
* ``'root newton'``
TODO: document
* ``'leastsq iter'``
TODO: document
parallel : bool (True)
Whether to use multiple cores for parallel processing.
threshold : float (None)
If the TS value corresponding to the initial flux estimate is not above
this threshold, the optimizing step is omitted to save computing time.
Returns
-------
images : `~gammapy.image.SkyImageList`
Images (ts, niter, amplitude)
Notes
-----
Negative :math:`TS` values are defined as following:
.. math::
TS = \\left \\{
\\begin{array}{ll}
-TS & : \\textnormal{if} \\ F < 0 \\\\
\\ \\ TS & : \\textnormal{else}
\\end{array}
\\right.
Where :math:`F` is the fitted flux amplitude.
References
----------
[Stewart2009]_
"""
t_0 = time()
log.info("Using method '{}'".format(method))
if not isinstance(kernel, Kernel2D):
kernel = CustomKernel(kernel)
wcs = counts.wcs.deepcopy()
# Parse data type
counts = counts.data.astype(float)
background = background.data.astype(float)
exposure = exposure.data.astype(float)
assert counts.shape == background.shape
assert counts.shape == exposure.shape
# in some image there are pixels, which have exposure, but zero
# background, which doesn't make sense and causes the TS computation
# to fail, this is a temporary fix
mask_ = np.logical_and(background == 0, exposure > 0)
if mask_.any():
log.warning('There are pixels in the data, that have exposure, but '
'zero background, which can cause the ts computation to '
'fail. Setting exposure of this pixels to zero.')
exposure[mask_] = 0
if (flux is None and method != 'root brentq') or threshold is not None:
from scipy.signal import fftconvolve
with np.errstate(invalid='ignore', divide='ignore'):
flux = (counts - background) / exposure / FLUX_FACTOR
flux[~np.isfinite(flux)] = 0
flux = fftconvolve(flux, kernel.array, mode='same') / np.sum(
kernel.array**2)
# Compute null statistics for the whole image
c_0_image = _cash_cython(counts, background)
x_min, x_max = kernel.shape[1] // 2, counts.shape[1] - kernel.shape[1] // 2
y_min, y_max = kernel.shape[0] // 2, counts.shape[0] - kernel.shape[0] // 2
positions = product(range(y_min, y_max), range(x_min, x_max))
# Positions where exposure == 0 are not processed
if mask is None:
mask = exposure > 0
positions = [(j, i) for j, i in positions if mask[j][i]]
wrap = partial(_ts_value,
counts=counts,
exposure=exposure,
background=background,
c_0_image=c_0_image,
kernel=kernel,
flux=flux,
method=method,
threshold=threshold)
if parallel:
log.info('Using {0} cores to compute TS image.'.format(cpu_count()))
pool = Pool()
results = pool.map(wrap, positions)
pool.close()
pool.join()
else:
results = map(wrap, positions)
assert positions, (
"Positions are empty: possibly kernel " +
"{} is larger than counts {}".format(kernel.shape, counts.shape))
# Set TS values at given positions
j, i = zip(*positions)
ts = np.ones(counts.shape) * np.nan
amplitudes = np.ones(counts.shape) * np.nan
niter = np.ones(counts.shape) * np.nan
ts[j, i] = [_[0] for _ in results]
amplitudes[j, i] = [_[1] for _ in results]
niter[j, i] = [_[2] for _ in results]
# Handle negative TS values
with np.errstate(invalid='ignore', divide='ignore'):
sqrt_ts = np.where(ts > 0, np.sqrt(ts), -np.sqrt(-ts))
runtime = np.round(time() - t_0, 2)
meta = OrderedDict(runtime=runtime)
return SkyImageList([
SkyImage(name='ts', data=ts.astype('float32'), wcs=wcs),
SkyImage(name='sqrt_ts', data=sqrt_ts.astype('float32'), wcs=wcs),
SkyImage(name='amplitude', data=amplitudes.astype('float32'), wcs=wcs),
SkyImage(name='niter', data=niter.astype('int16'), wcs=wcs),
],
meta=meta)
# Example with a few sources
#sources = create_sources.multiple(br=[1.0,0.5,0.3,0.1],Dec_offset=[0.1,-0.2,0.3,-0.4],\
#RA_offset=[0.1,-0.3,0.35,0.5],imsize=imsize,pixscale=pixscale)
# Fill in point sources
for source in sources:
data[sources[source]['Dec'],
sources[source]['RA']] = sources[source]['Brightness']
# Write model image with point sources to fits
fits.writeto('model.fits', data, overwrite=True)
# Create custom convolution Kernel from PSF
try:
psf_kern = CustomKernel(psf_data)
except:
print('Adding dummy row and column to psf data for CustomKernel to work.')
psfnpix_y = psf_data.shape[0] + 1
psfnpix_x = psf_data.shape[1] + 1
new_psf = np.zeros((psfnpix_y, psfnpix_x), dtype=np.float64)
new_psf[:psfnpix_y - 1, :psfnpix_x - 1] = psf_data
#hdu_new_psf = fits.PrimaryHDU(new_psf)
psf_header['NAXIS1'] = psfnpix_y
psf_header['NAXIS2'] = psfnpix_x
fits.writeto('new_psf.fits', new_psf, psf_header, overwrite=True)
psf_kern = CustomKernel(new_psf)
#Convolve model image with PSF and save FITS file
convl_image = convolve_fft(data, psf_kern, allow_huge=True)
im_head = psf_header
print mtp
base_year = projection_year
# pop2=scenario_subset['TotalPop_IIASA'].ix[projection_year,scenario]
pop2_urban = scenario_subset['UrbanPop'].ix[projection_year, scenario]
pop2_rural = scenario_subset['RuralPop'].ix[projection_year, scenario]
print 'projected urban population number for ', projection_year, 'is ', pop2_urban
print 'projected rural population number for ', projection_year, 'is ', pop2_rural
pop1_urban = np.sum(mup)
urb_pop_change = pop2_urban - pop1_urban
pop1_rural = np.sum(mrp)
rur_pop_change = pop2_rural - pop1_rural
astropy_kernel1 = CustomKernel(
urban_window) ## astropy uses object kernels
astropy_kernel2 = CustomKernel(rural_window)
### ---This is where the potential for each cell is calculated---------------------------------------------------------------------------------------------###
urban_convolved = np.add(convolve(mtp, astropy_kernel1),
np.power(mtp, urban_alpha))
rural_convolved = np.add(
convolve(mtp, astropy_kernel2), np.power(mtp, rural_alpha)
) ## inner parenthesis in surface formula + itself in the power of cell-specific alpha
urban_potential = mm * urban_convolved
rural_potential = mm * rural_convolved
print 'pop change between base and projection year=', rur_pop_change + urb_pop_change
###--------------------------------------------------------------------------------------------------------------------------------------------------------###
if urb_pop_change > 0:
urban_total_potential = urban_potential.sum()
urban_pot_surface = urban_potential / urban_total_potential
urban_projected_surface = mup + (
def run(self, images, kernel, which='all'):
"""
Run TS image estimation.
Requires `counts`, `exposure` and `background` image to run.
Parameters
----------
kernel : `astropy.convolution.Kernel2D` or 2D `~numpy.ndarray`
Source model kernel.
images : `SkyImageList`
List of input sky images.
which : list of str or 'all'
Which images to compute.
Returns
-------
images : `~gammapy.image.SkyImageList`
Result images.
"""
t_0 = time()
images.check_required(['counts', 'background', 'exposure'])
p = self.parameters
result = SkyImageList()
if not isinstance(kernel, Kernel2D):
kernel = CustomKernel(kernel)
if which == 'all':
which = ['ts', 'sqrt_ts', 'flux', 'flux_err', 'flux_ul', 'niter']
for name in which:
result[name] = SkyImage.empty_like(images['counts'], fill=np.nan)
mask = self.mask_default(images, kernel)
if 'mask' in images.names:
mask.data &= images['mask'].data
x, y = np.where(mask)
positions = list(zip(x, y))
if p['method'] == 'root newton':
flux = self.flux_default(images, kernel).data
else:
flux = None
# prpare dtype for cython methods
counts = images['counts'].data.astype(float)
background = images['background'].data.astype(float)
exposure = images['exposure'].data.astype(float)
# Compute null statistics for the whole image
c_0_image = _cash_cython(counts, background)
error_method = p['error_method'] if 'flux_err' in which else 'none'
ul_method = p['ul_method'] if 'flux_ul' in which else 'none'
wrap = partial(_ts_value,
counts=counts,
exposure=exposure,
background=background,
c_0_image=c_0_image,
kernel=kernel,
flux=flux,
method=p['method'],
error_method=error_method,
threshold=p['threshold'],
error_sigma=p['error_sigma'],
ul_method=ul_method,
ul_sigma=p['ul_sigma'],
rtol=p['rtol'])
if p['parallel']:
log.info('Using {} cores to compute TS image.'.format(cpu_count()))
pool = Pool()
results = pool.map(wrap, positions)
pool.close()
pool.join()
else:
results = map(wrap, positions)
# Set TS values at given positions
j, i = zip(*positions)
for name in ['ts', 'flux', 'niter']:
result[name].data[j, i] = [_[name] for _ in results]
if 'flux_err' in which:
result['flux_err'].data[j, i] = [_['flux_err'] for _ in results]
if 'flux_ul' in which:
result['flux_ul'].data[j, i] = [_['flux_ul'] for _ in results]
# Compute sqrt(TS) values
if 'sqrt_ts' in which:
result['sqrt_ts'] = self.sqrt_ts(result['ts'])
runtime = np.round(time() - t_0, 2)
result.meta = OrderedDict(runtime=runtime)
return result
def run(self, maps, kernel, which='all', downsampling_factor=None):
"""
Run TS map estimation.
Requires `counts`, `exposure` and `background` map to run.
Parameters
----------
kernel : `astropy.convolution.Kernel2D` or 2D `~numpy.ndarray`
Source model kernel.
maps : `OrderedDict`
List of input sky maps.
which : list of str or 'all'
Which maps to compute.
downsampling_factor : int
Sample down the input maps to speed up the computation. Only integer
values that are a multiple of 2 are allowed. Note that the kernel is
not sampled down, but must be provided with the downsampled bin size.
Returns
-------
maps : `OrderedDict`
Result maps.
"""
p = self.parameters
if downsampling_factor:
shape = maps['counts'].data.shape
pad_width = symmetric_crop_pad_width(shape, shape_2N(shape))[0]
for name in maps:
maps[name] = maps[name].pad(pad_width)
preserve_counts = name in ['counts', 'background', 'exclusion']
maps[name] = maps[name].downsample(downsampling_factor,
preserve_counts=preserve_counts)
if not isinstance(kernel, Kernel2D):
kernel = CustomKernel(kernel)
if which == 'all':
which = ['ts', 'sqrt_ts', 'flux', 'flux_err', 'flux_ul', 'niter']
result = OrderedDict()
for name in which:
data = np.nan * np.ones_like(maps['counts'].data)
result[name] = maps['counts'].copy(data=data)
mask = self.mask_default(maps, kernel)
if 'mask' in maps:
mask.data &= maps['mask'].data
if p['method'] == 'root newton':
flux = self.flux_default(maps, kernel).data
else:
flux = None
# prepare dtype for cython methods
counts = maps['counts'].data.astype(float)
background = maps['background'].data.astype(float)
exposure = maps['exposure'].data.astype(float)
# Compute null statistics per pixel for the whole image
c_0 = _cash_cython(counts, background)
error_method = p['error_method'] if 'flux_err' in which else 'none'
ul_method = p['ul_method'] if 'flux_ul' in which else 'none'
wrap = partial(
_ts_value,
counts=counts,
exposure=exposure,
background=background,
c_0=c_0,
kernel=kernel,
flux=flux,
method=p['method'],
error_method=error_method,
threshold=p['threshold'],
error_sigma=p['error_sigma'],
ul_method=ul_method,
ul_sigma=p['ul_sigma'],
rtol=p['rtol']
)
x, y = np.where(mask.data)
positions = list(zip(x, y))
with contextlib.closing(Pool(processes=p['n_jobs'])) as pool:
log.info('Using {} jobs to compute TS map.'.format(p['n_jobs']))
results = pool.map(wrap, positions)
# Set TS values at given positions
j, i = zip(*positions)
for name in ['ts', 'flux', 'niter']:
result[name].data[j, i] = [_[name] for _ in results]
if 'flux_err' in which:
result['flux_err'].data[j, i] = [_['flux_err'] for _ in results]
if 'flux_ul' in which:
result['flux_ul'].data[j, i] = [_['flux_ul'] for _ in results]
# Compute sqrt(TS) values
if 'sqrt_ts' in which:
result['sqrt_ts'] = self.sqrt_ts(result['ts'])
if downsampling_factor:
for name in which:
order = 0 if name == 'niter' else 1
result[name] = result[name].upsample(
factor=downsampling_factor,
preserve_counts=False,
order=order
)
result[name] = result[name].crop(crop_width=pad_width)
return result
